class NewProfileHandler(component): Inboxes = {"inbox" : "", "control" : "Shutdown the client stream", "_response": ""} Outboxes = {"outbox" : "", "signal" : "Shutdown signal", "log" : "log", "_request": ""} def __init__(self, base_dir, atompub): super(NewProfileHandler, self).__init__() self.base_dir = base_dir self.atompub = atompub def initComponents(self): self.client = SimpleHTTPClient() self.addChildren(self.client) self.link((self, '_request'), (self.client, 'inbox')) self.link((self.client, 'outbox'), (self, '_response')) self.client.activate() def main(self): yield self.initComponents() while 1: if self.dataReady("control"): mes = self.recv("control") if isinstance(mes, shutdownMicroprocess) or \ isinstance(mes, producerFinished): self.send(producerFinished(), "signal") break if self.dataReady("inbox"): feed = self.recv('inbox') for entry in feed.entries: for link in entry.links: if link.rel == 'edit-media' and link.type == 'application/xml': profile_name = urlparse(link.href)[2].rsplit('/', -1)[-1] pwd = ProfileManager.set_profile_password(self.base_dir, profile_name) profile = ProfileManager.load_profile(self.base_dir, self.atompub, profile_name) Client.register_jabber_user(self.atompub, profile_name.lower(), pwd, profile) params = {'url': link.href, 'method': 'DELETE'} self.send(params, '_request') continue if not self.anyReady(): self.pause() yield 1
def initComponents(self): sub = SubscribeTo("JID.%s" % self.session_id) self.link((sub, 'outbox'), (self, 'jid')) self.addChildren(sub) sub.activate() client = SimpleHTTPClient() self.addChildren(client) self.link((self, '_request'), (client, 'inbox')) self.link((client, 'outbox'), (self, '_response')) client.activate() return 1
def test_302response(self): responses = {} oldpath = 'old.addr' oldbody = 'nothing to see here' newpath = 'new.addr' newbody = 'found me!' responses['/' + oldpath] = dict(body=oldbody, code=302, locationAddr='http://localhost:%i/%s' % (PORT, newpath)) responses['/' + newpath] = dict( body=newbody, code=200, ) self.fakeHttpServer.setResponses(responses) p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, oldpath)), SimpleHTTPClient()) self.initializeSystem(p) self.assertEquals(newbody, self.get('outbox', timeout=30)) signalMessage = self.get('signal') self.assertTrue(isinstance(signalMessage, producerFinished)) self.assertFinished() self.assertOutboxEmpty('outbox') self.assertOutboxEmpty('signal')
def makeFeedParser(self, feedUrl): """ makeFeedParser(feedUrl) -> Pipeline It returns a pipeline which does not expect any input except for signals and sends the parsed data through the "outbox" outbox. """ started(feedUrl) return Pipeline( OneShot(feedUrl), SimpleHTTPClient( ), # TODO: SimpleHTTPClient doesn't seem to have proxy support )
def main(self): uri = self.request.get('uri-suffix', "/") p = likefile(SimpleHTTPClient()) p.put("http://kamaelia.sourceforge.net/%s" % uri) pagedata = p.get() p.shutdown() print pagedata resource = { "type": "text/html", "statuscode": "200", } self.send(resource, "outbox") page = {"data": pagedata} self.send(page, "outbox") self.send(Axon.Ipc.producerFinished(self), "signal")
def _test200response(self, body, timeout): responses = {} path = 'foo' responses['/' + path] = dict( body=body, contentType='text', code=200, ) self.fakeHttpServer.setResponses(responses) p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, path)), SimpleHTTPClient()) self.initializeSystem(p) self.assertEquals(body, self.get('outbox', timeout=timeout)) signalMessage = self.get('signal') self.assertTrue(isinstance(signalMessage, producerFinished)) self.assertFinished() self.assertOutboxEmpty('outbox') self.assertOutboxEmpty('signal')
def testNoAnswer(self): # This test fails because of a bug in # Kamaelia.Protocol.HTTP.HTTPParser.HTTPParser.getInitialLine # the self.shouldShutdown does not handle the fact that the # provider might have finished, sending a producerFinished # message self.fakeHttpServer.stop() self.fakeHttpServer.join() # Now there is no server path = 'server.was.shutdown' p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, path)), SimpleHTTPClient()) self.initializeSystem(p) signalMessage = self.get('signal', timeout=20) self.assertTrue(isinstance(signalMessage, producerFinished)) # Is this actually correct? message = self.get('outbox') self.assertEquals('', message) self.assertFinished() self.assertOutboxEmpty('outbox') self.assertOutboxEmpty('signal')
def test200withoutLength(self): # This test fails because of a bug in # Kamaelia.Protocol.HTTP.HTTPParser.HTTPParser.getBodyDependingOnHalfClose responses = {} path = 'foo' body = 'whatever' path = 'without.length' responses['/' + path] = dict( body=body, contentType='text', code=200, dontProvideLength=True, ) self.fakeHttpServer.setResponses(responses) p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, path)), SimpleHTTPClient()) self.initializeSystem(p) self.assertEquals(body, self.get('outbox', timeout=30)) signalMessage = self.get('signal') self.assertTrue(isinstance(signalMessage, producerFinished)) self.assertFinished() self.assertOutboxEmpty('outbox') self.assertOutboxEmpty('signal')
def initComponents(self): sub = SubscribeTo("JID.%s" % self.session_id) self.link((sub, 'outbox'), (self, 'jid')) self.addChildren(sub) sub.activate() feedreader = FeedReaderComponent(use_etags=False) self.addChildren(feedreader) feedreader.activate() client = SimpleHTTPClient() self.addChildren(client) self.link((self, '_feedrequest'), (client, 'inbox')) self.link((client, 'outbox'), (feedreader, 'inbox')) self.link((feedreader, 'outbox'), (self, '_feedresponse')) client.activate() client = SimpleHTTPClient() self.addChildren(client) self.link((self, '_delrequest'), (client, 'inbox')) self.link((client, 'outbox'), (self, '_delresponse')) client.activate() return 1
def initComponents(self): self.client = SimpleHTTPClient() self.addChildren(self.client) self.link((self, '_request'), (self.client, 'inbox')) self.link((self.client, 'outbox'), (self, '_response')) self.client.activate()
self.send(shutdown(), "signal") # and tell the HTTP client that we've finished which should cause # it to terminate gracefully, of its own accord self.send(producerFinished(self), "fetchersignal") __kamaelia_components__ = (TorrentTkWindow, ) if __name__ == "__main__": from Kamaelia.Chassis.Graphline import Graphline from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient Graphline( gui=TorrentTkWindow(), # our GUI httpclient=SimpleHTTPClient(), # used to download .torrent files backend=TorrentPatron(), # our BitTorrent client backend linkages={ ("backend", "outbox"): ("gui", "inbox"), ("gui", "outbox"): ("backend", "inbox"), ("gui", "signal"): ("backend", "control"), ("gui", "fetchersignal"): ("httpclient", "control"), ("gui", "fetcher"): ("httpclient", "inbox"), ("httpclient", "outbox"): ("backend", "inbox"), }).run() # BASIC TOPOLOGY # ------------------------------- # # httpclient <-- gui <--> backend # \ /
split=Fanout(["toHTTP-POSTer", "toTorrentPatron"]), # fileupload uploads each message it receives to a script on a webserver. fileupload=Pipeline( # convert messages received to HTTP POST requests # for the URL trackerpostuploader # (with the contents of the message as the payload/request body) HTTPMakePostRequest(trackerpostuploader), # SimpleHTTPClient then makes these requests # (connecting to the server given in the URL and sending the # POST request in an HTTP request format) # the result of which is (assuming a suitable upload script on the # webserver) .torrent files sent to this pipeline as messages # are uploaded to the webserver SimpleHTTPClient()), # TorrentPatron is a BitTorrent client which will automatically # upload chunks of the stream to users that request them bittorrentpatron=TorrentPatron(), linkages={ ("streamin", "outbox"): ("split", "inbox"), ("split", "toHTTP-POSTer"): ("fileupload", "inbox"), ("split", "toTorrentPatron"): ("bittorrentpatron", "inbox"), }).run() # BASIC TOPOLOGY # ------------------------------- # # streamin --->split----> fileupload # \
class RegistrationHandler(component): Inboxes = {"inbox" : "headstock.api.registration.Registration", "error" : "headstock.api.registration.Registration", "control" : "Shutdown the client stream", "_response": ""} Outboxes = {"outbox" : "headstock.api.registration.Registration", "signal" : "Shutdown signal", "log" : "log", "_request": ""} def __init__(self, username, password, session_id, profile): super(RegistrationHandler, self).__init__() self.username = username self.password = password self.profile = profile self.registration_id = None self.session_id = session_id def initComponents(self): self.client = SimpleHTTPClient() self.addChildren(self.client) self.link((self, '_request'), (self.client, 'inbox')) self.link((self.client, 'outbox'), (self, '_response')) self.client.activate() def main(self): yield self.initComponents() while 1: if self.dataReady("control"): mes = self.recv("control") if isinstance(mes, shutdownMicroprocess) or isinstance(mes, producerFinished): self.send(producerFinished(), "signal") break if self.dataReady("_response"): self.recv("_response") if self.dataReady("inbox"): r = self.recv('inbox') if r.registered: self.send("'%s' is already a registered username." % r.infos[u'username'], 'log') c = Client.Sessions[r.infos[u'username']] c.shutdown() del Client.Sessions[r.infos[u'username']] elif self.registration_id == r.stanza_id: c = Client.Sessions[r.infos[u'username']] c.shutdown() del Client.Sessions[r.infos[u'username']] if '.microblogging' not in r.infos[u'username']: body = self.profile.xml() params = {'url': 'http://localhost:8080/profile/', 'method': 'POST', 'postbody': body, 'extraheaders': {'content-type': 'application/xml', 'slug': self.profile.username, 'content-length': len(body)}} self.send(params, '_request') else: if 'username' in r.infos and 'password' in r.infos: self.registration_id = generate_unique() r = Registration(type=u'set', stanza_id=self.registration_id) r.infos[u'username'] = self.username r.infos[u'password'] = self.password self.send(r, 'outbox') if self.dataReady("error"): r = self.recv('error') if r.error.code == '409': self.send("'%s' is already a registered username." % r.infos[u'username'], 'log') c = Client.Sessions[r.infos[u'username']] c.shutdown() del Client.Sessions[r.infos[u'username']] self.send(r.error, 'log') if not self.anyReady(): self.pause() yield 1
self.send(Axon.Ipc.shutdownMicroprocess(), "control") # should be last, this is what we honour else: raise AttributeError, "shutdown was previously called, or we were never activated." self.inputComponent.isDead.wait(1) if not self.inputComponent.isDead.isSet( ): # we timed out instead of someone else setting the flag warnings.warn( "Timed out waiting on shutdown confirmation, may not be dead.") self.alive = False def __del__(self): if self.alive: self.shutdown() if __name__ == "__main__": background = schedulerThread(slowmo=0.01).start() time.sleep(0.1) from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient import time p = LikeFile(SimpleHTTPClient()) p.activate() p.send("http://google.com") p.send("http://slashdot.org") p.send("http://whatismyip.org") google = p.recv() slashdot = p.recv() whatismyip = p.recv() print "google is", len(google), "bytes long, and slashdot is", len( slashdot), "bytes long. Also, our IP address is:", whatismyip
class HelloPusher(component): def __init__(self): self.time = time.time() + 0.1 super(HelloPusher, self).__init__() def main(self): while True: if time.time() > self.time: self.time = time.time() + 0.1 self.send("hello, world!\n", 'outbox') yield 1 Pipeline(HelloPusher(), ConsoleEchoer(), ConsoleReader(">>> ", ""), SimpleHTTPClient(), ConsoleEchoer()).run() # # A comment - the first two items in the pipeline will spam the console whenever the scheduler is running normally. The rest # of the pipeline is processing http requests normally. However, with an imposed delay on DNS lookups, the entire scheduler halts. # Somewhere in simpleHTTPClient, then, gethostbyname() is being called directly. # # For reference - here is what I used to delay DNS lookups: # in /etc/resolv.conf: # # 192.168.0.1 # 127.0.0.1 # in a terminal:
class ListSplit(component): def main(self): while 1: yield 1 while self.dataReady("inbox"): msg = self.recv("inbox") for m in msg: self.send(m, "outbox") self.pause() Graphline(usersuppliedurls=ConsoleReader(eol=""), splitter=Fanout(["toHTTPClient", "toCorrectRelativeLinks"]), splittertwo=Fanout(["toSplitterOne", "toEchoer"]), newlinesuffixadder=PureTransformer(lambda x: x + "\n"), httpclient=SimpleHTTPClient(), htmlprocessor=HTMLProcess(), linkextractor=ExtractLinks(), linkcorrector=CorrectRelativeLinks(), listsplitter=ListSplit(), prefixmatcher=PureTransformer(suffixMatchOnly), newurlechoer=ConsoleEchoer(), unseenonly=UnseenOnly(), linkages={ ("usersuppliedurls", "outbox"): ("unseenonly", "inbox"), ("newurlechoer", "outbox"): ("splitter", "inbox"), ("splitter", "toHTTPClient"): ("httpclient", "inbox"), ("splitter", "toCorrectRelativeLinks"): ("linkcorrector", "sourceurl"), ("httpclient", "outbox"): ("htmlprocessor", "inbox"), ("htmlprocessor", "outbox"): ("linkextractor", "inbox"),
def P2PStreamer(torrentsfolder): """\ Arguments: - torrentsfolder, e.g. "http://my.server.example.org/radioFoo/" """ # Create a pipeline of components whose net result is to output the contents of a certain URL # (torrentsfolder + metafilename) every 60 seconds (the contents at the time of output, i.e. # it fetches the page every 60 seconds). poller = Pipeline( # This generates a message every 60 seconds to wake TriggeredSource # allowing us to poll the meta file without busy-waiting. CheapAndCheerfulClock(60.0), # This sends the string (torrentsfolder + "meta.txt") every time it receives a message # This string will be the URL of the meta file on the torrent hosting website # e.g. "http://my.server.example.org/radioFoo/meta.txt" TriggeredSource(torrentsfolder + "meta.txt"), # SimpleHTTPClient retrieves the resource specified by the message it receives, # which will be URL string. # i.e. It fetches the page whose URL is (torrentsfolder + "meta.txt) (the string # produced by TriggeredSource) and forwards on the contents of that page. # The contents of that particular page will always be a number # (in the form of a decimal ASCII string) which represents the number of # 'chunks' of the stream that exist SimpleHTTPClient()) # As a whole, streamer acts like a normal streaming client, outputting the contents of # a stream to its outbox, although in much larger chunks with longer in between chunks # than for a typical stream. streamer = Pipeline( # fetch the P2P-stream meta file every 60 seconds and send its contents on poller, # PartsFilenameGenerator uses the number retrived by poller # i.e. the number of chunks/torrents in the stream # to generate the URLs of all the .torrent files # (torrent metadata files) that make up the stream. # (They will have been named 1.torrent, # 2.torrent, 3.torrent ... etc. on the server). PartsFilenameGenerator(torrentsfolder, ".torrent"), # Download these .torrent files (each message received by resourcefetcher # will be the URL of one .torrent file it should download). The # contents of the page downloaded it forwarded on to the next component. # NOTE: this downloads the .torrent file (metadata about part of the # stream) not the stream itself SimpleHTTPClient(), # now use BitTorrent to download the stream itself using the # metadata retrieved from .torrent files (each has information about a # section of the stream - a section itself is typically a few MB of data) # (TorrentPatron is a BitTorrent client component) TorrentPatron(), # output the names of the chunks of the stream as soon as they and # all previous chunks have been downloaded StreamReconstructor(), # read the contents of these chunks (files) TriggeredFileReader(), ) return streamer
"control") # should be last, this is what we honour else: raise AttributeError, "shutdown was previously called, or we were never activated." self.inputComponent.isDead.wait(1) if not self.inputComponent.isDead.isSet( ): # we timed out instead of someone else setting the flag warnings.warn( "Timed out waiting on shutdown confirmation, may not be dead.") self.alive = False def __del__(self): if self.alive: self.shutdown() if __name__ == "__main__": #doesn't actually work as of now background = background().start() time.sleep(0.1) from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient import time p = likefile(SimpleHTTPClient()) p.put("http://google.com") p.put("http://slashdot.org") p.put("http://whatismyip.org") google = p.get() slashdot = p.get() whatismyip = p.get() p.shutdown() print "google is", len(google), "bytes long, and slashdot is", len( slashdot), "bytes long. Also, our IP address is:", whatismyip
#!/usr/bin/env python # -*- coding: utf-8 -*- # Copyright 2010 British Broadcasting Corporation and Kamaelia Contributors(1) # # (1) Kamaelia Contributors are listed in the AUTHORS file and at # http://www.kamaelia.org/AUTHORS - please extend this file, # not this notice. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient from Kamaelia.Util.Console import ConsoleReader, ConsoleEchoer from Kamaelia.Chassis.Pipeline import Pipeline Pipeline(ConsoleReader(">>> ", ""), SimpleHTTPClient(), ConsoleEchoer()).run()