class NewProfileHandler(component):
Inboxes = {"inbox" : "",
"control" : "Shutdown the client stream",
"_response": ""}
Outboxes = {"outbox" : "",
"signal" : "Shutdown signal",
"log" : "log",
"_request": ""}
def __init__(self, base_dir, atompub):
super(NewProfileHandler, self).__init__()
self.base_dir = base_dir
self.atompub = atompub
def initComponents(self):
self.client = SimpleHTTPClient()
self.addChildren(self.client)
self.link((self, '_request'), (self.client, 'inbox'))
self.link((self.client, 'outbox'), (self, '_response'))
self.client.activate()
def main(self):
yield self.initComponents()
while 1:
if self.dataReady("control"):
mes = self.recv("control")
if isinstance(mes, shutdownMicroprocess) or \
isinstance(mes, producerFinished):
self.send(producerFinished(), "signal")
break
if self.dataReady("inbox"):
feed = self.recv('inbox')
for entry in feed.entries:
for link in entry.links:
if link.rel == 'edit-media' and link.type == 'application/xml':
profile_name = urlparse(link.href)[2].rsplit('/', -1)[-1]
pwd = ProfileManager.set_profile_password(self.base_dir, profile_name)
profile = ProfileManager.load_profile(self.base_dir, self.atompub, profile_name)
Client.register_jabber_user(self.atompub, profile_name.lower(), pwd, profile)
params = {'url': link.href, 'method': 'DELETE'}
self.send(params, '_request')
continue
if not self.anyReady():
self.pause()
yield 1
def initComponents(self):
sub = SubscribeTo("JID.%s" % self.session_id)
self.link((sub, 'outbox'), (self, 'jid'))
self.addChildren(sub)
sub.activate()
client = SimpleHTTPClient()
self.addChildren(client)
self.link((self, '_request'), (client, 'inbox'))
self.link((client, 'outbox'), (self, '_response'))
client.activate()
return 1
def test_302response(self):
responses = {}
oldpath = 'old.addr'
oldbody = 'nothing to see here'
newpath = 'new.addr'
newbody = 'found me!'
responses['/' + oldpath] = dict(body=oldbody,
code=302,
locationAddr='http://localhost:%i/%s' %
(PORT, newpath))
responses['/' + newpath] = dict(
body=newbody,
code=200,
)
self.fakeHttpServer.setResponses(responses)
p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, oldpath)),
SimpleHTTPClient())
self.initializeSystem(p)
self.assertEquals(newbody, self.get('outbox', timeout=30))
signalMessage = self.get('signal')
self.assertTrue(isinstance(signalMessage, producerFinished))
self.assertFinished()
self.assertOutboxEmpty('outbox')
self.assertOutboxEmpty('signal')
def makeFeedParser(self, feedUrl):
"""
makeFeedParser(feedUrl) -> Pipeline
It returns a pipeline which does not expect any input except for signals and
sends the parsed data through the "outbox" outbox.
"""
started(feedUrl)
return Pipeline(
OneShot(feedUrl),
SimpleHTTPClient(
), # TODO: SimpleHTTPClient doesn't seem to have proxy support
)
def main(self):
uri = self.request.get('uri-suffix', "/")
p = likefile(SimpleHTTPClient())
p.put("http://kamaelia.sourceforge.net/%s" % uri)
pagedata = p.get()
p.shutdown()
print pagedata
resource = {
"type": "text/html",
"statuscode": "200",
}
self.send(resource, "outbox")
page = {"data": pagedata}
self.send(page, "outbox")
self.send(Axon.Ipc.producerFinished(self), "signal")
def _test200response(self, body, timeout):
responses = {}
path = 'foo'
responses['/' + path] = dict(
body=body,
contentType='text',
code=200,
)
self.fakeHttpServer.setResponses(responses)
p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, path)),
SimpleHTTPClient())
self.initializeSystem(p)
self.assertEquals(body, self.get('outbox', timeout=timeout))
signalMessage = self.get('signal')
self.assertTrue(isinstance(signalMessage, producerFinished))
self.assertFinished()
self.assertOutboxEmpty('outbox')
self.assertOutboxEmpty('signal')
def testNoAnswer(self):
# This test fails because of a bug in
# Kamaelia.Protocol.HTTP.HTTPParser.HTTPParser.getInitialLine
# the self.shouldShutdown does not handle the fact that the
# provider might have finished, sending a producerFinished
# message
self.fakeHttpServer.stop()
self.fakeHttpServer.join()
# Now there is no server
path = 'server.was.shutdown'
p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, path)),
SimpleHTTPClient())
self.initializeSystem(p)
signalMessage = self.get('signal', timeout=20)
self.assertTrue(isinstance(signalMessage, producerFinished))
# Is this actually correct?
message = self.get('outbox')
self.assertEquals('', message)
self.assertFinished()
self.assertOutboxEmpty('outbox')
self.assertOutboxEmpty('signal')
def test200withoutLength(self):
# This test fails because of a bug in
# Kamaelia.Protocol.HTTP.HTTPParser.HTTPParser.getBodyDependingOnHalfClose
responses = {}
path = 'foo'
body = 'whatever'
path = 'without.length'
responses['/' + path] = dict(
body=body,
contentType='text',
code=200,
dontProvideLength=True,
)
self.fakeHttpServer.setResponses(responses)
p = Pipeline(OneShot('http://localhost:%i/%s' % (PORT, path)),
SimpleHTTPClient())
self.initializeSystem(p)
self.assertEquals(body, self.get('outbox', timeout=30))
signalMessage = self.get('signal')
self.assertTrue(isinstance(signalMessage, producerFinished))
self.assertFinished()
self.assertOutboxEmpty('outbox')
self.assertOutboxEmpty('signal')
def initComponents(self):
sub = SubscribeTo("JID.%s" % self.session_id)
self.link((sub, 'outbox'), (self, 'jid'))
self.addChildren(sub)
sub.activate()
feedreader = FeedReaderComponent(use_etags=False)
self.addChildren(feedreader)
feedreader.activate()
client = SimpleHTTPClient()
self.addChildren(client)
self.link((self, '_feedrequest'), (client, 'inbox'))
self.link((client, 'outbox'), (feedreader, 'inbox'))
self.link((feedreader, 'outbox'), (self, '_feedresponse'))
client.activate()
client = SimpleHTTPClient()
self.addChildren(client)
self.link((self, '_delrequest'), (client, 'inbox'))
self.link((client, 'outbox'), (self, '_delresponse'))
client.activate()
return 1
def initComponents(self):
self.client = SimpleHTTPClient()
self.addChildren(self.client)
self.link((self, '_request'), (self.client, 'inbox'))
self.link((self.client, 'outbox'), (self, '_response'))
self.client.activate()
self.send(shutdown(), "signal")
# and tell the HTTP client that we've finished which should cause
# it to terminate gracefully, of its own accord
self.send(producerFinished(self), "fetchersignal")
__kamaelia_components__ = (TorrentTkWindow, )
if __name__ == "__main__":
from Kamaelia.Chassis.Graphline import Graphline
from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient
Graphline(
gui=TorrentTkWindow(), # our GUI
httpclient=SimpleHTTPClient(), # used to download .torrent files
backend=TorrentPatron(), # our BitTorrent client backend
linkages={
("backend", "outbox"): ("gui", "inbox"),
("gui", "outbox"): ("backend", "inbox"),
("gui", "signal"): ("backend", "control"),
("gui", "fetchersignal"): ("httpclient", "control"),
("gui", "fetcher"): ("httpclient", "inbox"),
("httpclient", "outbox"): ("backend", "inbox"),
}).run()
# BASIC TOPOLOGY
# -------------------------------
#
# httpclient <-- gui <--> backend
# \ /
split=Fanout(["toHTTP-POSTer", "toTorrentPatron"]),
# fileupload uploads each message it receives to a script on a webserver.
fileupload=Pipeline(
# convert messages received to HTTP POST requests
# for the URL trackerpostuploader
# (with the contents of the message as the payload/request body)
HTTPMakePostRequest(trackerpostuploader),
# SimpleHTTPClient then makes these requests
# (connecting to the server given in the URL and sending the
# POST request in an HTTP request format)
# the result of which is (assuming a suitable upload script on the
# webserver) .torrent files sent to this pipeline as messages
# are uploaded to the webserver
SimpleHTTPClient()),
# TorrentPatron is a BitTorrent client which will automatically
# upload chunks of the stream to users that request them
bittorrentpatron=TorrentPatron(),
linkages={
("streamin", "outbox"): ("split", "inbox"),
("split", "toHTTP-POSTer"): ("fileupload", "inbox"),
("split", "toTorrentPatron"): ("bittorrentpatron", "inbox"),
}).run()
# BASIC TOPOLOGY
# -------------------------------
#
# streamin --->split----> fileupload
# \
class RegistrationHandler(component):
Inboxes = {"inbox" : "headstock.api.registration.Registration",
"error" : "headstock.api.registration.Registration",
"control" : "Shutdown the client stream",
"_response": ""}
Outboxes = {"outbox" : "headstock.api.registration.Registration",
"signal" : "Shutdown signal",
"log" : "log",
"_request": ""}
def __init__(self, username, password, session_id, profile):
super(RegistrationHandler, self).__init__()
self.username = username
self.password = password
self.profile = profile
self.registration_id = None
self.session_id = session_id
def initComponents(self):
self.client = SimpleHTTPClient()
self.addChildren(self.client)
self.link((self, '_request'), (self.client, 'inbox'))
self.link((self.client, 'outbox'), (self, '_response'))
self.client.activate()
def main(self):
yield self.initComponents()
while 1:
if self.dataReady("control"):
mes = self.recv("control")
if isinstance(mes, shutdownMicroprocess) or isinstance(mes, producerFinished):
self.send(producerFinished(), "signal")
break
if self.dataReady("_response"):
self.recv("_response")
if self.dataReady("inbox"):
r = self.recv('inbox')
if r.registered:
self.send("'%s' is already a registered username." % r.infos[u'username'], 'log')
c = Client.Sessions[r.infos[u'username']]
c.shutdown()
del Client.Sessions[r.infos[u'username']]
elif self.registration_id == r.stanza_id:
c = Client.Sessions[r.infos[u'username']]
c.shutdown()
del Client.Sessions[r.infos[u'username']]
if '.microblogging' not in r.infos[u'username']:
body = self.profile.xml()
params = {'url': 'http://localhost:8080/profile/',
'method': 'POST', 'postbody': body,
'extraheaders': {'content-type': 'application/xml',
'slug': self.profile.username,
'content-length': len(body)}}
self.send(params, '_request')
else:
if 'username' in r.infos and 'password' in r.infos:
self.registration_id = generate_unique()
r = Registration(type=u'set', stanza_id=self.registration_id)
r.infos[u'username'] = self.username
r.infos[u'password'] = self.password
self.send(r, 'outbox')
if self.dataReady("error"):
r = self.recv('error')
if r.error.code == '409':
self.send("'%s' is already a registered username." % r.infos[u'username'], 'log')
c = Client.Sessions[r.infos[u'username']]
c.shutdown()
del Client.Sessions[r.infos[u'username']]
self.send(r.error, 'log')
if not self.anyReady():
self.pause()
yield 1
self.send(Axon.Ipc.shutdownMicroprocess(),
"control") # should be last, this is what we honour
else:
raise AttributeError, "shutdown was previously called, or we were never activated."
self.inputComponent.isDead.wait(1)
if not self.inputComponent.isDead.isSet(
): # we timed out instead of someone else setting the flag
warnings.warn(
"Timed out waiting on shutdown confirmation, may not be dead.")
self.alive = False
def __del__(self):
if self.alive:
self.shutdown()
if __name__ == "__main__":
background = schedulerThread(slowmo=0.01).start()
time.sleep(0.1)
from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient
import time
p = LikeFile(SimpleHTTPClient())
p.activate()
p.send("http://google.com")
p.send("http://slashdot.org")
p.send("http://whatismyip.org")
google = p.recv()
slashdot = p.recv()
whatismyip = p.recv()
print "google is", len(google), "bytes long, and slashdot is", len(
slashdot), "bytes long. Also, our IP address is:", whatismyip
class HelloPusher(component):
def __init__(self):
self.time = time.time() + 0.1
super(HelloPusher, self).__init__()
def main(self):
while True:
if time.time() > self.time:
self.time = time.time() + 0.1
self.send("hello, world!\n", 'outbox')
yield 1
Pipeline(HelloPusher(), ConsoleEchoer(), ConsoleReader(">>> ", ""),
SimpleHTTPClient(), ConsoleEchoer()).run()
#
# A comment - the first two items in the pipeline will spam the console whenever the scheduler is running normally. The rest
# of the pipeline is processing http requests normally. However, with an imposed delay on DNS lookups, the entire scheduler halts.
# Somewhere in simpleHTTPClient, then, gethostbyname() is being called directly.
#
# For reference - here is what I used to delay DNS lookups:
# in /etc/resolv.conf:
# # 192.168.0.1
# 127.0.0.1
# in a terminal:
class ListSplit(component):
def main(self):
while 1:
yield 1
while self.dataReady("inbox"):
msg = self.recv("inbox")
for m in msg:
self.send(m, "outbox")
self.pause()
Graphline(usersuppliedurls=ConsoleReader(eol=""),
splitter=Fanout(["toHTTPClient", "toCorrectRelativeLinks"]),
splittertwo=Fanout(["toSplitterOne", "toEchoer"]),
newlinesuffixadder=PureTransformer(lambda x: x + "\n"),
httpclient=SimpleHTTPClient(),
htmlprocessor=HTMLProcess(),
linkextractor=ExtractLinks(),
linkcorrector=CorrectRelativeLinks(),
listsplitter=ListSplit(),
prefixmatcher=PureTransformer(suffixMatchOnly),
newurlechoer=ConsoleEchoer(),
unseenonly=UnseenOnly(),
linkages={
("usersuppliedurls", "outbox"): ("unseenonly", "inbox"),
("newurlechoer", "outbox"): ("splitter", "inbox"),
("splitter", "toHTTPClient"): ("httpclient", "inbox"),
("splitter", "toCorrectRelativeLinks"):
("linkcorrector", "sourceurl"),
("httpclient", "outbox"): ("htmlprocessor", "inbox"),
("htmlprocessor", "outbox"): ("linkextractor", "inbox"),
def P2PStreamer(torrentsfolder):
"""\
Arguments:
- torrentsfolder, e.g. "http://my.server.example.org/radioFoo/"
"""
# Create a pipeline of components whose net result is to output the contents of a certain URL
# (torrentsfolder + metafilename) every 60 seconds (the contents at the time of output, i.e.
# it fetches the page every 60 seconds).
poller = Pipeline(
# This generates a message every 60 seconds to wake TriggeredSource
# allowing us to poll the meta file without busy-waiting.
CheapAndCheerfulClock(60.0),
# This sends the string (torrentsfolder + "meta.txt") every time it receives a message
# This string will be the URL of the meta file on the torrent hosting website
# e.g. "http://my.server.example.org/radioFoo/meta.txt"
TriggeredSource(torrentsfolder + "meta.txt"),
# SimpleHTTPClient retrieves the resource specified by the message it receives,
# which will be URL string.
# i.e. It fetches the page whose URL is (torrentsfolder + "meta.txt) (the string
# produced by TriggeredSource) and forwards on the contents of that page.
# The contents of that particular page will always be a number
# (in the form of a decimal ASCII string) which represents the number of
# 'chunks' of the stream that exist
SimpleHTTPClient())
# As a whole, streamer acts like a normal streaming client, outputting the contents of
# a stream to its outbox, although in much larger chunks with longer in between chunks
# than for a typical stream.
streamer = Pipeline(
# fetch the P2P-stream meta file every 60 seconds and send its contents on
poller,
# PartsFilenameGenerator uses the number retrived by poller
# i.e. the number of chunks/torrents in the stream
# to generate the URLs of all the .torrent files
# (torrent metadata files) that make up the stream.
# (They will have been named 1.torrent,
# 2.torrent, 3.torrent ... etc. on the server).
PartsFilenameGenerator(torrentsfolder, ".torrent"),
# Download these .torrent files (each message received by resourcefetcher
# will be the URL of one .torrent file it should download). The
# contents of the page downloaded it forwarded on to the next component.
# NOTE: this downloads the .torrent file (metadata about part of the
# stream) not the stream itself
SimpleHTTPClient(),
# now use BitTorrent to download the stream itself using the
# metadata retrieved from .torrent files (each has information about a
# section of the stream - a section itself is typically a few MB of data)
# (TorrentPatron is a BitTorrent client component)
TorrentPatron(),
# output the names of the chunks of the stream as soon as they and
# all previous chunks have been downloaded
StreamReconstructor(),
# read the contents of these chunks (files)
TriggeredFileReader(),
)
return streamer
"control") # should be last, this is what we honour
else:
raise AttributeError, "shutdown was previously called, or we were never activated."
self.inputComponent.isDead.wait(1)
if not self.inputComponent.isDead.isSet(
): # we timed out instead of someone else setting the flag
warnings.warn(
"Timed out waiting on shutdown confirmation, may not be dead.")
self.alive = False
def __del__(self):
if self.alive:
self.shutdown()
if __name__ == "__main__":
#doesn't actually work as of now
background = background().start()
time.sleep(0.1)
from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient
import time
p = likefile(SimpleHTTPClient())
p.put("http://google.com")
p.put("http://slashdot.org")
p.put("http://whatismyip.org")
google = p.get()
slashdot = p.get()
whatismyip = p.get()
p.shutdown()
print "google is", len(google), "bytes long, and slashdot is", len(
slashdot), "bytes long. Also, our IP address is:", whatismyip
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2010 British Broadcasting Corporation and Kamaelia Contributors(1)
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
# http://www.kamaelia.org/AUTHORS - please extend this file,
# not this notice.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from Kamaelia.Protocol.HTTP.HTTPClient import SimpleHTTPClient
from Kamaelia.Util.Console import ConsoleReader, ConsoleEchoer
from Kamaelia.Chassis.Pipeline import Pipeline
Pipeline(ConsoleReader(">>> ", ""), SimpleHTTPClient(), ConsoleEchoer()).run()
