def test_Unplugging(self):
"""Plug will unplug and shutdown when child component dies."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1, 100):
execute.next()
#send shutdown msg
msg = producerFinished()
target._deliver(msg, "control")
for i in xrange(1, 100):
execute.next()
# verify it reached the target
self.assert_(target.controllog == [msg])
# verify the plug has shutdown
self.assert_(plug._isStopped())
# verify the plug has no linkages
self.assert_(not plug.postoffice.linkages)
# verify that splitter only has outboxes "outbox" and "signal" now
self.assert_(len(splitter.outboxes) == 2)
self.assert_("outbox" in splitter.outboxes)
self.assert_("signal" in splitter.outboxes)
def test_Unplugging(self):
"""Plug will unplug and shutdown when child component dies."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1,100):
execute.next()
#send shutdown msg
msg = producerFinished()
target._deliver(msg, "control")
for i in xrange(1,100):
execute.next()
# verify it reached the target
self.assert_(target.controllog == [msg])
# verify the plug has shutdown
self.assert_(plug._isStopped())
# verify the plug has no linkages
self.assert_(not plug.postoffice.linkages)
# verify that splitter only has outboxes "outbox" and "signal" now
self.assert_( len(splitter.outboxes) == 2)
self.assert_( "outbox" in splitter.outboxes)
self.assert_( "signal" in splitter.outboxes)
def setUp(self):
self.src = component()
self.dst = component()
self.dst2 = TestComponent()
self.controller = component()
self.split = Splitter()
self.runner = self.split.main()
self.linkin = linkage(self.src, self.split)
self.linkcont = linkage(self.controller,
self.split,
sinkbox="configuration")
self.links = [self.linkin, self.linkcont]
# self.linkout1 = linkage(self.split,self.dst)
# self.linkout2 = linkage(self.split,self.dst2, sourcebox="out2")
#
# -- NEW STUFF ------
#
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
self.execute = Axon.Scheduler.scheduler.run.main()
self.S = Splitter().activate()
self.D = component().activate()
self.D2 = TestComponent().activate()
self.W = component().activate()
self.W.link((self.W, "outbox"), (self.S, "configuration"))
def test_SplitterDelLinkBoth(self):
"""Sending an delSink message to splitter unlinks in the extra outbox and signal"""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
split = Splitter().activate()
target1 = Axon.Component.component().activate()
target2 = Axon.Component.component().activate()
target1.link( (split,"outbox"), (target1, "inbox") )
target1.link( (split,"signal"), (target1, "control") )
addmsg = addsink(target2, "inbox", "control")
split._deliver(addmsg, "configuration")
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1,10):
execute.next()
for i in xrange(1,10):
if i == 5:
delmsg = removesink(target2, "inbox", "control")
split._deliver(delmsg, "configuration")
split._deliver(i, "inbox")
split._deliver(10+i, "control")
for j in xrange(1,10):
execute.next()
for i in xrange(1,40):
execute.next()
# verify that the data has made it to the targets
for i in xrange(1,5):
self.assert_(target1.dataReady("inbox"))
self.assert_(target1.dataReady("control"))
self.assert_(i == target1.recv("inbox"))
self.assert_(10+i == target1.recv("control"))
self.assert_(target2.dataReady("inbox"))
self.assert_(target2.dataReady("control"))
self.assert_(i == target2.recv("inbox"))
self.assert_(10+i == target2.recv("control"))
for i in xrange(5,10):
self.assert_(target1.dataReady("inbox"))
self.assert_(target1.dataReady("control"))
self.assert_(i == target1.recv("inbox"))
self.assert_(10+i == target1.recv("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(not target2.dataReady("control"))
# verify there is nothing left
self.assert_(not target1.dataReady("inbox"))
self.assert_(not target1.dataReady("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(not target2.dataReady("control"))
def test_SplitterDelLinkBoth(self):
"""Sending an delSink message to splitter unlinks in the extra outbox and signal"""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
split = Splitter().activate()
target1 = Axon.Component.component().activate()
target2 = Axon.Component.component().activate()
target1.link((split, "outbox"), (target1, "inbox"))
target1.link((split, "signal"), (target1, "control"))
addmsg = addsink(target2, "inbox", "control")
split._deliver(addmsg, "configuration")
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1, 10):
execute.next()
for i in xrange(1, 10):
if i == 5:
delmsg = removesink(target2, "inbox", "control")
split._deliver(delmsg, "configuration")
split._deliver(i, "inbox")
split._deliver(10 + i, "control")
for j in xrange(1, 10):
execute.next()
for i in xrange(1, 40):
execute.next()
# verify that the data has made it to the targets
for i in xrange(1, 5):
self.assert_(target1.dataReady("inbox"))
self.assert_(target1.dataReady("control"))
self.assert_(i == target1.recv("inbox"))
self.assert_(10 + i == target1.recv("control"))
self.assert_(target2.dataReady("inbox"))
self.assert_(target2.dataReady("control"))
self.assert_(i == target2.recv("inbox"))
self.assert_(10 + i == target2.recv("control"))
for i in xrange(5, 10):
self.assert_(target1.dataReady("inbox"))
self.assert_(target1.dataReady("control"))
self.assert_(i == target1.recv("inbox"))
self.assert_(10 + i == target1.recv("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(not target2.dataReady("control"))
# verify there is nothing left
self.assert_(not target1.dataReady("inbox"))
self.assert_(not target1.dataReady("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(not target2.dataReady("control"))
def test_PluggingInAndTxfer(self):
"""Plug instantiated with splitter and component and passes data through to component."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1, 1000):
execute.next()
#pass some data in
for i in xrange(1, 10):
splitter._deliver(i, "inbox")
splitter._deliver(10 + i, "control")
for i in xrange(1, 100):
execute.next()
# verify it reached the target
self.assert_(target.inboxlog == range(1, 10))
self.assert_(target.controllog == range(11, 20))
def test_PluggingInAndTxfer(self):
"""Plug instantiated with splitter and component and passes data through to component."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1,1000):
execute.next()
#pass some data in
for i in xrange(1,10):
splitter._deliver(i, "inbox")
splitter._deliver(10+i, "control")
for i in xrange(1,100):
execute.next()
# verify it reached the target
self.assert_(target.inboxlog == range(1,10))
self.assert_(target.controllog == range(11,20))
def setUp(self):
self.src = component()
self.dst = component()
self.dst2 = TestComponent()
self.controller = component()
self.split = Splitter()
self.runner = self.split.main()
self.linkin = linkage(self.src,self.split)
self.linkcont = linkage(self.controller, self.split, sinkbox="configuration")
self.links = [self.linkin, self.linkcont]
# self.linkout1 = linkage(self.split,self.dst)
# self.linkout2 = linkage(self.split,self.dst2, sourcebox="out2")
#
# -- NEW STUFF ------
#
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
self.execute = Axon.Scheduler.scheduler.run.main()
self.S = Splitter().activate()
self.D = component().activate()
self.D2 = TestComponent().activate()
self.W = component().activate()
self.W.link( (self.W, "outbox"), (self.S, "configuration") )
def Tee(*destinations):
splitter = PlugSplitter()
plugs = {}
i = 1
for dest in destinations:
plugs[str(i)] = Plug(splitter, dest)
i = i + 1
return Graphline(splitter=splitter,
linkages={
("", "inbox"): ("splitter", "inbox"),
("", "control"): ("splitter", "control"),
("splitter", "outbox"): ("", "outbox"),
("splitter", "signal"): ("", "signal")
},
**plugs)
def test_SplitterAddLinkSignalOnly(self):
"""Sending an addSink message to splitter links in an extra signal"""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
split = Splitter().activate()
target1 = Axon.Component.component().activate()
target2 = Axon.Component.component().activate()
target1.link( (split,"outbox"), (target1, "inbox") )
target1.link( (split,"signal"), (target1, "control") )
addmsg = addsink(target2, None, "control")
split._deliver(addmsg, "configuration")
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1,10):
execute.next()
for i in xrange(1,10):
split._deliver(i, "inbox")
split._deliver(10+i, "control")
for j in xrange(1,10):
execute.next()
# verify that the data has made it to the targets
for i in xrange(1,10):
self.assert_(target1.dataReady("inbox"))
self.assert_(target1.dataReady("control"))
self.assert_(i == target1.recv("inbox"))
self.assert_(10+i == target1.recv("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(target2.dataReady("control"))
self.assert_(10+i == target2.recv("control"))
# verify there is nothing left
self.assert_(not target1.dataReady("inbox"))
self.assert_(not target1.dataReady("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(not target2.dataReady("control"))
def test_SplitterAddLinkSignalOnly(self):
"""Sending an addSink message to splitter links in an extra signal"""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
split = Splitter().activate()
target1 = Axon.Component.component().activate()
target2 = Axon.Component.component().activate()
target1.link((split, "outbox"), (target1, "inbox"))
target1.link((split, "signal"), (target1, "control"))
addmsg = addsink(target2, None, "control")
split._deliver(addmsg, "configuration")
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1, 10):
execute.next()
for i in xrange(1, 10):
split._deliver(i, "inbox")
split._deliver(10 + i, "control")
for j in xrange(1, 10):
execute.next()
# verify that the data has made it to the targets
for i in xrange(1, 10):
self.assert_(target1.dataReady("inbox"))
self.assert_(target1.dataReady("control"))
self.assert_(i == target1.recv("inbox"))
self.assert_(10 + i == target1.recv("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(target2.dataReady("control"))
self.assert_(10 + i == target2.recv("control"))
# verify there is nothing left
self.assert_(not target1.dataReady("inbox"))
self.assert_(not target1.dataReady("control"))
self.assert_(not target2.dataReady("inbox"))
self.assert_(not target2.dataReady("control"))
from Kamaelia.Util.Comparator import Comparator
from Kamaelia.Util.Splitter import Plug, PlugSplitter
from Kamaelia.Util.PassThrough import PassThrough
from Kamaelia.File.Writing import SimpleFileWriter
TS_FILE = "/home/matteh/dvb/2008-05-16 11.27.13 MUX1_EIT_TOT_TDT.ts"
# def AddInVersion():
# def transf(x):
# x["version"] = 0
# return x
# return PureTransformer(transf)
print "run a diff over the two output files to compare the results"
splitter=PlugSplitter()
Pipeline(
RateControlledFileReader(TS_FILE, readmode="bytes", rate=1000*1000, chunksize=188),
DVB_SoftDemuxer( {0x12:["outbox"]} ),
ReassemblePSITables(),
ParseEventInformationTable_Subset( \
actual_presentFollowing = True,
other_presentFollowing = True,
actual_schedule = True,
other_schedule = True,
),
splitter
).activate()
Plug(splitter, Pipeline(
def test_SplitterShutdown(self):
"""If producerFinished or shutdownMicroprocess is received on the 'control' inbox they are passed on and the component shuts down"""
for msg in [producerFinished(self), shutdownMicroprocess(self)]:
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for _ in xrange(0,10):
split.next()
self.assert_(0==len(split.outboxes["outbox"]))
self.assert_(0==len(split.outboxes["signal"]))
split._deliver( msg, "control" )
try:
for _ in xrange(0,10):
split.next()
self.fail()
except StopIteration:
pass
self.assert_(0==len(split.outboxes["outbox"]))
self.assert_(1==len(split.outboxes["signal"]))
# received = split._collect("signal")
received = Dummy.recv("control")
self.assert_( msg == received )
def test_PassThroughControlSignal(self):
"""Data sent to the inbox is sent on to the outbox"""
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for i in xrange(1,10):
split._deliver( i, "control" )
for _ in xrange(0,100):
split.next()
for i in xrange(1,10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0==len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_( i == Dummy.recv("control") )
for i in xrange(1,10):
split._deliver( i, "control" )
split.next()
split.next()
for _ in xrange(0,10):
split.next()
for i in xrange(1,10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0==len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_( i == Dummy.recv("control") )
def test_InstantiateNoArgs(self):
"""Splitter instantiated with no args is just passthrough"""
split = Splitter()
split.activate()
def test_SplitterShutdown(self):
"""If producerFinished or shutdownMicroprocess is received on the 'control' inbox they are passed on and the component shuts down"""
for msg in [producerFinished(self), shutdownMicroprocess(self)]:
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for _ in xrange(0, 10):
split.next()
self.assert_(0 == len(split.outboxes["outbox"]))
self.assert_(0 == len(split.outboxes["signal"]))
split._deliver(msg, "control")
try:
for _ in xrange(0, 10):
split.next()
self.fail()
except StopIteration:
pass
self.assert_(0 == len(split.outboxes["outbox"]))
self.assert_(1 == len(split.outboxes["signal"]))
# received = split._collect("signal")
received = Dummy.recv("control")
self.assert_(msg == received)
class Splitter_Test(unittest.TestCase):
def setUp(self):
self.src = component()
self.dst = component()
self.dst2 = TestComponent()
self.controller = component()
self.split = Splitter()
self.runner = self.split.main()
self.linkin = linkage(self.src,self.split)
self.linkcont = linkage(self.controller, self.split, sinkbox="configuration")
self.links = [self.linkin, self.linkcont]
# self.linkout1 = linkage(self.split,self.dst)
# self.linkout2 = linkage(self.split,self.dst2, sourcebox="out2")
#
# -- NEW STUFF ------
#
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
self.execute = Axon.Scheduler.scheduler.run.main()
self.S = Splitter().activate()
self.D = component().activate()
self.D2 = TestComponent().activate()
self.W = component().activate()
self.W.link( (self.W, "outbox"), (self.S, "configuration") )
def waitEvent(self, cycles, conditionFunc, *args):
i = 0
while 1:
self.execute.next()
if conditionFunc(*args):
break
else:
i += 1
if i > cycles:
raise Timeout(i)
def runCycles(self, cycles=20):
for _ in xrange(cycles):
self.execute.next()
def deliverhelper(self):
# Next bit not really needed due to direct delivery was implemented since then...
pass
def test_isacomponent(self):
"__init__ - Splitter is a component."
self.failUnless(isinstance(self.split,component))
def test_simplepassthrough_defaultbox(self):
"""mainBody - This test sets up a sink and checks it receives sent messages using the default box."""
self.W.send(addsink(self.D), "outbox")
data = [ 1,2,3,4,5,6]
for i in data:
self.S._deliver(i, "inbox")
try:
self.waitEvent(10, self.D.dataReady, "inbox" )
except Timeout, e:
self.fail("Data hasn't arrived after "+str(e.t)+" cycles")
R = []
while 1:
try:
R.append(self.D.recv("inbox"))
except:
break
self.assert_( R == data )
def initiateInternalSplitter(self):
self.internalSplitter = PlugSplitter()
self.link((self, '_signal'), (self.internalSplitter, 'control'))
self.addChildren(self.internalSplitter)
self.internalSplitter.activate()
class FeedParserFactory(Axon.Component.component):
"""
FeedParserFactory() -> FeedParserFactory object
It receives different feed URLs throught the "inbox" inbox
and returns each post parsed through the "outbox" outbox.
This class can handles multiple concurrent petitions, retrieves
the content of the feed and parses it with the feedparser library.
The result is a feedparser.FeedParserDict per each feed URL
provided.
"""
Inboxes = {
"inbox": "Strings representing different URLs of feeds",
"control": "From component...",
"_parsed-feeds":
"Parsed feeds retrieved from FeedParserFactory children",
}
Outboxes = {
"outbox":
"feedparser.FeedParserDict object representing a parsed feed",
"signal": "From component...",
"_signal": "To the internal parsers",
}
def __init__(self, **argd):
"""x.__init__(...) initializes x; see x.__class__.__doc__ for signature"""
super(FeedParserFactory, self).__init__(**argd)
self.mustStop = None
self.providerFinished = False
def makeFeedParser(self, feedUrl):
"""
makeFeedParser(feedUrl) -> Pipeline
It returns a pipeline which does not expect any input except for signals and
sends the parsed data through the "outbox" outbox.
"""
started(feedUrl)
return Pipeline(
OneShot(feedUrl),
SimpleHTTPClient(
), # TODO: SimpleHTTPClient doesn't seem to have proxy support
)
def checkControl(self):
while self.dataReady("control"):
msg = self.recv("control")
if isinstance(msg, producerFinished):
self.providerFinished = True
elif isinstance(msg, shutdownMicroprocess):
self.mustStop = msg
return self.mustStop, self.providerFinished
def handleChildTerminations(self): #taken from Carousel.py
for child in self.childComponents():
if child._isStopped():
self.removeChild(child)
def initiateInternalSplitter(self):
self.internalSplitter = PlugSplitter()
self.link((self, '_signal'), (self.internalSplitter, 'control'))
self.addChildren(self.internalSplitter)
self.internalSplitter.activate()
def linkChildToInternalSplitter(self, child):
forwarder = Forwarder()
plug = Plug(self.internalSplitter, forwarder)
plug.activate()
plug.link((plug, 'signal'), (child, 'control'))
child.link((child, 'signal'), (plug, 'control'))
def createChild(self, feed):
child = self.makeFeedParser(feed.url)
child = Pipeline(child, Feedparser(feed.url))
self.link((child, 'outbox'), (self, '_parsed-feeds'))
self.linkChildToInternalSplitter(child)
return child
def waitForChildren(self, signalMessage):
self.send(signalMessage, "_signal")
while len(self.childComponents()) > 0:
self.handleChildTerminations()
yield 1
def main(self):
self.initiateInternalSplitter()
yield 1
while True:
mustStop, providerFinished = self.checkControl()
if mustStop:
self.send(mustStop, "signal")
return
self.handleChildTerminations()
while self.dataReady("inbox"):
feed = self.recv("inbox")
child = self.createChild(feed)
self.addChildren(child)
child.activate()
while self.dataReady("_parsed-feeds"):
parseddata = self.recv("_parsed-feeds")
self.send(parseddata, "outbox")
if providerFinished and len(self.childComponents()) == 1:
# TODO: CHECK IF THIS IS THE PROBLEM
# It's actually only waiting for the plugsplitter
for _ in self.waitForChildren(producerFinished(self)):
yield 1
pfinished = producerFinished(self)
self.send(pfinished, "signal")
return
if not self.anyReady():
self.pause()
yield 1
class Splitter_Test(unittest.TestCase):
def setUp(self):
self.src = component()
self.dst = component()
self.dst2 = TestComponent()
self.controller = component()
self.split = Splitter()
self.runner = self.split.main()
self.linkin = linkage(self.src, self.split)
self.linkcont = linkage(self.controller,
self.split,
sinkbox="configuration")
self.links = [self.linkin, self.linkcont]
# self.linkout1 = linkage(self.split,self.dst)
# self.linkout2 = linkage(self.split,self.dst2, sourcebox="out2")
#
# -- NEW STUFF ------
#
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
self.execute = Axon.Scheduler.scheduler.run.main()
self.S = Splitter().activate()
self.D = component().activate()
self.D2 = TestComponent().activate()
self.W = component().activate()
self.W.link((self.W, "outbox"), (self.S, "configuration"))
def waitEvent(self, cycles, conditionFunc, *args):
i = 0
while 1:
self.execute.next()
if conditionFunc(*args):
break
else:
i += 1
if i > cycles:
raise Timeout(i)
def runCycles(self, cycles=20):
for _ in xrange(cycles):
self.execute.next()
def deliverhelper(self):
# Next bit not really needed due to direct delivery was implemented since then...
pass
def test_isacomponent(self):
"__init__ - Splitter is a component."
self.failUnless(isinstance(self.split, component))
def test_simplepassthrough_defaultbox(self):
"""mainBody - This test sets up a sink and checks it receives sent messages using the default box."""
self.W.send(addsink(self.D), "outbox")
data = [1, 2, 3, 4, 5, 6]
for i in data:
self.S._deliver(i, "inbox")
try:
self.waitEvent(10, self.D.dataReady, "inbox")
except Timeout, e:
self.fail("Data hasn't arrived after " + str(e.t) + " cycles")
R = []
while 1:
try:
R.append(self.D.recv("inbox"))
except:
break
self.assert_(R == data)
def test_InstantiateNoArgs(self):
"""Splitter instantiated with no args is just passthrough"""
split = Splitter()
split.activate()
def test_PassThroughControlSignal(self):
"""Data sent to the inbox is sent on to the outbox"""
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for i in xrange(1, 10):
split._deliver(i, "control")
for _ in xrange(0, 100):
split.next()
for i in xrange(1, 10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0 == len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_(i == Dummy.recv("control"))
for i in xrange(1, 10):
split._deliver(i, "control")
split.next()
split.next()
for _ in xrange(0, 10):
split.next()
for i in xrange(1, 10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0 == len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_(i == Dummy.recv("control"))
print "From control in forwarder: %s: %s" % (self.name, data)
self.send(data, "signal")
return
while self.dataReady("secondary-control"):
data = self.recv("secondary-control")
print "From control in secondary-control: %s: %s" % (self.name, data)
self.send(data, "signal")
return
if not self.anyReady():
self.pause()
yield 1
if 0:
mysplitter = PlugSplitter( Producer() )
mysplitter.activate()
Plug(mysplitter, Consumer("consumerA") ).activate()
Plug(mysplitter, Consumer("consumerB") ).activate()
mysplitter.run()
if 0:
producer = Producer()
mysplitter = PlugSplitter()
pipe = Pipeline(producer, mysplitter)
Plug(mysplitter, Consumer("consumerA") ).activate()
Plug(mysplitter, Consumer("consumerB") ).activate()
return
while self.dataReady("secondary-control"):
data = self.recv("secondary-control")
print "From control in secondary-control: %s: %s" % (self.name,
data)
self.send(data, "signal")
return
if not self.anyReady():
self.pause()
yield 1
if 0:
mysplitter = PlugSplitter(Producer())
mysplitter.activate()
Plug(mysplitter, Consumer("consumerA")).activate()
Plug(mysplitter, Consumer("consumerB")).activate()
mysplitter.run()
if 0:
producer = Producer()
mysplitter = PlugSplitter()
pipe = Pipeline(producer, mysplitter)
Plug(mysplitter, Consumer("consumerA")).activate()
Plug(mysplitter, Consumer("consumerB")).activate()
class FeedParserFactory(Axon.Component.component):
"""
FeedParserFactory() -> FeedParserFactory object
It receives different feed URLs throught the "inbox" inbox
and returns each post parsed through the "outbox" outbox.
This class can handles multiple concurrent petitions, retrieves
the content of the feed and parses it with the feedparser library.
The result is a feedparser.FeedParserDict per each feed URL
provided.
"""
Inboxes = {
"inbox" : "Strings representing different URLs of feeds",
"control" : "From component...",
"_parsed-feeds" : "Parsed feeds retrieved from FeedParserFactory children",
}
Outboxes = {
"outbox" : "feedparser.FeedParserDict object representing a parsed feed",
"signal" : "From component...",
"_signal" : "To the internal parsers",
}
def __init__(self, **argd):
"""x.__init__(...) initializes x; see x.__class__.__doc__ for signature"""
super(FeedParserFactory, self).__init__(**argd)
self.mustStop = None
self.providerFinished = False
def makeFeedParser(self, feedUrl):
"""
makeFeedParser(feedUrl) -> Pipeline
It returns a pipeline which does not expect any input except for signals and
sends the parsed data through the "outbox" outbox.
"""
started(feedUrl)
return Pipeline(
OneShot(feedUrl),
SimpleHTTPClient(), # TODO: SimpleHTTPClient doesn't seem to have proxy support
)
def checkControl(self):
while self.dataReady("control"):
msg = self.recv("control")
if isinstance(msg,producerFinished):
self.providerFinished = True
elif isinstance(msg,shutdownMicroprocess):
self.mustStop = msg
return self.mustStop, self.providerFinished
def handleChildTerminations(self): #taken from Carousel.py
for child in self.childComponents():
if child._isStopped():
self.removeChild(child)
def initiateInternalSplitter(self):
self.internalSplitter = PlugSplitter()
self.link((self,'_signal'), (self.internalSplitter, 'control'))
self.addChildren(self.internalSplitter)
self.internalSplitter.activate()
def linkChildToInternalSplitter(self, child):
forwarder = Forwarder()
plug = Plug(self.internalSplitter, forwarder)
plug.activate()
plug.link((plug, 'signal'), (child, 'control'))
child.link((child, 'signal'), (plug, 'control'))
def createChild(self, feed):
child = self.makeFeedParser(feed.url)
child = Pipeline(child, Feedparser(feed.url))
self.link( (child, 'outbox'), (self, '_parsed-feeds') )
self.linkChildToInternalSplitter(child)
return child
def waitForChildren(self, signalMessage):
self.send(signalMessage,"_signal")
while len(self.childComponents()) > 0:
self.handleChildTerminations()
yield 1
def main(self):
self.initiateInternalSplitter()
yield 1
while True:
mustStop, providerFinished = self.checkControl()
if mustStop:
self.send(mustStop,"signal")
return
self.handleChildTerminations()
while self.dataReady("inbox"):
feed = self.recv("inbox")
child = self.createChild(feed)
self.addChildren(child)
child.activate()
while self.dataReady("_parsed-feeds"):
parseddata = self.recv("_parsed-feeds")
self.send(parseddata,"outbox")
if providerFinished and len(self.childComponents()) == 1:
# TODO: CHECK IF THIS IS THE PROBLEM
# It's actually only waiting for the plugsplitter
for _ in self.waitForChildren(producerFinished(self)):
yield 1
pfinished = producerFinished(self)
self.send(pfinished,"signal")
return
if not self.anyReady():
self.pause()
yield 1
def start(self):
channels_plugsplitter = PlugSplitter()
feeds_plugsplitter = PlugSplitter()
posts_plugsplitter = PlugSplitter()
config_plugsplitter = PlugSplitter()
config_parser_finished_plugsplitter = PlugSplitter()
self.createPosts2fileManager(channels_plugsplitter, feeds_plugsplitter,
posts_plugsplitter, config_plugsplitter,
'rssTemplateName',
'rssFileName').activate()
self.createPosts2fileManager(channels_plugsplitter, feeds_plugsplitter,
posts_plugsplitter, config_plugsplitter,
'htmlTemplateName',
'htmlFileName').activate()
self.component = Graphline(
XML_PARSER=self.create_xml_parser(),
CONFIG_PARSER=ConfigFileParser(),
FEED_PARSER_FACTORY=FeedParserFactory(),
CHANNELS_SUBSCRIBER=self.subscribeToPlugsplitter(
channels_plugsplitter),
CHANNELS_SPLITTER=channels_plugsplitter,
POST_SORTER=PostSorter(),
POSTS_SUBSCRIBER=self.subscribeToPlugsplitter(posts_plugsplitter),
POSTS_SPLITTER=posts_plugsplitter,
FEEDS_SUBSCRIBER=self.subscribeToPlugsplitter(feeds_plugsplitter),
FEEDS_SPLITTER=feeds_plugsplitter,
CONFIG_SUBSCRIBER=self.subscribeToPlugsplitter(
config_plugsplitter),
CONFIG_SPLITTER=config_plugsplitter,
CONFIG_PARSER_FINISHED_SUBS1=self.subscribeToPlugsplitter(
config_parser_finished_plugsplitter),
CONFIG_PARSER_FINISHED_SUBS2=self.subscribeToPlugsplitter(
config_parser_finished_plugsplitter),
CONFIG_PARSER_FINISHED_SUBS3=self.subscribeToPlugsplitter(
config_parser_finished_plugsplitter),
CONFIG_PARSER_FINISHED=config_parser_finished_plugsplitter,
linkages={
('XML_PARSER', 'outbox'): ('CONFIG_PARSER', 'inbox'),
('CONFIG_PARSER', 'feeds-outbox'):
('CHANNELS_SPLITTER', 'inbox'),
('CONFIG_PARSER', 'config-outbox'):
('CONFIG_SPLITTER', 'inbox'),
('CONFIG_SUBSCRIBER', 'outbox'): ('POST_SORTER',
'config-inbox'),
('CHANNELS_SUBSCRIBER', 'outbox'): ('FEED_PARSER_FACTORY',
'inbox'),
('FEED_PARSER_FACTORY', 'outbox'): ('FEEDS_SPLITTER', 'inbox'),
('FEEDS_SUBSCRIBER', 'outbox'): ('POST_SORTER', 'inbox'),
('POST_SORTER', 'outbox'): ('POSTS_SPLITTER', 'inbox'),
# Signals
('XML_PARSER', 'signal'): ('CONFIG_PARSER', 'control'),
('CONFIG_PARSER', 'signal'): ('CONFIG_PARSER_FINISHED',
'control'),
('CONFIG_PARSER_FINISHED_SUBS1', 'signal'):
('FEED_PARSER_FACTORY', 'control'),
('CONFIG_PARSER_FINISHED_SUBS2', 'signal'):
('CHANNELS_SPLITTER', 'control'),
('CONFIG_PARSER_FINISHED_SUBS3', 'signal'): ('CONFIG_SPLITTER',
'control'),
('FEED_PARSER_FACTORY', 'signal'): ('POST_SORTER', 'control'),
('POST_SORTER', 'signal'): ('POSTS_SPLITTER', 'control'),
('POSTS_SUBSCRIBER', 'signal'): ('FEEDS_SPLITTER', 'control'),
})
def initiateInternalSplitter(self):
self.internalSplitter = PlugSplitter()
self.link((self,'_signal'), (self.internalSplitter, 'control'))
self.addChildren(self.internalSplitter)
self.internalSplitter.activate()
