def test_Unplugging(self):
"""Plug will unplug and shutdown when child component dies."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1,100):
execute.next()
#send shutdown msg
msg = producerFinished()
target._deliver(msg, "control")
for i in xrange(1,100):
execute.next()
# verify it reached the target
self.assert_(target.controllog == [msg])
# verify the plug has shutdown
self.assert_(plug._isStopped())
# verify the plug has no linkages
self.assert_(not plug.postoffice.linkages)
# verify that splitter only has outboxes "outbox" and "signal" now
self.assert_( len(splitter.outboxes) == 2)
self.assert_( "outbox" in splitter.outboxes)
self.assert_( "signal" in splitter.outboxes)
def test_SplitterShutdown(self):
"""If producerFinished or shutdownMicroprocess is received on the 'control' inbox they are passed on and the component shuts down"""
for msg in [producerFinished(self), shutdownMicroprocess(self)]:
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for _ in xrange(0,10):
split.next()
self.assert_(0==len(split.outboxes["outbox"]))
self.assert_(0==len(split.outboxes["signal"]))
split._deliver( msg, "control" )
try:
for _ in xrange(0,10):
split.next()
self.fail()
except StopIteration:
pass
self.assert_(0==len(split.outboxes["outbox"]))
self.assert_(1==len(split.outboxes["signal"]))
# received = split._collect("signal")
received = Dummy.recv("control")
self.assert_( msg == received )
def test_PluggingInAndTxfer(self):
"""Plug instantiated with splitter and component and passes data through to component."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1,1000):
execute.next()
#pass some data in
for i in xrange(1,10):
splitter._deliver(i, "inbox")
splitter._deliver(10+i, "control")
for i in xrange(1,100):
execute.next()
# verify it reached the target
self.assert_(target.inboxlog == range(1,10))
self.assert_(target.controllog == range(11,20))
def test_PluggingInAndTxfer(self):
"""Plug instantiated with splitter and component and passes data through to component."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1, 1000):
execute.next()
#pass some data in
for i in xrange(1, 10):
splitter._deliver(i, "inbox")
splitter._deliver(10 + i, "control")
for i in xrange(1, 100):
execute.next()
# verify it reached the target
self.assert_(target.inboxlog == range(1, 10))
self.assert_(target.controllog == range(11, 20))
def test_PassThroughControlSignal(self):
"""Data sent to the inbox is sent on to the outbox"""
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for i in xrange(1,10):
split._deliver( i, "control" )
for _ in xrange(0,100):
split.next()
for i in xrange(1,10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0==len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_( i == Dummy.recv("control") )
for i in xrange(1,10):
split._deliver( i, "control" )
split.next()
split.next()
for _ in xrange(0,10):
split.next()
for i in xrange(1,10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0==len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_( i == Dummy.recv("control") )
def test_Unplugging(self):
"""Plug will unplug and shutdown when child component dies."""
Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler()
splitter = Splitter()
splitter.activate()
target = DummyComponent()
plug = Plug(splitter, target).activate()
execute = Axon.Scheduler.scheduler.run.main()
for i in xrange(1, 100):
execute.next()
#send shutdown msg
msg = producerFinished()
target._deliver(msg, "control")
for i in xrange(1, 100):
execute.next()
# verify it reached the target
self.assert_(target.controllog == [msg])
# verify the plug has shutdown
self.assert_(plug._isStopped())
# verify the plug has no linkages
self.assert_(not plug.postoffice.linkages)
# verify that splitter only has outboxes "outbox" and "signal" now
self.assert_(len(splitter.outboxes) == 2)
self.assert_("outbox" in splitter.outboxes)
self.assert_("signal" in splitter.outboxes)
def test_SplitterShutdown(self):
"""If producerFinished or shutdownMicroprocess is received on the 'control' inbox they are passed on and the component shuts down"""
for msg in [producerFinished(self), shutdownMicroprocess(self)]:
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for _ in xrange(0, 10):
split.next()
self.assert_(0 == len(split.outboxes["outbox"]))
self.assert_(0 == len(split.outboxes["signal"]))
split._deliver(msg, "control")
try:
for _ in xrange(0, 10):
split.next()
self.fail()
except StopIteration:
pass
self.assert_(0 == len(split.outboxes["outbox"]))
self.assert_(1 == len(split.outboxes["signal"]))
# received = split._collect("signal")
received = Dummy.recv("control")
self.assert_(msg == received)
def test_PassThroughControlSignal(self):
"""Data sent to the inbox is sent on to the outbox"""
split = Splitter()
Dummy = Axon.Component.component()
split.link((split, "outbox"), (Dummy, "inbox"))
split.link((split, "signal"), (Dummy, "control"))
split.activate()
for i in xrange(1, 10):
split._deliver(i, "control")
for _ in xrange(0, 100):
split.next()
for i in xrange(1, 10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0 == len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_(i == Dummy.recv("control"))
for i in xrange(1, 10):
split._deliver(i, "control")
split.next()
split.next()
for _ in xrange(0, 10):
split.next()
for i in xrange(1, 10):
self.assert_(len(split.outboxes["signal"]))
self.assert_(0 == len(split.outboxes["outbox"]))
# self.assert_( i == split._collect("signal") )
self.assert_(i == Dummy.recv("control"))
def test_InstantiateNoArgs(self):
"""Splitter instantiated with no args is just passthrough"""
split = Splitter()
split.activate()
class FeedParserFactory(Axon.Component.component):
"""
FeedParserFactory() -> FeedParserFactory object
It receives different feed URLs throught the "inbox" inbox
and returns each post parsed through the "outbox" outbox.
This class can handles multiple concurrent petitions, retrieves
the content of the feed and parses it with the feedparser library.
The result is a feedparser.FeedParserDict per each feed URL
provided.
"""
Inboxes = {
"inbox" : "Strings representing different URLs of feeds",
"control" : "From component...",
"_parsed-feeds" : "Parsed feeds retrieved from FeedParserFactory children",
}
Outboxes = {
"outbox" : "feedparser.FeedParserDict object representing a parsed feed",
"signal" : "From component...",
"_signal" : "To the internal parsers",
}
def __init__(self, **argd):
"""x.__init__(...) initializes x; see x.__class__.__doc__ for signature"""
super(FeedParserFactory, self).__init__(**argd)
self.mustStop = None
self.providerFinished = False
def makeFeedParser(self, feedUrl):
"""
makeFeedParser(feedUrl) -> Pipeline
It returns a pipeline which does not expect any input except for signals and
sends the parsed data through the "outbox" outbox.
"""
started(feedUrl)
return Pipeline(
OneShot(feedUrl),
SimpleHTTPClient(), # TODO: SimpleHTTPClient doesn't seem to have proxy support
)
def checkControl(self):
while self.dataReady("control"):
msg = self.recv("control")
if isinstance(msg,producerFinished):
self.providerFinished = True
elif isinstance(msg,shutdownMicroprocess):
self.mustStop = msg
return self.mustStop, self.providerFinished
def handleChildTerminations(self): #taken from Carousel.py
for child in self.childComponents():
if child._isStopped():
self.removeChild(child)
def initiateInternalSplitter(self):
self.internalSplitter = PlugSplitter()
self.link((self,'_signal'), (self.internalSplitter, 'control'))
self.addChildren(self.internalSplitter)
self.internalSplitter.activate()
def linkChildToInternalSplitter(self, child):
forwarder = Forwarder()
plug = Plug(self.internalSplitter, forwarder)
plug.activate()
plug.link((plug, 'signal'), (child, 'control'))
child.link((child, 'signal'), (plug, 'control'))
def createChild(self, feed):
child = self.makeFeedParser(feed.url)
child = Pipeline(child, Feedparser(feed.url))
self.link( (child, 'outbox'), (self, '_parsed-feeds') )
self.linkChildToInternalSplitter(child)
return child
def waitForChildren(self, signalMessage):
self.send(signalMessage,"_signal")
while len(self.childComponents()) > 0:
self.handleChildTerminations()
yield 1
def main(self):
self.initiateInternalSplitter()
yield 1
while True:
mustStop, providerFinished = self.checkControl()
if mustStop:
self.send(mustStop,"signal")
return
self.handleChildTerminations()
while self.dataReady("inbox"):
feed = self.recv("inbox")
child = self.createChild(feed)
self.addChildren(child)
child.activate()
while self.dataReady("_parsed-feeds"):
parseddata = self.recv("_parsed-feeds")
self.send(parseddata,"outbox")
if providerFinished and len(self.childComponents()) == 1:
# TODO: CHECK IF THIS IS THE PROBLEM
# It's actually only waiting for the plugsplitter
for _ in self.waitForChildren(producerFinished(self)):
yield 1
pfinished = producerFinished(self)
self.send(pfinished,"signal")
return
if not self.anyReady():
self.pause()
yield 1
class FeedParserFactory(Axon.Component.component):
"""
FeedParserFactory() -> FeedParserFactory object
It receives different feed URLs throught the "inbox" inbox
and returns each post parsed through the "outbox" outbox.
This class can handles multiple concurrent petitions, retrieves
the content of the feed and parses it with the feedparser library.
The result is a feedparser.FeedParserDict per each feed URL
provided.
"""
Inboxes = {
"inbox": "Strings representing different URLs of feeds",
"control": "From component...",
"_parsed-feeds":
"Parsed feeds retrieved from FeedParserFactory children",
}
Outboxes = {
"outbox":
"feedparser.FeedParserDict object representing a parsed feed",
"signal": "From component...",
"_signal": "To the internal parsers",
}
def __init__(self, **argd):
"""x.__init__(...) initializes x; see x.__class__.__doc__ for signature"""
super(FeedParserFactory, self).__init__(**argd)
self.mustStop = None
self.providerFinished = False
def makeFeedParser(self, feedUrl):
"""
makeFeedParser(feedUrl) -> Pipeline
It returns a pipeline which does not expect any input except for signals and
sends the parsed data through the "outbox" outbox.
"""
started(feedUrl)
return Pipeline(
OneShot(feedUrl),
SimpleHTTPClient(
), # TODO: SimpleHTTPClient doesn't seem to have proxy support
)
def checkControl(self):
while self.dataReady("control"):
msg = self.recv("control")
if isinstance(msg, producerFinished):
self.providerFinished = True
elif isinstance(msg, shutdownMicroprocess):
self.mustStop = msg
return self.mustStop, self.providerFinished
def handleChildTerminations(self): #taken from Carousel.py
for child in self.childComponents():
if child._isStopped():
self.removeChild(child)
def initiateInternalSplitter(self):
self.internalSplitter = PlugSplitter()
self.link((self, '_signal'), (self.internalSplitter, 'control'))
self.addChildren(self.internalSplitter)
self.internalSplitter.activate()
def linkChildToInternalSplitter(self, child):
forwarder = Forwarder()
plug = Plug(self.internalSplitter, forwarder)
plug.activate()
plug.link((plug, 'signal'), (child, 'control'))
child.link((child, 'signal'), (plug, 'control'))
def createChild(self, feed):
child = self.makeFeedParser(feed.url)
child = Pipeline(child, Feedparser(feed.url))
self.link((child, 'outbox'), (self, '_parsed-feeds'))
self.linkChildToInternalSplitter(child)
return child
def waitForChildren(self, signalMessage):
self.send(signalMessage, "_signal")
while len(self.childComponents()) > 0:
self.handleChildTerminations()
yield 1
def main(self):
self.initiateInternalSplitter()
yield 1
while True:
mustStop, providerFinished = self.checkControl()
if mustStop:
self.send(mustStop, "signal")
return
self.handleChildTerminations()
while self.dataReady("inbox"):
feed = self.recv("inbox")
child = self.createChild(feed)
self.addChildren(child)
child.activate()
while self.dataReady("_parsed-feeds"):
parseddata = self.recv("_parsed-feeds")
self.send(parseddata, "outbox")
if providerFinished and len(self.childComponents()) == 1:
# TODO: CHECK IF THIS IS THE PROBLEM
# It's actually only waiting for the plugsplitter
for _ in self.waitForChildren(producerFinished(self)):
yield 1
pfinished = producerFinished(self)
self.send(pfinished, "signal")
return
if not self.anyReady():
self.pause()
yield 1
def test_InstantiateNoArgs(self):
"""Splitter instantiated with no args is just passthrough"""
split = Splitter()
split.activate()
self.send(data, "signal")
return
while self.dataReady("secondary-control"):
data = self.recv("secondary-control")
print "From control in secondary-control: %s: %s" % (self.name, data)
self.send(data, "signal")
return
if not self.anyReady():
self.pause()
yield 1
if 0:
mysplitter = PlugSplitter( Producer() )
mysplitter.activate()
Plug(mysplitter, Consumer("consumerA") ).activate()
Plug(mysplitter, Consumer("consumerB") ).activate()
mysplitter.run()
if 0:
producer = Producer()
mysplitter = PlugSplitter()
pipe = Pipeline(producer, mysplitter)
Plug(mysplitter, Consumer("consumerA") ).activate()
Plug(mysplitter, Consumer("consumerB") ).activate()
pipe.run()
while self.dataReady("secondary-control"):
data = self.recv("secondary-control")
print "From control in secondary-control: %s: %s" % (self.name,
data)
self.send(data, "signal")
return
if not self.anyReady():
self.pause()
yield 1
if 0:
mysplitter = PlugSplitter(Producer())
mysplitter.activate()
Plug(mysplitter, Consumer("consumerA")).activate()
Plug(mysplitter, Consumer("consumerB")).activate()
mysplitter.run()
if 0:
producer = Producer()
mysplitter = PlugSplitter()
pipe = Pipeline(producer, mysplitter)
Plug(mysplitter, Consumer("consumerA")).activate()
Plug(mysplitter, Consumer("consumerB")).activate()
pipe.run()
