def test_Unplugging(self): """Plug will unplug and shutdown when child component dies.""" Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler() splitter = Splitter() splitter.activate() target = DummyComponent() plug = Plug(splitter, target).activate() execute = Axon.Scheduler.scheduler.run.main() for i in xrange(1,100): execute.next() #send shutdown msg msg = producerFinished() target._deliver(msg, "control") for i in xrange(1,100): execute.next() # verify it reached the target self.assert_(target.controllog == [msg]) # verify the plug has shutdown self.assert_(plug._isStopped()) # verify the plug has no linkages self.assert_(not plug.postoffice.linkages) # verify that splitter only has outboxes "outbox" and "signal" now self.assert_( len(splitter.outboxes) == 2) self.assert_( "outbox" in splitter.outboxes) self.assert_( "signal" in splitter.outboxes)
def test_SplitterShutdown(self): """If producerFinished or shutdownMicroprocess is received on the 'control' inbox they are passed on and the component shuts down""" for msg in [producerFinished(self), shutdownMicroprocess(self)]: split = Splitter() Dummy = Axon.Component.component() split.link((split, "outbox"), (Dummy, "inbox")) split.link((split, "signal"), (Dummy, "control")) split.activate() for _ in xrange(0,10): split.next() self.assert_(0==len(split.outboxes["outbox"])) self.assert_(0==len(split.outboxes["signal"])) split._deliver( msg, "control" ) try: for _ in xrange(0,10): split.next() self.fail() except StopIteration: pass self.assert_(0==len(split.outboxes["outbox"])) self.assert_(1==len(split.outboxes["signal"])) # received = split._collect("signal") received = Dummy.recv("control") self.assert_( msg == received )
def test_PluggingInAndTxfer(self): """Plug instantiated with splitter and component and passes data through to component.""" Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler() splitter = Splitter() splitter.activate() target = DummyComponent() plug = Plug(splitter, target).activate() execute = Axon.Scheduler.scheduler.run.main() for i in xrange(1,1000): execute.next() #pass some data in for i in xrange(1,10): splitter._deliver(i, "inbox") splitter._deliver(10+i, "control") for i in xrange(1,100): execute.next() # verify it reached the target self.assert_(target.inboxlog == range(1,10)) self.assert_(target.controllog == range(11,20))
def test_PluggingInAndTxfer(self): """Plug instantiated with splitter and component and passes data through to component.""" Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler() splitter = Splitter() splitter.activate() target = DummyComponent() plug = Plug(splitter, target).activate() execute = Axon.Scheduler.scheduler.run.main() for i in xrange(1, 1000): execute.next() #pass some data in for i in xrange(1, 10): splitter._deliver(i, "inbox") splitter._deliver(10 + i, "control") for i in xrange(1, 100): execute.next() # verify it reached the target self.assert_(target.inboxlog == range(1, 10)) self.assert_(target.controllog == range(11, 20))
def test_PassThroughControlSignal(self): """Data sent to the inbox is sent on to the outbox""" split = Splitter() Dummy = Axon.Component.component() split.link((split, "outbox"), (Dummy, "inbox")) split.link((split, "signal"), (Dummy, "control")) split.activate() for i in xrange(1,10): split._deliver( i, "control" ) for _ in xrange(0,100): split.next() for i in xrange(1,10): self.assert_(len(split.outboxes["signal"])) self.assert_(0==len(split.outboxes["outbox"])) # self.assert_( i == split._collect("signal") ) self.assert_( i == Dummy.recv("control") ) for i in xrange(1,10): split._deliver( i, "control" ) split.next() split.next() for _ in xrange(0,10): split.next() for i in xrange(1,10): self.assert_(len(split.outboxes["signal"])) self.assert_(0==len(split.outboxes["outbox"])) # self.assert_( i == split._collect("signal") ) self.assert_( i == Dummy.recv("control") )
def test_Unplugging(self): """Plug will unplug and shutdown when child component dies.""" Axon.Scheduler.scheduler.run = Axon.Scheduler.scheduler() splitter = Splitter() splitter.activate() target = DummyComponent() plug = Plug(splitter, target).activate() execute = Axon.Scheduler.scheduler.run.main() for i in xrange(1, 100): execute.next() #send shutdown msg msg = producerFinished() target._deliver(msg, "control") for i in xrange(1, 100): execute.next() # verify it reached the target self.assert_(target.controllog == [msg]) # verify the plug has shutdown self.assert_(plug._isStopped()) # verify the plug has no linkages self.assert_(not plug.postoffice.linkages) # verify that splitter only has outboxes "outbox" and "signal" now self.assert_(len(splitter.outboxes) == 2) self.assert_("outbox" in splitter.outboxes) self.assert_("signal" in splitter.outboxes)
def test_SplitterShutdown(self): """If producerFinished or shutdownMicroprocess is received on the 'control' inbox they are passed on and the component shuts down""" for msg in [producerFinished(self), shutdownMicroprocess(self)]: split = Splitter() Dummy = Axon.Component.component() split.link((split, "outbox"), (Dummy, "inbox")) split.link((split, "signal"), (Dummy, "control")) split.activate() for _ in xrange(0, 10): split.next() self.assert_(0 == len(split.outboxes["outbox"])) self.assert_(0 == len(split.outboxes["signal"])) split._deliver(msg, "control") try: for _ in xrange(0, 10): split.next() self.fail() except StopIteration: pass self.assert_(0 == len(split.outboxes["outbox"])) self.assert_(1 == len(split.outboxes["signal"])) # received = split._collect("signal") received = Dummy.recv("control") self.assert_(msg == received)
def test_PassThroughControlSignal(self): """Data sent to the inbox is sent on to the outbox""" split = Splitter() Dummy = Axon.Component.component() split.link((split, "outbox"), (Dummy, "inbox")) split.link((split, "signal"), (Dummy, "control")) split.activate() for i in xrange(1, 10): split._deliver(i, "control") for _ in xrange(0, 100): split.next() for i in xrange(1, 10): self.assert_(len(split.outboxes["signal"])) self.assert_(0 == len(split.outboxes["outbox"])) # self.assert_( i == split._collect("signal") ) self.assert_(i == Dummy.recv("control")) for i in xrange(1, 10): split._deliver(i, "control") split.next() split.next() for _ in xrange(0, 10): split.next() for i in xrange(1, 10): self.assert_(len(split.outboxes["signal"])) self.assert_(0 == len(split.outboxes["outbox"])) # self.assert_( i == split._collect("signal") ) self.assert_(i == Dummy.recv("control"))
def test_InstantiateNoArgs(self): """Splitter instantiated with no args is just passthrough""" split = Splitter() split.activate()
class FeedParserFactory(Axon.Component.component): """ FeedParserFactory() -> FeedParserFactory object It receives different feed URLs throught the "inbox" inbox and returns each post parsed through the "outbox" outbox. This class can handles multiple concurrent petitions, retrieves the content of the feed and parses it with the feedparser library. The result is a feedparser.FeedParserDict per each feed URL provided. """ Inboxes = { "inbox" : "Strings representing different URLs of feeds", "control" : "From component...", "_parsed-feeds" : "Parsed feeds retrieved from FeedParserFactory children", } Outboxes = { "outbox" : "feedparser.FeedParserDict object representing a parsed feed", "signal" : "From component...", "_signal" : "To the internal parsers", } def __init__(self, **argd): """x.__init__(...) initializes x; see x.__class__.__doc__ for signature""" super(FeedParserFactory, self).__init__(**argd) self.mustStop = None self.providerFinished = False def makeFeedParser(self, feedUrl): """ makeFeedParser(feedUrl) -> Pipeline It returns a pipeline which does not expect any input except for signals and sends the parsed data through the "outbox" outbox. """ started(feedUrl) return Pipeline( OneShot(feedUrl), SimpleHTTPClient(), # TODO: SimpleHTTPClient doesn't seem to have proxy support ) def checkControl(self): while self.dataReady("control"): msg = self.recv("control") if isinstance(msg,producerFinished): self.providerFinished = True elif isinstance(msg,shutdownMicroprocess): self.mustStop = msg return self.mustStop, self.providerFinished def handleChildTerminations(self): #taken from Carousel.py for child in self.childComponents(): if child._isStopped(): self.removeChild(child) def initiateInternalSplitter(self): self.internalSplitter = PlugSplitter() self.link((self,'_signal'), (self.internalSplitter, 'control')) self.addChildren(self.internalSplitter) self.internalSplitter.activate() def linkChildToInternalSplitter(self, child): forwarder = Forwarder() plug = Plug(self.internalSplitter, forwarder) plug.activate() plug.link((plug, 'signal'), (child, 'control')) child.link((child, 'signal'), (plug, 'control')) def createChild(self, feed): child = self.makeFeedParser(feed.url) child = Pipeline(child, Feedparser(feed.url)) self.link( (child, 'outbox'), (self, '_parsed-feeds') ) self.linkChildToInternalSplitter(child) return child def waitForChildren(self, signalMessage): self.send(signalMessage,"_signal") while len(self.childComponents()) > 0: self.handleChildTerminations() yield 1 def main(self): self.initiateInternalSplitter() yield 1 while True: mustStop, providerFinished = self.checkControl() if mustStop: self.send(mustStop,"signal") return self.handleChildTerminations() while self.dataReady("inbox"): feed = self.recv("inbox") child = self.createChild(feed) self.addChildren(child) child.activate() while self.dataReady("_parsed-feeds"): parseddata = self.recv("_parsed-feeds") self.send(parseddata,"outbox") if providerFinished and len(self.childComponents()) == 1: # TODO: CHECK IF THIS IS THE PROBLEM # It's actually only waiting for the plugsplitter for _ in self.waitForChildren(producerFinished(self)): yield 1 pfinished = producerFinished(self) self.send(pfinished,"signal") return if not self.anyReady(): self.pause() yield 1
class FeedParserFactory(Axon.Component.component): """ FeedParserFactory() -> FeedParserFactory object It receives different feed URLs throught the "inbox" inbox and returns each post parsed through the "outbox" outbox. This class can handles multiple concurrent petitions, retrieves the content of the feed and parses it with the feedparser library. The result is a feedparser.FeedParserDict per each feed URL provided. """ Inboxes = { "inbox": "Strings representing different URLs of feeds", "control": "From component...", "_parsed-feeds": "Parsed feeds retrieved from FeedParserFactory children", } Outboxes = { "outbox": "feedparser.FeedParserDict object representing a parsed feed", "signal": "From component...", "_signal": "To the internal parsers", } def __init__(self, **argd): """x.__init__(...) initializes x; see x.__class__.__doc__ for signature""" super(FeedParserFactory, self).__init__(**argd) self.mustStop = None self.providerFinished = False def makeFeedParser(self, feedUrl): """ makeFeedParser(feedUrl) -> Pipeline It returns a pipeline which does not expect any input except for signals and sends the parsed data through the "outbox" outbox. """ started(feedUrl) return Pipeline( OneShot(feedUrl), SimpleHTTPClient( ), # TODO: SimpleHTTPClient doesn't seem to have proxy support ) def checkControl(self): while self.dataReady("control"): msg = self.recv("control") if isinstance(msg, producerFinished): self.providerFinished = True elif isinstance(msg, shutdownMicroprocess): self.mustStop = msg return self.mustStop, self.providerFinished def handleChildTerminations(self): #taken from Carousel.py for child in self.childComponents(): if child._isStopped(): self.removeChild(child) def initiateInternalSplitter(self): self.internalSplitter = PlugSplitter() self.link((self, '_signal'), (self.internalSplitter, 'control')) self.addChildren(self.internalSplitter) self.internalSplitter.activate() def linkChildToInternalSplitter(self, child): forwarder = Forwarder() plug = Plug(self.internalSplitter, forwarder) plug.activate() plug.link((plug, 'signal'), (child, 'control')) child.link((child, 'signal'), (plug, 'control')) def createChild(self, feed): child = self.makeFeedParser(feed.url) child = Pipeline(child, Feedparser(feed.url)) self.link((child, 'outbox'), (self, '_parsed-feeds')) self.linkChildToInternalSplitter(child) return child def waitForChildren(self, signalMessage): self.send(signalMessage, "_signal") while len(self.childComponents()) > 0: self.handleChildTerminations() yield 1 def main(self): self.initiateInternalSplitter() yield 1 while True: mustStop, providerFinished = self.checkControl() if mustStop: self.send(mustStop, "signal") return self.handleChildTerminations() while self.dataReady("inbox"): feed = self.recv("inbox") child = self.createChild(feed) self.addChildren(child) child.activate() while self.dataReady("_parsed-feeds"): parseddata = self.recv("_parsed-feeds") self.send(parseddata, "outbox") if providerFinished and len(self.childComponents()) == 1: # TODO: CHECK IF THIS IS THE PROBLEM # It's actually only waiting for the plugsplitter for _ in self.waitForChildren(producerFinished(self)): yield 1 pfinished = producerFinished(self) self.send(pfinished, "signal") return if not self.anyReady(): self.pause() yield 1
self.send(data, "signal") return while self.dataReady("secondary-control"): data = self.recv("secondary-control") print "From control in secondary-control: %s: %s" % (self.name, data) self.send(data, "signal") return if not self.anyReady(): self.pause() yield 1 if 0: mysplitter = PlugSplitter( Producer() ) mysplitter.activate() Plug(mysplitter, Consumer("consumerA") ).activate() Plug(mysplitter, Consumer("consumerB") ).activate() mysplitter.run() if 0: producer = Producer() mysplitter = PlugSplitter() pipe = Pipeline(producer, mysplitter) Plug(mysplitter, Consumer("consumerA") ).activate() Plug(mysplitter, Consumer("consumerB") ).activate() pipe.run()
while self.dataReady("secondary-control"): data = self.recv("secondary-control") print "From control in secondary-control: %s: %s" % (self.name, data) self.send(data, "signal") return if not self.anyReady(): self.pause() yield 1 if 0: mysplitter = PlugSplitter(Producer()) mysplitter.activate() Plug(mysplitter, Consumer("consumerA")).activate() Plug(mysplitter, Consumer("consumerB")).activate() mysplitter.run() if 0: producer = Producer() mysplitter = PlugSplitter() pipe = Pipeline(producer, mysplitter) Plug(mysplitter, Consumer("consumerA")).activate() Plug(mysplitter, Consumer("consumerB")).activate() pipe.run()