class SpreadFlowService(service.Service): def __init__(self, options): self.options = options self._scheduler = None self._eventdispatcher = None def startService(self): super(SpreadFlowService, self).startService() if self.options['confpath']: confpath = self.options['confpath'] else: confpath = os.path.join(os.getcwd(), 'spreadflow.conf') stream = config_eval(confpath) pipeline = list() pipeline.append(AliasResolverPass()) pipeline.append(PortsValidatorPass()) if self.options['multiprocess']: pipeline.append(PartitionExpanderPass()) pipeline.append(PartitionBoundsPass()) if self.options['partition']: pipeline.append(PartitionWorkerPass()) partition = self.options['partition'] stream.append(AddTokenOp(PartitionSelectToken(partition))) else: pipeline.append(PartitionControllersPass()) pipeline.append(ComponentsPurgePass()) pipeline.append(EventHandlersPass()) for compiler_step in pipeline: stream = compiler_step(stream) self._eventdispatcher = EventDispatcher() if self.options['oneshot']: self._eventdispatcher.add_listener(JobEvent, 0, self._oneshot_job_event_handler) connection_parser = ConnectionParser() stream = connection_parser.extract(stream) self._scheduler = Scheduler(connection_parser.get_portmap(), self._eventdispatcher) event_handler_parser = EventHandlerParser() stream = event_handler_parser.extract(stream) for event_type, priority, callback in event_handler_parser.get_handlers(): self._eventdispatcher.add_listener(event_type, priority, callback) if self.options['queuestatus']: statuslog = SpreadFlowQueuestatusLogger(self.options['queuestatus']) statuslog.watch(1, self._scheduler) globalLogPublisher.addObserver(statuslog.logstatus) self._scheduler.run().addBoth(self._stop) def stopService(self): super(SpreadFlowService, self).stopService() return self._scheduler.join() def _stop(self, result): from twisted.internet import reactor try: reactor.stop() except error.ReactorNotRunning: pass return result def _oneshot_job_event_handler(self, event): scheduler = event.scheduler completed = event.completed def _stop_scheduler_when_done(result): if len(scheduler.pending) == 0: scheduler.stop(self) return result completed.addCallback(_stop_scheduler_when_done)
class SchedulerTestCase(TestCase): epsilon = task._EPSILON def setUp(self): super(SchedulerTestCase, self).setUp() self.clock = task.Clock() self.cooperator = task.Cooperator( terminationPredicateFactory=_no_termination_factory, scheduler=lambda x: self.clock.callLater(self.epsilon, x) ) self.cooperate = self.cooperator.cooperate self.dispatcher = EventDispatcher() self.flowmap = dict() self.scheduler = Scheduler(self.flowmap, self.dispatcher, self.cooperate) def test_run_scheduler(self): """ Tests run(), stop() and join() and ensures that appropriate events are fired. """ job_handler = Mock() attach_handler = Mock() start_handler = Mock() join_handler = Mock() detach_handler = Mock() self.dispatcher.add_listener(JobEvent, 0, job_handler) self.dispatcher.add_listener(AttachEvent, 0, attach_handler) self.dispatcher.add_listener(DetachEvent, 0, detach_handler) # Start the scheduler. run_deferred = self.scheduler.run(self.clock) assert_that(run_deferred, twistedsupport.has_no_result()) self.assertEquals(job_handler.call_count, 0) attach_handler.assert_called_once_with(AttachEvent(scheduler=self.scheduler, reactor=self.clock)) self.assertEquals(join_handler.call_count, 0) self.assertEquals(detach_handler.call_count, 0) job_handler.reset_mock() attach_handler.reset_mock() start_handler.reset_mock() join_handler.reset_mock() detach_handler.reset_mock() # Turn a bit on the clock. self.clock.advance(self.epsilon) assert_that(run_deferred, twistedsupport.has_no_result()) self.assertEquals(job_handler.call_count, 0) self.assertEquals(attach_handler.call_count, 0) self.assertEquals(start_handler.call_count, 0) self.assertEquals(join_handler.call_count, 0) self.assertEquals(detach_handler.call_count, 0) job_handler.reset_mock() attach_handler.reset_mock() start_handler.reset_mock() join_handler.reset_mock() detach_handler.reset_mock() # Stop the scheduler. self.scheduler.stop('some reason') assert_that(run_deferred, twistedsupport.succeeded(matchers.Equals('some reason'))) self.assertEquals(job_handler.call_count, 0) self.assertEquals(attach_handler.call_count, 0) self.assertEquals(start_handler.call_count, 0) self.assertEquals(join_handler.call_count, 0) self.assertEquals(detach_handler.call_count, 0) job_handler.reset_mock() attach_handler.reset_mock() start_handler.reset_mock() join_handler.reset_mock() detach_handler.reset_mock() # Stop the queue and join processes. join_deferred = self.scheduler.join() self.clock.advance(self.epsilon) assert_that(join_deferred, twistedsupport.succeeded(matchers.Always())) self.assertEquals(job_handler.call_count, 0) self.assertEquals(attach_handler.call_count, 0) self.assertEquals(start_handler.call_count, 0) detach_handler.assert_called_once_with(DetachEvent(scheduler=self.scheduler)) def test_run_job(self): """ Tests send() and ensure that the job-event is fired. """ job_handler = Mock() self.dispatcher.add_listener(JobEvent, 0, job_handler) port_out = object() port_in = Mock(spec=_port_callback) self.flowmap[port_out] = port_in self.scheduler.run(self.clock) self.scheduler.send('some item', port_out) expected_job = Job(port_in, 'some item', self.scheduler.send, port_out) self.assertEquals(job_handler.call_count, 1) assert_that(job_handler.call_args, MatchesInvocation( MatchesEvent(JobEvent, scheduler=matchers.Equals(self.scheduler), job=matchers.Equals(expected_job), completed=twistedsupport.has_no_result()) )) self.assertEquals(port_in.call_count, 0) self.assertEquals(len(list(self.scheduler.pending)), 1) # Trigger queue run. self.clock.advance(self.epsilon) port_in.assert_called_once_with('some item', self.scheduler.send) self.assertEquals(len(list(self.scheduler.pending)), 0) def test_fail_job(self): """ Tests that scheduler is stopped whenever a port is failing. """ port_out = object() port_in = Mock(spec=_port_callback, side_effect=RuntimeError('failed!')) self.flowmap[port_out] = port_in run_deferred = self.scheduler.run(self.clock) self.scheduler.send('some item', port_out) expected_message = 'Job failed on {:s} while processing {:s}'.format( str(port_in), 'some item') from testtools.twistedsupport._runtest import _NoTwistedLogObservers with _NoTwistedLogObservers(): with twistedsupport.CaptureTwistedLogs() as twisted_logs: # Trigger queue run. self.clock.advance(self.epsilon) assert_that(twisted_logs.getDetails(), matchers.MatchesDict({ 'twisted-log': matchers.AfterPreprocessing( lambda log: log.as_text(), matchers.Contains(expected_message)) })) port_in.assert_called_once_with('some item', self.scheduler.send) matcher = matchers.AfterPreprocessing(lambda f: f.value, matchers.IsInstance(RuntimeError)) assert_that(run_deferred, twistedsupport.failed(matcher)) def test_cancel_job(self): """ Tests that scheduler is stopped whenever a port is failing. """ port_out = object() port_in = Mock(spec=_port_callback) self.flowmap[port_out] = port_in run_deferred = self.scheduler.run(self.clock) self.scheduler.send('some item', port_out) self.assertEquals(len(list(self.scheduler.pending)), 1) self.scheduler.stop('bye!') self.assertEquals(len(list(self.scheduler.pending)), 1) join_deferred = self.scheduler.join() self.clock.advance(self.epsilon) assert_that(join_deferred, twistedsupport.succeeded(matchers.Always())) assert_that(run_deferred, twistedsupport.succeeded(matchers.Equals('bye!'))) self.assertEquals(len(list(self.scheduler.pending)), 0) self.assertEquals(port_in.call_count, 0)