Beispiel #1
0
    def startService(self):
        super(SpreadFlowService, self).startService()

        if self.options['confpath']:
            confpath = self.options['confpath']
        else:
            confpath = os.path.join(os.getcwd(), 'spreadflow.conf')

        stream = config_eval(confpath)

        pipeline = list()
        pipeline.append(AliasResolverPass())
        pipeline.append(PortsValidatorPass())

        if self.options['multiprocess']:
            pipeline.append(PartitionExpanderPass())
            pipeline.append(PartitionBoundsPass())
            if self.options['partition']:
                pipeline.append(PartitionWorkerPass())
                partition = self.options['partition']
                stream.append(AddTokenOp(PartitionSelectToken(partition)))
            else:
                pipeline.append(PartitionControllersPass())

        pipeline.append(ComponentsPurgePass())
        pipeline.append(EventHandlersPass())

        for compiler_step in pipeline:
            stream = compiler_step(stream)

        self._eventdispatcher = EventDispatcher()

        if self.options['oneshot']:
            self._eventdispatcher.add_listener(JobEvent, 0, self._oneshot_job_event_handler)

        connection_parser = ConnectionParser()
        stream = connection_parser.extract(stream)
        self._scheduler = Scheduler(connection_parser.get_portmap(), self._eventdispatcher)

        event_handler_parser = EventHandlerParser()
        stream = event_handler_parser.extract(stream)
        for event_type, priority, callback in event_handler_parser.get_handlers():
            self._eventdispatcher.add_listener(event_type, priority, callback)

        if self.options['queuestatus']:
            statuslog = SpreadFlowQueuestatusLogger(self.options['queuestatus'])
            statuslog.watch(1, self._scheduler)
            globalLogPublisher.addObserver(statuslog.logstatus)

        self._scheduler.run().addBoth(self._stop)
    def setUp(self):
        super(SchedulerTestCase, self).setUp()

        self.clock = task.Clock()

        self.cooperator = task.Cooperator(
            terminationPredicateFactory=_no_termination_factory,
            scheduler=lambda x: self.clock.callLater(self.epsilon, x)
        )
        self.cooperate = self.cooperator.cooperate

        self.dispatcher = EventDispatcher()
        self.flowmap = dict()

        self.scheduler = Scheduler(self.flowmap, self.dispatcher, self.cooperate)
Beispiel #3
0
class SpreadFlowService(service.Service):
    def __init__(self, options):
        self.options = options
        self._scheduler = None
        self._eventdispatcher = None

    def startService(self):
        super(SpreadFlowService, self).startService()

        if self.options['confpath']:
            confpath = self.options['confpath']
        else:
            confpath = os.path.join(os.getcwd(), 'spreadflow.conf')

        stream = config_eval(confpath)

        pipeline = list()
        pipeline.append(AliasResolverPass())
        pipeline.append(PortsValidatorPass())

        if self.options['multiprocess']:
            pipeline.append(PartitionExpanderPass())
            pipeline.append(PartitionBoundsPass())
            if self.options['partition']:
                pipeline.append(PartitionWorkerPass())
                partition = self.options['partition']
                stream.append(AddTokenOp(PartitionSelectToken(partition)))
            else:
                pipeline.append(PartitionControllersPass())

        pipeline.append(ComponentsPurgePass())
        pipeline.append(EventHandlersPass())

        for compiler_step in pipeline:
            stream = compiler_step(stream)

        self._eventdispatcher = EventDispatcher()

        if self.options['oneshot']:
            self._eventdispatcher.add_listener(JobEvent, 0, self._oneshot_job_event_handler)

        connection_parser = ConnectionParser()
        stream = connection_parser.extract(stream)
        self._scheduler = Scheduler(connection_parser.get_portmap(), self._eventdispatcher)

        event_handler_parser = EventHandlerParser()
        stream = event_handler_parser.extract(stream)
        for event_type, priority, callback in event_handler_parser.get_handlers():
            self._eventdispatcher.add_listener(event_type, priority, callback)

        if self.options['queuestatus']:
            statuslog = SpreadFlowQueuestatusLogger(self.options['queuestatus'])
            statuslog.watch(1, self._scheduler)
            globalLogPublisher.addObserver(statuslog.logstatus)

        self._scheduler.run().addBoth(self._stop)

    def stopService(self):
        super(SpreadFlowService, self).stopService()
        return self._scheduler.join()

    def _stop(self, result):
        from twisted.internet import reactor
        try:
            reactor.stop()
        except error.ReactorNotRunning:
            pass
        return result

    def _oneshot_job_event_handler(self, event):
        scheduler = event.scheduler
        completed = event.completed

        def _stop_scheduler_when_done(result):
            if len(scheduler.pending) == 0:
                scheduler.stop(self)
            return result

        completed.addCallback(_stop_scheduler_when_done)
class SchedulerTestCase(TestCase):

    epsilon = task._EPSILON

    def setUp(self):
        super(SchedulerTestCase, self).setUp()

        self.clock = task.Clock()

        self.cooperator = task.Cooperator(
            terminationPredicateFactory=_no_termination_factory,
            scheduler=lambda x: self.clock.callLater(self.epsilon, x)
        )
        self.cooperate = self.cooperator.cooperate

        self.dispatcher = EventDispatcher()
        self.flowmap = dict()

        self.scheduler = Scheduler(self.flowmap, self.dispatcher, self.cooperate)

    def test_run_scheduler(self):
        """
        Tests run(), stop() and join() and ensures that appropriate events are fired.
        """
        job_handler = Mock()
        attach_handler = Mock()
        start_handler = Mock()
        join_handler = Mock()
        detach_handler = Mock()

        self.dispatcher.add_listener(JobEvent, 0, job_handler)
        self.dispatcher.add_listener(AttachEvent, 0, attach_handler)
        self.dispatcher.add_listener(DetachEvent, 0, detach_handler)

        # Start the scheduler.
        run_deferred = self.scheduler.run(self.clock)
        assert_that(run_deferred, twistedsupport.has_no_result())

        self.assertEquals(job_handler.call_count, 0)
        attach_handler.assert_called_once_with(AttachEvent(scheduler=self.scheduler, reactor=self.clock))
        self.assertEquals(join_handler.call_count, 0)
        self.assertEquals(detach_handler.call_count, 0)

        job_handler.reset_mock()
        attach_handler.reset_mock()
        start_handler.reset_mock()
        join_handler.reset_mock()
        detach_handler.reset_mock()

        # Turn a bit on the clock.
        self.clock.advance(self.epsilon)
        assert_that(run_deferred, twistedsupport.has_no_result())

        self.assertEquals(job_handler.call_count, 0)
        self.assertEquals(attach_handler.call_count, 0)
        self.assertEquals(start_handler.call_count, 0)
        self.assertEquals(join_handler.call_count, 0)
        self.assertEquals(detach_handler.call_count, 0)

        job_handler.reset_mock()
        attach_handler.reset_mock()
        start_handler.reset_mock()
        join_handler.reset_mock()
        detach_handler.reset_mock()

        # Stop the scheduler.
        self.scheduler.stop('some reason')
        assert_that(run_deferred, twistedsupport.succeeded(matchers.Equals('some reason')))

        self.assertEquals(job_handler.call_count, 0)
        self.assertEquals(attach_handler.call_count, 0)
        self.assertEquals(start_handler.call_count, 0)
        self.assertEquals(join_handler.call_count, 0)
        self.assertEquals(detach_handler.call_count, 0)

        job_handler.reset_mock()
        attach_handler.reset_mock()
        start_handler.reset_mock()
        join_handler.reset_mock()
        detach_handler.reset_mock()

        # Stop the queue and join processes.
        join_deferred = self.scheduler.join()
        self.clock.advance(self.epsilon)
        assert_that(join_deferred, twistedsupport.succeeded(matchers.Always()))

        self.assertEquals(job_handler.call_count, 0)
        self.assertEquals(attach_handler.call_count, 0)
        self.assertEquals(start_handler.call_count, 0)
        detach_handler.assert_called_once_with(DetachEvent(scheduler=self.scheduler))

    def test_run_job(self):
        """
        Tests send() and ensure that the job-event is fired.
        """
        job_handler = Mock()
        self.dispatcher.add_listener(JobEvent, 0, job_handler)

        port_out = object()
        port_in = Mock(spec=_port_callback)
        self.flowmap[port_out] = port_in

        self.scheduler.run(self.clock)
        self.scheduler.send('some item', port_out)

        expected_job = Job(port_in, 'some item', self.scheduler.send, port_out)
        self.assertEquals(job_handler.call_count, 1)
        assert_that(job_handler.call_args, MatchesInvocation(
            MatchesEvent(JobEvent,
                         scheduler=matchers.Equals(self.scheduler),
                         job=matchers.Equals(expected_job),
                         completed=twistedsupport.has_no_result())
        ))
        self.assertEquals(port_in.call_count, 0)

        self.assertEquals(len(list(self.scheduler.pending)), 1)

        # Trigger queue run.
        self.clock.advance(self.epsilon)

        port_in.assert_called_once_with('some item', self.scheduler.send)

        self.assertEquals(len(list(self.scheduler.pending)), 0)

    def test_fail_job(self):
        """
        Tests that scheduler is stopped whenever a port is failing.
        """
        port_out = object()
        port_in = Mock(spec=_port_callback, side_effect=RuntimeError('failed!'))
        self.flowmap[port_out] = port_in

        run_deferred = self.scheduler.run(self.clock)
        self.scheduler.send('some item', port_out)

        expected_message = 'Job failed on {:s} while processing {:s}'.format(
            str(port_in), 'some item')

        from testtools.twistedsupport._runtest import _NoTwistedLogObservers
        with _NoTwistedLogObservers():
            with twistedsupport.CaptureTwistedLogs() as twisted_logs:
                # Trigger queue run.
                self.clock.advance(self.epsilon)
                assert_that(twisted_logs.getDetails(), matchers.MatchesDict({
                    'twisted-log': matchers.AfterPreprocessing(
                        lambda log: log.as_text(), matchers.Contains(expected_message))
                }))

        port_in.assert_called_once_with('some item', self.scheduler.send)

        matcher = matchers.AfterPreprocessing(lambda f: f.value, matchers.IsInstance(RuntimeError))
        assert_that(run_deferred, twistedsupport.failed(matcher))

    def test_cancel_job(self):
        """
        Tests that scheduler is stopped whenever a port is failing.
        """
        port_out = object()
        port_in = Mock(spec=_port_callback)
        self.flowmap[port_out] = port_in

        run_deferred = self.scheduler.run(self.clock)
        self.scheduler.send('some item', port_out)

        self.assertEquals(len(list(self.scheduler.pending)), 1)

        self.scheduler.stop('bye!')

        self.assertEquals(len(list(self.scheduler.pending)), 1)

        join_deferred = self.scheduler.join()
        self.clock.advance(self.epsilon)
        assert_that(join_deferred, twistedsupport.succeeded(matchers.Always()))
        assert_that(run_deferred, twistedsupport.succeeded(matchers.Equals('bye!')))

        self.assertEquals(len(list(self.scheduler.pending)), 0)

        self.assertEquals(port_in.call_count, 0)