def test_dispatch_with_args_kwds(self):
dispatcher = EventDispatcher()
callback = Mock()
dispatcher.add_listener(TestEvent, 0, callback, 'hello', 'world', some='kwds', also='here')
event = TestEvent()
d = dispatcher.dispatch(event)
self.assertTrue(d.called)
callback.assert_called_once_with(event, 'hello', 'world', some='kwds', also='here')
def test_remove_non_existing_listeners(self):
dispatcher = EventDispatcher()
key = dispatcher.add_listener(TestEvent, 0, lambda event: None)
# Valid key on invalid event type.
self.assertRaises(KeyError, dispatcher.remove_listener, OtherEvent, key)
# Invalid key on valid event type.
self.assertRaises(KeyError, dispatcher.remove_listener, TestEvent, key._replace(priority=99))
# Really remove listener.
dispatcher.remove_listener(TestEvent, key)
# Double removal with valid key on valid event type.
self.assertRaises(KeyError, dispatcher.remove_listener, TestEvent, key)
def test_dispatch_with_return_fails(self):
dispatcher = EventDispatcher()
test_callback_prio_0_cb_0 = Mock(return_value='hello')
test_callback_prio_0_cb_1 = Mock(side_effect=RuntimeError('boom!'))
test_callback_prio_1_cb_0 = Mock(return_value='world')
dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_0)
dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_1)
dispatcher.add_listener(TestEvent, 1, test_callback_prio_1_cb_0)
event = TestEvent()
d = dispatcher.dispatch(event, fail_mode=FailMode.RETURN)
matcher = matchers.MatchesListwise([
matchers.MatchesListwise([
matchers.Equals(0), matchers.MatchesListwise([
matchers.Equals((True, 'hello')),
matchers.MatchesListwise([
matchers.Equals(False),
matchers.AfterPreprocessing(lambda f: f.value, matchers.IsInstance(HandlerError)),
])
]),
]),
matchers.MatchesListwise([
matchers.Equals(1), matchers.MatchesListwise([
matchers.Equals((True, 'world')),
]),
]),
])
assert_that(d, twistedsupport.succeeded(matcher))
def startService(self):
super(SpreadFlowService, self).startService()
if self.options['confpath']:
confpath = self.options['confpath']
else:
confpath = os.path.join(os.getcwd(), 'spreadflow.conf')
stream = config_eval(confpath)
pipeline = list()
pipeline.append(AliasResolverPass())
pipeline.append(PortsValidatorPass())
if self.options['multiprocess']:
pipeline.append(PartitionExpanderPass())
pipeline.append(PartitionBoundsPass())
if self.options['partition']:
pipeline.append(PartitionWorkerPass())
partition = self.options['partition']
stream.append(AddTokenOp(PartitionSelectToken(partition)))
else:
pipeline.append(PartitionControllersPass())
pipeline.append(ComponentsPurgePass())
pipeline.append(EventHandlersPass())
for compiler_step in pipeline:
stream = compiler_step(stream)
self._eventdispatcher = EventDispatcher()
if self.options['oneshot']:
self._eventdispatcher.add_listener(JobEvent, 0, self._oneshot_job_event_handler)
connection_parser = ConnectionParser()
stream = connection_parser.extract(stream)
self._scheduler = Scheduler(connection_parser.get_portmap(), self._eventdispatcher)
event_handler_parser = EventHandlerParser()
stream = event_handler_parser.extract(stream)
for event_type, priority, callback in event_handler_parser.get_handlers():
self._eventdispatcher.add_listener(event_type, priority, callback)
if self.options['queuestatus']:
statuslog = SpreadFlowQueuestatusLogger(self.options['queuestatus'])
statuslog.watch(1, self._scheduler)
globalLogPublisher.addObserver(statuslog.logstatus)
self._scheduler.run().addBoth(self._stop)
def setUp(self):
super(SchedulerTestCase, self).setUp()
self.clock = task.Clock()
self.cooperator = task.Cooperator(
terminationPredicateFactory=_no_termination_factory,
scheduler=lambda x: self.clock.callLater(self.epsilon, x)
)
self.cooperate = self.cooperator.cooperate
self.dispatcher = EventDispatcher()
self.flowmap = dict()
self.scheduler = Scheduler(self.flowmap, self.dispatcher, self.cooperate)
def test_dispatch_with_fails(self):
dispatcher = EventDispatcher()
test_callback_prio_0_cb_0 = Mock()
test_callback_prio_0_cb_1 = Mock(side_effect=RuntimeError('boom!'))
test_callback_prio_1_cb_0 = Mock()
dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_0)
dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_1)
dispatcher.add_listener(TestEvent, 1, test_callback_prio_1_cb_0)
event = TestEvent()
d = dispatcher.dispatch(event)
matcher = matchers.AfterPreprocessing(lambda f: f.value, matchers.IsInstance(HandlerError))
assert_that(d, twistedsupport.failed(matcher))
def test_dispatch_event(self):
dispatcher = EventDispatcher()
test_callback_prio_0_cb_0 = Mock()
test_callback_prio_0_cb_1 = Mock()
test_callback_prio_1_cb_0 = Mock()
other_callback_prio_2_cb_0 = Mock()
dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_0)
key_prio_0_cb_1 = dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_1)
dispatcher.add_listener(TestEvent, 1, test_callback_prio_1_cb_0)
dispatcher.add_listener(OtherEvent, 2, other_callback_prio_2_cb_0)
event = TestEvent()
d = dispatcher.dispatch(event)
self.assertTrue(d.called)
test_callback_prio_0_cb_0.assert_called_once_with(event)
test_callback_prio_0_cb_1.assert_called_once_with(event)
test_callback_prio_1_cb_0.assert_called_once_with(event)
self.assertEqual(other_callback_prio_2_cb_0.call_count, 0)
dispatcher.remove_listener(TestEvent, key_prio_0_cb_1)
test_callback_prio_0_cb_0.reset_mock()
test_callback_prio_0_cb_1.reset_mock()
test_callback_prio_1_cb_0.reset_mock()
other_callback_prio_2_cb_0.reset_mock()
event = TestEvent()
d = dispatcher.dispatch(event)
self.assertTrue(d.called)
test_callback_prio_0_cb_0.assert_called_once_with(event)
self.assertEqual(test_callback_prio_0_cb_1.call_count, 0)
test_callback_prio_1_cb_0.assert_called_once_with(event)
self.assertEqual(other_callback_prio_2_cb_0.call_count, 0)
def test_manage_listeners(self):
dispatcher = EventDispatcher()
listeners = list(dispatcher.get_listeners(TestEvent))
self.assertEqual(len(listeners), 0)
test_callback_prio_0_cb_0 = Mock()
test_callback_prio_0_cb_1 = Mock()
test_callback_prio_1_cb_0 = Mock()
other_callback_prio_2_cb_0 = Mock()
# Register callbacks, first priority 1 ...
dispatcher.add_listener(TestEvent, 1, test_callback_prio_1_cb_0)
# ... aftwerwards priority 0 ...
key_prio_0_cb_0 = dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_0)
dispatcher.add_listener(TestEvent, 0, test_callback_prio_0_cb_1)
# ... and finally priority 2 for another event.
dispatcher.add_listener(OtherEvent, 2, other_callback_prio_2_cb_0)
# Collect callbacks from listeners list.
actual_handlers = [(key.priority, handler.callback) for key, handler in dispatcher.get_listeners(TestEvent)]
expected_handlers = [
(0, test_callback_prio_0_cb_0),
(0, test_callback_prio_0_cb_1),
(1, test_callback_prio_1_cb_0),
]
self.assertEqual(expected_handlers, actual_handlers)
# Remove one listener.
dispatcher.remove_listener(TestEvent, key_prio_0_cb_0)
# Collect callbacks from listeners list.
actual_handlers = [(key.priority, handler.callback) for key, handler in dispatcher.get_listeners(TestEvent)]
expected_handlers = [
(0, test_callback_prio_0_cb_1),
(1, test_callback_prio_1_cb_0),
]
self.assertEqual(expected_handlers, actual_handlers)
class SpreadFlowService(service.Service):
def __init__(self, options):
self.options = options
self._scheduler = None
self._eventdispatcher = None
def startService(self):
super(SpreadFlowService, self).startService()
if self.options['confpath']:
confpath = self.options['confpath']
else:
confpath = os.path.join(os.getcwd(), 'spreadflow.conf')
stream = config_eval(confpath)
pipeline = list()
pipeline.append(AliasResolverPass())
pipeline.append(PortsValidatorPass())
if self.options['multiprocess']:
pipeline.append(PartitionExpanderPass())
pipeline.append(PartitionBoundsPass())
if self.options['partition']:
pipeline.append(PartitionWorkerPass())
partition = self.options['partition']
stream.append(AddTokenOp(PartitionSelectToken(partition)))
else:
pipeline.append(PartitionControllersPass())
pipeline.append(ComponentsPurgePass())
pipeline.append(EventHandlersPass())
for compiler_step in pipeline:
stream = compiler_step(stream)
self._eventdispatcher = EventDispatcher()
if self.options['oneshot']:
self._eventdispatcher.add_listener(JobEvent, 0, self._oneshot_job_event_handler)
connection_parser = ConnectionParser()
stream = connection_parser.extract(stream)
self._scheduler = Scheduler(connection_parser.get_portmap(), self._eventdispatcher)
event_handler_parser = EventHandlerParser()
stream = event_handler_parser.extract(stream)
for event_type, priority, callback in event_handler_parser.get_handlers():
self._eventdispatcher.add_listener(event_type, priority, callback)
if self.options['queuestatus']:
statuslog = SpreadFlowQueuestatusLogger(self.options['queuestatus'])
statuslog.watch(1, self._scheduler)
globalLogPublisher.addObserver(statuslog.logstatus)
self._scheduler.run().addBoth(self._stop)
def stopService(self):
super(SpreadFlowService, self).stopService()
return self._scheduler.join()
def _stop(self, result):
from twisted.internet import reactor
try:
reactor.stop()
except error.ReactorNotRunning:
pass
return result
def _oneshot_job_event_handler(self, event):
scheduler = event.scheduler
completed = event.completed
def _stop_scheduler_when_done(result):
if len(scheduler.pending) == 0:
scheduler.stop(self)
return result
completed.addCallback(_stop_scheduler_when_done)
class SchedulerTestCase(TestCase):
epsilon = task._EPSILON
def setUp(self):
super(SchedulerTestCase, self).setUp()
self.clock = task.Clock()
self.cooperator = task.Cooperator(
terminationPredicateFactory=_no_termination_factory,
scheduler=lambda x: self.clock.callLater(self.epsilon, x)
)
self.cooperate = self.cooperator.cooperate
self.dispatcher = EventDispatcher()
self.flowmap = dict()
self.scheduler = Scheduler(self.flowmap, self.dispatcher, self.cooperate)
def test_run_scheduler(self):
"""
Tests run(), stop() and join() and ensures that appropriate events are fired.
"""
job_handler = Mock()
attach_handler = Mock()
start_handler = Mock()
join_handler = Mock()
detach_handler = Mock()
self.dispatcher.add_listener(JobEvent, 0, job_handler)
self.dispatcher.add_listener(AttachEvent, 0, attach_handler)
self.dispatcher.add_listener(DetachEvent, 0, detach_handler)
# Start the scheduler.
run_deferred = self.scheduler.run(self.clock)
assert_that(run_deferred, twistedsupport.has_no_result())
self.assertEquals(job_handler.call_count, 0)
attach_handler.assert_called_once_with(AttachEvent(scheduler=self.scheduler, reactor=self.clock))
self.assertEquals(join_handler.call_count, 0)
self.assertEquals(detach_handler.call_count, 0)
job_handler.reset_mock()
attach_handler.reset_mock()
start_handler.reset_mock()
join_handler.reset_mock()
detach_handler.reset_mock()
# Turn a bit on the clock.
self.clock.advance(self.epsilon)
assert_that(run_deferred, twistedsupport.has_no_result())
self.assertEquals(job_handler.call_count, 0)
self.assertEquals(attach_handler.call_count, 0)
self.assertEquals(start_handler.call_count, 0)
self.assertEquals(join_handler.call_count, 0)
self.assertEquals(detach_handler.call_count, 0)
job_handler.reset_mock()
attach_handler.reset_mock()
start_handler.reset_mock()
join_handler.reset_mock()
detach_handler.reset_mock()
# Stop the scheduler.
self.scheduler.stop('some reason')
assert_that(run_deferred, twistedsupport.succeeded(matchers.Equals('some reason')))
self.assertEquals(job_handler.call_count, 0)
self.assertEquals(attach_handler.call_count, 0)
self.assertEquals(start_handler.call_count, 0)
self.assertEquals(join_handler.call_count, 0)
self.assertEquals(detach_handler.call_count, 0)
job_handler.reset_mock()
attach_handler.reset_mock()
start_handler.reset_mock()
join_handler.reset_mock()
detach_handler.reset_mock()
# Stop the queue and join processes.
join_deferred = self.scheduler.join()
self.clock.advance(self.epsilon)
assert_that(join_deferred, twistedsupport.succeeded(matchers.Always()))
self.assertEquals(job_handler.call_count, 0)
self.assertEquals(attach_handler.call_count, 0)
self.assertEquals(start_handler.call_count, 0)
detach_handler.assert_called_once_with(DetachEvent(scheduler=self.scheduler))
def test_run_job(self):
"""
Tests send() and ensure that the job-event is fired.
"""
job_handler = Mock()
self.dispatcher.add_listener(JobEvent, 0, job_handler)
port_out = object()
port_in = Mock(spec=_port_callback)
self.flowmap[port_out] = port_in
self.scheduler.run(self.clock)
self.scheduler.send('some item', port_out)
expected_job = Job(port_in, 'some item', self.scheduler.send, port_out)
self.assertEquals(job_handler.call_count, 1)
assert_that(job_handler.call_args, MatchesInvocation(
MatchesEvent(JobEvent,
scheduler=matchers.Equals(self.scheduler),
job=matchers.Equals(expected_job),
completed=twistedsupport.has_no_result())
))
self.assertEquals(port_in.call_count, 0)
self.assertEquals(len(list(self.scheduler.pending)), 1)
# Trigger queue run.
self.clock.advance(self.epsilon)
port_in.assert_called_once_with('some item', self.scheduler.send)
self.assertEquals(len(list(self.scheduler.pending)), 0)
def test_fail_job(self):
"""
Tests that scheduler is stopped whenever a port is failing.
"""
port_out = object()
port_in = Mock(spec=_port_callback, side_effect=RuntimeError('failed!'))
self.flowmap[port_out] = port_in
run_deferred = self.scheduler.run(self.clock)
self.scheduler.send('some item', port_out)
expected_message = 'Job failed on {:s} while processing {:s}'.format(
str(port_in), 'some item')
from testtools.twistedsupport._runtest import _NoTwistedLogObservers
with _NoTwistedLogObservers():
with twistedsupport.CaptureTwistedLogs() as twisted_logs:
# Trigger queue run.
self.clock.advance(self.epsilon)
assert_that(twisted_logs.getDetails(), matchers.MatchesDict({
'twisted-log': matchers.AfterPreprocessing(
lambda log: log.as_text(), matchers.Contains(expected_message))
}))
port_in.assert_called_once_with('some item', self.scheduler.send)
matcher = matchers.AfterPreprocessing(lambda f: f.value, matchers.IsInstance(RuntimeError))
assert_that(run_deferred, twistedsupport.failed(matcher))
def test_cancel_job(self):
"""
Tests that scheduler is stopped whenever a port is failing.
"""
port_out = object()
port_in = Mock(spec=_port_callback)
self.flowmap[port_out] = port_in
run_deferred = self.scheduler.run(self.clock)
self.scheduler.send('some item', port_out)
self.assertEquals(len(list(self.scheduler.pending)), 1)
self.scheduler.stop('bye!')
self.assertEquals(len(list(self.scheduler.pending)), 1)
join_deferred = self.scheduler.join()
self.clock.advance(self.epsilon)
assert_that(join_deferred, twistedsupport.succeeded(matchers.Always()))
assert_that(run_deferred, twistedsupport.succeeded(matchers.Equals('bye!')))
self.assertEquals(len(list(self.scheduler.pending)), 0)
self.assertEquals(port_in.call_count, 0)
