def test_fires_deferred_when_ready(self):
"""
A deferred returned from :meth:`spreadflow_core.jobqueue.JobQueue.next`
is fired after a new job is added to the queue.
"""
channel = object()
queue = JobQueue()
queue_ready_1 = _next(queue)
self.assertIsInstance(queue_ready_1, defer.Deferred)
self.assertFalse(queue_ready_1.called)
job_completed = queue.put(channel, lambda: "Bazinga!")
job_completed.addCallback(lambda result: self.assertEqual(result, "Bazinga!"))
# Scheduling a job must result in the deferred being called.
self.assertTrue(queue_ready_1.called)
# Scheduling a job must in direct execution.
self.assertFalse(job_completed.called)
# Iterating the queue should obviously execute the job.
queue_ready_2 = _next(queue)
self.assertTrue(job_completed.called)
# After executing one job, the queue must not return a deferred in
# order to signal the cooperator task that it wants to run again
# immediately.
self.assertIsNone(queue_ready_2)
# Should return a new deferred if the queue gets empty again.
queue_ready_3 = _next(queue)
self.assertNotEqual(queue_ready_1, queue_ready_3)
self.assertFalse(queue_ready_3.called)
def test_propagates_cancel_to_job(self):
"""
Cancellation is propagated to the inner deferred if a job is being
executed.
"""
channel = object()
queue = JobQueue()
inner = defer.Deferred()
outer = queue.put(channel, lambda: inner)
queue_ready = _next(queue)
self.assertIsNone(queue_ready)
outer.addErrback(lambda failure: failure.trap(defer.CancelledError))
outer.cancel()
self.assertTrue(outer.called)
self.assertTrue(inner.called)
def test_cancel_job_early(self):
"""
A job which is cancelled while it waits in the backlog does not execute
at a later stage.
"""
def job():
self.fail("Must not be executed after cancellation")
channel = object()
queue = JobQueue()
outer = queue.put(channel, job)
outer.addErrback(lambda failure: failure.trap(defer.CancelledError))
outer.cancel()
queue_ready = _next(queue)
self.assertIsInstance(queue_ready, defer.Deferred)
self.assertTrue(outer.called)
def __init__(self, flowmap, eventdispatcher, cooperate=None):
if cooperate is None:
from twisted.internet.task import cooperate
self.flowmap = flowmap
self.eventdispatcher = eventdispatcher
self.cooperate = cooperate
self._done = defer.Deferred()
self._pending = {}
self._queue = JobQueue()
self._queue_done = None
self._queue_task = None
self._stopped = False
self._detached = False
def test_iterates_in_fifo_order(self):
"""
Jobs are executed in FIFO order.
"""
results = []
def job(*args, **kwds):
results.append((args, kwds))
channel = object()
queue = JobQueue()
self.assertEqual(len(results), 0)
queue.put(channel, job, "first", arg="one")
queue.put(channel, job, "second")
queue.put(channel, job)
queue.put(channel, job, "fourth", "additional", "positional", plus="some", key="words")
queue_ready = []
for times in range(4):
self.assertEqual(len(results), times)
queue_ready = _next(queue)
self.assertIsNone(queue_ready)
self.assertEqual(
results,
[
(("first",), {"arg": "one"}),
(("second",), {}),
((), {}),
(("fourth", "additional", "positional"), {"plus": "some", "key": "words"}),
],
)
queue_ready = _next(queue)
self.assertIsInstance(queue_ready, defer.Deferred)
class Scheduler(object):
log = Logger()
def __init__(self, flowmap, eventdispatcher, cooperate=None):
if cooperate is None:
from twisted.internet.task import cooperate
self.flowmap = flowmap
self.eventdispatcher = eventdispatcher
self.cooperate = cooperate
self._done = defer.Deferred()
self._pending = {}
self._queue = JobQueue()
self._queue_done = None
self._queue_task = None
self._stopped = False
self._detached = False
def _job_callback(self, result, completed):
self._pending.pop(completed)
return result
def _job_errback(self, reason, job):
if not self._stopped:
self.log.failure('Job failed on {job.port} while processing {job.item}', reason, job=job)
self.stop(reason)
else:
return reason
def _job_cancel(self, entry):
subtask, _ = self._pending[entry.deferred]
subtask.cancel()
def _enqueue(self, job):
completed = defer.Deferred(lambda dfr: self._job_cancel(Entry(dfr, job)))
defered = self.eventdispatcher.dispatch(JobEvent(scheduler=self, job=job, completed=completed))
defered.addCallback(lambda ignored, job: self._queue.put(job.port, job.handler, job.item, job.send), job)
defered.pause()
self._pending[completed] = Entry(defered, job)
defered.addBoth(self._job_callback, completed)
defered.chainDeferred(completed)
defered.unpause()
return completed
def send(self, item, port_out):
assert not self._detached, 'Must not send() any items after ports have been detached'
if port_out in self.flowmap:
port_in = self.flowmap[port_out]
job = Job(port_in, item, self.send, origin=port_out)
completed = self._enqueue(job)
completed.addErrback(self._job_errback, job)
@property
def pending(self):
return self._pending.items()
@defer.inlineCallbacks
def run(self, reactor=None):
assert self._queue_task is None and not self._stopped, 'Must not call start() more than once'
if reactor == None:
from twisted.internet import reactor
self.log.info('Starting scheduler')
self.log.debug('Attaching sources and services')
yield self.eventdispatcher.dispatch(AttachEvent(scheduler=self, reactor=reactor))
self.log.debug('Attached sources and services')
self.log.debug('Starting queue')
self._queue_task = self.cooperate(self._queue)
self._queue_done = self._queue_task.whenDone()
self.log.debug('Started queue')
self.log.info('Started scheduler')
reason = yield self._done
defer.returnValue(reason)
def stop(self, reason):
if not self._stopped:
self.log.info('Stopping scheduler', reason=reason)
self._stopped = True
self._done.callback(reason)
return reason
def _logfail(self, failure, fmt, *args, **kwds):
"""
Errback: Logs and consumes a failure.
"""
self.log.failure(fmt, failure, *args, **kwds)
@defer.inlineCallbacks
def join(self, reactor=None, timeout=10.0):
if reactor == None:
from twisted.internet import reactor
# Prevent that any queued items are run.
self._stopped = True
self._queue_task.pause()
self.log.debug('Detaching sources and services')
event = DetachEvent(scheduler=self)
deferred_detach = self.eventdispatcher.dispatch(event, fail_mode=FailMode.RETURN).addCallback(self.eventdispatcher.log_failures, event)
delayed_call = reactor.callLater(timeout, deferred_detach.cancel)
yield deferred_detach
delayed_call.cancel()
self.log.debug('Detached sources and services')
# Prevent that new items are enqueued.
self._detached = True
# Clear the backlog and wait for queue termination.
self.log.debug('Cancel {pending_len} pending jobs', pending=self._pending, pending_len=len(self._pending))
_trapcancel = lambda f: f.trap(defer.CancelledError)
for deferred_job, (_, job) in self._pending.items():
deferred_job.addErrback(_trapcancel)
deferred_job.addErrback(self._logfail, 'Failed to cancel job', job=job)
for deferred_job in list(self._pending.keys()):
deferred_job.cancel()
self.log.debug('Stopping queue')
self._queue.stopempty = True
self._queue_task.resume()
yield self._queue_done
self.log.debug('Stopped queue')
self._queue_done = None
self._queue_task = None
self.log.info('Stopped scheduler')