def dispatch(self, data, topics=None, source=None):
"""
Passes an event (aka the data) received to the right handlers.
If topic is not None, the event is dispatched to handlers registered
to that topic only + global handlers. Else it is dispatched to global
handlers only.
Even if a handler was registered multiple times in multiple topics, it
will be executed only once per call to `dispatch()`.
"""
# Always call registered global handlers
handlers = self._global_handlers
# Parse topics if iterable
if isinstance(topics, str):
topics = (topics, )
if topics is not None:
for topic in topics:
try:
handlers = handlers | self._topic_handlers[topic]
except KeyError:
pass
# Schedule the execution of all registered handlers
for handler in handlers:
# Automatically wrap input data into an event object
if isinstance(data, Event):
event = Event(data, topic=data.topic, source=data.source)
else:
event = Event(data, topics, source)
if asyncio.iscoroutinefunction(handler):
asyncio.ensure_future(handler(event), loop=self._loop)
else:
self._loop.call_soon(handler, event)
async def _run_next_tasks(self, task):
"""
A callback to be added to each task in order to select and schedule
asynchronously downstream tasks once the parent task is done.
"""
self._done_tasks.add(task)
if self._must_cancel:
self._try_mark_done()
return
# Don't execute downstream tasks if the task's result is an exception
# (may include task cancellation) but don't stop executing the other
# branches of the workflow.
try:
result = task.result()
except SkipTask:
# If the task has been skipped, we just forward the previous task
# inputs to the next one.
result = task.inputs
log.warning('task %s has been skipped', task.template)
except asyncio.CancelledError:
log.warning('task %s has been cancelled', task.template)
self._try_mark_done()
return
except Exception as exc:
log.warning('task %s ended on exception', task.template)
log.exception(exc)
self._try_mark_done()
return
# Ensure the workflow is not suspended, else wait for a resume.
try:
await self._committed.wait()
except asyncio.CancelledError:
# If cancelled, the workflow will return with the statement below.
log.debug('Suspended workflow cancelled')
return
source = EventSource(workflow_template_id=self.template.uid,
workflow_exec_id=self.uid,
task_template_id=task.template.uid,
task_exec_id=task.uid)
# Go through each child task
for tmpl in self._get_next_task_templates(task.template, task):
# Wrap result from parent task into an event object
event = Event(result, source=source)
next_task = self._tasks_by_id.get(tmpl.uid)
if next_task:
# Ignore done tasks
if next_task.done():
continue
# Downstream task already running, join it!
self._join_task(next_task, event)
# Create new task
else:
next_task = self._new_task(tmpl, event)
if not next_task:
break
self._try_mark_done()
async def data_received(self, data, topic=None):
"""
This method should be called to pass an event to the workflow engine
which in turn will disptach this event to the right running workflows
and may trigger new workflow executions.
"""
log.debug("data received: %s (topic=%s)", data, topic)
event = Event(data, topic=topic)
# Disptatch data to 'listening' tasks at all cases
self._broker.dispatch(event, topic)
# Don't start new workflow instances if `stop()` was called.
if self._must_stop:
log.debug("The engine is stopping, cannot trigger new workflows")
return
async with self._lock:
if asyncio.iscoroutinefunction(self._selector.select):
templates = await self._selector.select(topic)
else:
templates = self._selector.select(topic)
# Try to trigger new workflows from the current dict of workflow
# templates at all times!
wflows = []
for tmpl in templates:
wflow = self._try_run(tmpl, event)
if wflow:
wflows.append(wflow)
return wflows
async def run_once(self, template, data):
"""
Starts a new execution of the workflow template regardless of the
overrun policy and already running workflows.
Note: it does NOT load the workflow template in the engine.
"""
if self._must_stop:
log.debug("The engine is stopping, cannot run a new workflow from"
"template id %s", template.uid)
return None
with await self._lock:
wflow = Workflow(template, loop=self._loop)
self._do_run(wflow, Event(data))
return wflow
async def trigger(self, template_id, data):
"""
Trigger a new execution of the workflow template identified by
`template_id`. Use this method instead of a reserved topic +
`data_received` to trigger a specific workflow.
"""
with await self._lock:
# Ignore unknown (aka not loaded) workflow templates
try:
template = self._selector.get(template_id)
except KeyError:
log.error('workflow template %s not loaded', template_id)
return None
return self._try_run(template, Event(data))
async def trigger(self, template_id, data):
"""
Trigger a new execution of the workflow template identified by
`template_id`. Use this method instead of a reserved topic +
`data_received` to trigger a specific workflow.
"""
async with self._lock:
# Ignore unknown (aka not loaded) workflow templates
if asyncio.iscoroutinefunction(self._selector.get):
template = await self._selector.get(template_id)
else:
template = self._selector.get(template_id)
if not template:
return None
return self._try_run(template, Event(data))
def resume(self):
"""
Resume the execution of this workflow allows new tasks to start.
"""
if self._committed.is_set():
log.error('Cannot resume a workflow that has not been suspended')
return
self._committed.set()
# Next tasks are done waiting for '_committed' asyncio event
# The 'suspended' ones needs to be re-executed
for task in self._done_tasks:
if FutureState.get(task) is not FutureState.suspended:
continue
event = Event(task.inputs, source=task.event_source)
self._new_task(task.template, event)
self._dispatch_exec_event(WorkflowExecState.resume)
log.info('workflow %s has been resumed', self)
def fast_forward(self, report):
"""
Restore a workflow and its tasks states, parsing an execution report.
"""
self.uid = report['exec']['id']
self._start = report['exec']['start']
self._end = report['exec'].get('end')
self._source = EventSource(workflow_template_id=self._template.uid,
workflow_exec_id=self.uid)
log.info('fast forward workflow %s', self)
# Build a list of task in the graph that need to be executed
resume = list()
def browse(entry, parent=None):
"""
Dive into the dag to:
- ensure executed task contexts are restored.
- find tasks that need to be executed.
"""
t_template = next(t for t in report['tasks']
if t['id'] == entry.uid)
t_report = t_template.get('exec')
if not t_report:
if not parent:
raise RescueError(self.uid, 'root task never been started')
# No execution report found, the task needs to be executed
resume.append((entry, None, parent))
return
t_state = FutureState(t_report['state'])
if not t_state.done():
# Pending, suspended or cancelled tasks need to be executed
resume.append((entry, t_template, parent))
return
if t_state.done():
# We won't manually create an asyncio task since it won't be
# run (the task is 'done'). Yet, we need a memory print of its
# execution (mainly for reporting purposes).
t_shadow = type(
'TukioTaskShadow', (object, ), {
'done': lambda: True,
'cancelled': lambda: False,
'_exception': None,
'as_dict': lambda: t_report
})
# Add this task to the tracking sets
self.tasks.add(t_shadow)
self._done_tasks.add(t_shadow)
self._tasks_by_id[entry.uid] = t_shadow
# Recursive browing
for t_next in self._template.dag.successors(entry):
browse(t_next, t_template)
browse(self._template.root())
# Resume/start tasks that need to be executed
for task_template, task_report, parent_report in resume:
if parent_report:
inputs = parent_report['exec']['outputs']
source = EventSource(workflow_template_id=self.template.uid,
workflow_exec_id=self.uid,
task_template_id=parent_report['id'],
task_exec_id=parent_report['exec']['id'])
event = Event(inputs, source=source)
else:
# No parent report = root task
event = Event(task_report['exec']['inputs'])
self._new_task(task_template, event)
self._dispatch_exec_event(WorkflowExecState.begin)