def dispatch(self, data, topics=None, source=None): """ Passes an event (aka the data) received to the right handlers. If topic is not None, the event is dispatched to handlers registered to that topic only + global handlers. Else it is dispatched to global handlers only. Even if a handler was registered multiple times in multiple topics, it will be executed only once per call to `dispatch()`. """ # Always call registered global handlers handlers = self._global_handlers # Parse topics if iterable if isinstance(topics, str): topics = (topics, ) if topics is not None: for topic in topics: try: handlers = handlers | self._topic_handlers[topic] except KeyError: pass # Schedule the execution of all registered handlers for handler in handlers: # Automatically wrap input data into an event object if isinstance(data, Event): event = Event(data, topic=data.topic, source=data.source) else: event = Event(data, topics, source) if asyncio.iscoroutinefunction(handler): asyncio.ensure_future(handler(event), loop=self._loop) else: self._loop.call_soon(handler, event)
async def _run_next_tasks(self, task): """ A callback to be added to each task in order to select and schedule asynchronously downstream tasks once the parent task is done. """ self._done_tasks.add(task) if self._must_cancel: self._try_mark_done() return # Don't execute downstream tasks if the task's result is an exception # (may include task cancellation) but don't stop executing the other # branches of the workflow. try: result = task.result() except SkipTask: # If the task has been skipped, we just forward the previous task # inputs to the next one. result = task.inputs log.warning('task %s has been skipped', task.template) except asyncio.CancelledError: log.warning('task %s has been cancelled', task.template) self._try_mark_done() return except Exception as exc: log.warning('task %s ended on exception', task.template) log.exception(exc) self._try_mark_done() return # Ensure the workflow is not suspended, else wait for a resume. try: await self._committed.wait() except asyncio.CancelledError: # If cancelled, the workflow will return with the statement below. log.debug('Suspended workflow cancelled') return source = EventSource(workflow_template_id=self.template.uid, workflow_exec_id=self.uid, task_template_id=task.template.uid, task_exec_id=task.uid) # Go through each child task for tmpl in self._get_next_task_templates(task.template, task): # Wrap result from parent task into an event object event = Event(result, source=source) next_task = self._tasks_by_id.get(tmpl.uid) if next_task: # Ignore done tasks if next_task.done(): continue # Downstream task already running, join it! self._join_task(next_task, event) # Create new task else: next_task = self._new_task(tmpl, event) if not next_task: break self._try_mark_done()
async def data_received(self, data, topic=None): """ This method should be called to pass an event to the workflow engine which in turn will disptach this event to the right running workflows and may trigger new workflow executions. """ log.debug("data received: %s (topic=%s)", data, topic) event = Event(data, topic=topic) # Disptatch data to 'listening' tasks at all cases self._broker.dispatch(event, topic) # Don't start new workflow instances if `stop()` was called. if self._must_stop: log.debug("The engine is stopping, cannot trigger new workflows") return async with self._lock: if asyncio.iscoroutinefunction(self._selector.select): templates = await self._selector.select(topic) else: templates = self._selector.select(topic) # Try to trigger new workflows from the current dict of workflow # templates at all times! wflows = [] for tmpl in templates: wflow = self._try_run(tmpl, event) if wflow: wflows.append(wflow) return wflows
async def run_once(self, template, data): """ Starts a new execution of the workflow template regardless of the overrun policy and already running workflows. Note: it does NOT load the workflow template in the engine. """ if self._must_stop: log.debug("The engine is stopping, cannot run a new workflow from" "template id %s", template.uid) return None with await self._lock: wflow = Workflow(template, loop=self._loop) self._do_run(wflow, Event(data)) return wflow
async def trigger(self, template_id, data): """ Trigger a new execution of the workflow template identified by `template_id`. Use this method instead of a reserved topic + `data_received` to trigger a specific workflow. """ with await self._lock: # Ignore unknown (aka not loaded) workflow templates try: template = self._selector.get(template_id) except KeyError: log.error('workflow template %s not loaded', template_id) return None return self._try_run(template, Event(data))
async def trigger(self, template_id, data): """ Trigger a new execution of the workflow template identified by `template_id`. Use this method instead of a reserved topic + `data_received` to trigger a specific workflow. """ async with self._lock: # Ignore unknown (aka not loaded) workflow templates if asyncio.iscoroutinefunction(self._selector.get): template = await self._selector.get(template_id) else: template = self._selector.get(template_id) if not template: return None return self._try_run(template, Event(data))
def resume(self): """ Resume the execution of this workflow allows new tasks to start. """ if self._committed.is_set(): log.error('Cannot resume a workflow that has not been suspended') return self._committed.set() # Next tasks are done waiting for '_committed' asyncio event # The 'suspended' ones needs to be re-executed for task in self._done_tasks: if FutureState.get(task) is not FutureState.suspended: continue event = Event(task.inputs, source=task.event_source) self._new_task(task.template, event) self._dispatch_exec_event(WorkflowExecState.resume) log.info('workflow %s has been resumed', self)
def fast_forward(self, report): """ Restore a workflow and its tasks states, parsing an execution report. """ self.uid = report['exec']['id'] self._start = report['exec']['start'] self._end = report['exec'].get('end') self._source = EventSource(workflow_template_id=self._template.uid, workflow_exec_id=self.uid) log.info('fast forward workflow %s', self) # Build a list of task in the graph that need to be executed resume = list() def browse(entry, parent=None): """ Dive into the dag to: - ensure executed task contexts are restored. - find tasks that need to be executed. """ t_template = next(t for t in report['tasks'] if t['id'] == entry.uid) t_report = t_template.get('exec') if not t_report: if not parent: raise RescueError(self.uid, 'root task never been started') # No execution report found, the task needs to be executed resume.append((entry, None, parent)) return t_state = FutureState(t_report['state']) if not t_state.done(): # Pending, suspended or cancelled tasks need to be executed resume.append((entry, t_template, parent)) return if t_state.done(): # We won't manually create an asyncio task since it won't be # run (the task is 'done'). Yet, we need a memory print of its # execution (mainly for reporting purposes). t_shadow = type( 'TukioTaskShadow', (object, ), { 'done': lambda: True, 'cancelled': lambda: False, '_exception': None, 'as_dict': lambda: t_report }) # Add this task to the tracking sets self.tasks.add(t_shadow) self._done_tasks.add(t_shadow) self._tasks_by_id[entry.uid] = t_shadow # Recursive browing for t_next in self._template.dag.successors(entry): browse(t_next, t_template) browse(self._template.root()) # Resume/start tasks that need to be executed for task_template, task_report, parent_report in resume: if parent_report: inputs = parent_report['exec']['outputs'] source = EventSource(workflow_template_id=self.template.uid, workflow_exec_id=self.uid, task_template_id=parent_report['id'], task_exec_id=parent_report['exec']['id']) event = Event(inputs, source=source) else: # No parent report = root task event = Event(task_report['exec']['inputs']) self._new_task(task_template, event) self._dispatch_exec_event(WorkflowExecState.begin)