def create_associated_triggers(db_workbook): if not db_workbook['definition']: return workbook = parser.get_workbook(db_workbook['definition']) triggers = workbook.get_triggers() # Prepare all triggers data in advance to make db transaction shorter. db_triggers = [] for e in triggers: pattern = e['parameters']['cron-pattern'] next_time = _get_next_execution_time(pattern, datetime.now()) db_triggers.append({ "name": e['name'], "pattern": pattern, "next_execution_time": next_time, "workbook_name": db_workbook['name'] }) db_api.start_tx() try: for e in db_triggers: db_api.trigger_create(e) db_api.commit_tx() finally: db_api.end_tx()
def run_delayed_task(context): """Runs the delayed task. Performs all the steps required to setup a task to run which are not already done. This is mostly code copied over from convey_task_result. :param context Mistral authentication context inherited from a caller thread. """ auth_context.set_ctx(context) db_api.start_tx() try: execution_id = task['execution_id'] execution = db_api.execution_get(execution_id) # Change state from DELAYED to RUNNING. WORKFLOW_TRACE.info("Task '%s' [%s -> %s]" % (task['name'], task['state'], states.RUNNING)) executables = data_flow.prepare_tasks([task], outbound_context, workbook) db_api.commit_tx() finally: db_api.end_tx() if states.is_stopped_or_finished(execution['state']): return for task_id, action_name, action_params in executables: self._run_task(task_id, action_name, action_params)
def run_delayed_task(): """ Runs the delayed task. Performs all the steps required to setup a task to run which are not already done. This is mostly code copied over from convey_task_result. """ db_api.start_tx() try: workbook_name = task['workbook_name'] execution_id = task['execution_id'] execution = db_api.execution_get(workbook_name, execution_id) # Change state from DELAYED to IDLE to unblock processing. WORKFLOW_TRACE.info("Task '%s' [%s -> %s]" % (task['name'], task['state'], states.IDLE)) db_task = db_api.task_update(workbook_name, execution_id, task['id'], {"state": states.IDLE}) task_to_start = [db_task] data_flow.prepare_tasks(task_to_start, outbound_context) db_api.commit_tx() finally: db_api.end_tx() if not states.is_stopped_or_finished(execution["state"]): cls._run_tasks(task_to_start)
def start_workflow_execution(cls, workbook_name, task_name, context): db_api.start_tx() workbook = cls._get_workbook(workbook_name) # Persist execution and tasks in DB. try: execution = cls._create_execution(workbook_name, task_name, context) tasks = cls._create_tasks( workflow.find_workflow_tasks(workbook, task_name), workbook, workbook_name, execution['id'] ) tasks_to_start = workflow.find_resolved_tasks(tasks) data_flow.prepare_tasks(tasks_to_start, context) db_api.commit_tx() except Exception as e: raise exc.EngineException("Failed to create necessary DB objects:" " %s" % e) finally: db_api.end_tx() cls._run_tasks(tasks_to_start) return execution
def start_workflow_execution(self, cntx, **kwargs): """Starts a workflow execution based on the specified workbook name and target task. :param cntx: a request context dict :type cntx: MistralContext :param kwargs: a dict of method arguments :type kwargs: dict :return: Workflow execution. """ workbook_name = kwargs.get('workbook_name') task_name = kwargs.get('task_name') context = kwargs.get('context', None) context = copy.copy(context) if context else {} WORKFLOW_TRACE.info("New execution started - [workbook_name = '%s', " "task_name = '%s']" % (workbook_name, task_name)) db_api.start_tx() # Persist execution and tasks in DB. try: workbook = self._get_workbook(workbook_name) execution = self._create_execution(workbook_name, task_name, context) tasks = self._create_tasks( workflow.find_workflow_tasks(workbook, task_name), workbook, workbook_name, execution['id'] ) tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) self._add_variables_to_data_flow_context(context, execution) data_flow.prepare_tasks(tasks_to_start, context) db_api.commit_tx() except Exception as e: msg = "Failed to create necessary DB objects: %s" % e LOG.exception(msg) raise exc.EngineException(msg) finally: db_api.end_tx() for task in delayed_tasks: self._schedule_run(workbook, task, context) self._run_tasks(tasks_to_start) return execution
def convey_task_result(cls, workbook_name, execution_id, task_id, state, result): db_api.start_tx() workbook = cls._get_workbook(workbook_name) try: #TODO(rakhmerov): validate state transition task = db_api.task_get(workbook_name, execution_id, task_id) task_output = data_flow.get_task_output(task, result) # Update task state. task = db_api.task_update(workbook_name, execution_id, task_id, {"state": state, "output": task_output}) execution = db_api.execution_get(workbook_name, execution_id) # Calculate task outbound context. outbound_context = data_flow.get_outbound_context(task) cls._create_next_tasks(task, workbook) # Determine what tasks need to be started. tasks = db_api.tasks_get(workbook_name, execution_id) new_exec_state = cls._determine_execution_state(execution, tasks) if execution['state'] != new_exec_state: execution = \ db_api.execution_update(workbook_name, execution_id, { "state": new_exec_state }) LOG.info("Changed execution state: %s" % execution) tasks_to_start = workflow.find_resolved_tasks(tasks) data_flow.prepare_tasks(tasks_to_start, outbound_context) db_api.commit_tx() except Exception as e: raise exc.EngineException("Failed to create necessary DB objects:" " %s" % e) finally: db_api.end_tx() if states.is_stopped_or_finished(execution["state"]): return task if tasks_to_start: cls._run_tasks(tasks_to_start) return task
def _handle_task_error(self, task, exception): """Handle exception from the task execution. :param task: the task corresponding to the exception :type task: dict :param exception: an exception thrown during the execution of the task :type exception: Exception """ try: db_api.start_tx() try: db_api.execution_update(task['execution_id'], {'state': states.ERROR}) db_api.task_update(task['id'], {'state': states.ERROR}) db_api.commit_tx() finally: db_api.end_tx() except Exception as e: LOG.exception(e)
def start_workflow_execution(cls, workbook_name, task_name, context): context = copy.copy(context) if context else {} db_api.start_tx() WORKFLOW_TRACE.info("New execution started - [workbook_name = '%s', " "task_name = '%s']" % (workbook_name, task_name)) # Persist execution and tasks in DB. try: workbook = cls._get_workbook(workbook_name) execution = cls._create_execution(workbook_name, task_name, context) tasks = cls._create_tasks( workflow.find_workflow_tasks(workbook, task_name), workbook, workbook_name, execution['id'] ) tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) cls._add_variables_to_data_flow_context(context, execution) data_flow.prepare_tasks(tasks_to_start, context) db_api.commit_tx() except Exception as e: LOG.exception("Failed to create necessary DB objects.") raise exc.EngineException("Failed to create necessary DB objects:" " %s" % e) finally: db_api.end_tx() for task in delayed_tasks: cls._schedule_run(workbook, task, context) cls._run_tasks(tasks_to_start) return execution
def _handle_task_error(self, task, exception): """Handle exception from the task execution. :param task: the task corresponding to the exception :type task: dict :param exception: an exception thrown during the execution of the task :type exception: Exception """ try: db_api.start_tx() try: db_api.execution_update(task['workbook_name'], task['execution_id'], {'state': states.ERROR}) db_api.task_update(task['workbook_name'], task['execution_id'], task['id'], {'state': states.ERROR}) db_api.commit_tx() finally: db_api.end_tx() except Exception as e: LOG.exception(e)
def start_workflow_execution(cls, workbook_name, task_name, context): context = copy.copy(context) if context else {} db_api.start_tx() WORKFLOW_TRACE.info("New execution started - [workbook_name = '%s', " "task_name = '%s']" % (workbook_name, task_name)) # Persist execution and tasks in DB. try: workbook = cls._get_workbook(workbook_name) execution = cls._create_execution(workbook_name, task_name, context) tasks = cls._create_tasks( workflow.find_workflow_tasks(workbook, task_name), workbook, workbook_name, execution['id']) tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) cls._add_variables_to_data_flow_context(context, execution) data_flow.prepare_tasks(tasks_to_start, context) db_api.commit_tx() except Exception as e: LOG.exception("Failed to create necessary DB objects.") raise exc.EngineException("Failed to create necessary DB objects:" " %s" % e) finally: db_api.end_tx() for task in delayed_tasks: cls._schedule_run(workbook, task, context) cls._run_tasks(tasks_to_start) return execution
def start_workflow_execution(self, cntx, **kwargs): """Starts a workflow execution based on the specified workbook name and target task. :param cntx: a request context dict :type cntx: MistralContext :param kwargs: a dict of method arguments :type kwargs: dict :return: Workflow execution. """ workbook_name = kwargs.get('workbook_name') task_name = kwargs.get('task_name') context = kwargs.get('context', None) context = copy.copy(context) if context else {} WORKFLOW_TRACE.info("New execution started - [workbook_name = '%s', " "task_name = '%s']" % (workbook_name, task_name)) db_api.start_tx() # Persist execution and tasks in DB. try: workbook = self._get_workbook(workbook_name) execution = self._create_execution(workbook_name, task_name, context) # Create the whole tree of tasks required by target task, including # target task itself. tasks = self._create_tasks( workflow.find_workflow_tasks(workbook, task_name), workbook, workbook_name, execution['id'] ) # Create a list of tasks that can be executed immediately (have # their requirements satisfied, or, at that point, rather don't # have them at all) along with the list of tasks that require some # delay before they'll be executed. tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) # Populate context with special variables such as `openstack` and # `__execution`. self._add_variables_to_data_flow_context(context, execution) # Update task with new context and params. executables = data_flow.prepare_tasks(tasks_to_start, context, workbook) db_api.commit_tx() except Exception as e: msg = "Failed to create necessary DB objects: %s" % e LOG.exception(msg) raise exc.EngineException(msg) finally: db_api.end_tx() for task in delayed_tasks: self._schedule_run(workbook, task, context) for task_id, action_name, action_params in executables: self._run_task(task_id, action_name, action_params) return execution
def convey_task_result(self, cntx, **kwargs): """Conveys task result to Mistral Engine. This method should be used by clients of Mistral Engine to update state of a task once task action has been performed. One of the clients of this method is Mistral REST API server that receives task result from the outside action handlers. Note: calling this method serves an event notifying Mistral that it possibly needs to move the workflow on, i.e. run other workflow tasks for which all dependencies are satisfied. :param cntx: a request context dict :type cntx: dict :param kwargs: a dict of method arguments :type kwargs: dict :return: Task. """ task_id = kwargs.get('task_id') state = kwargs.get('state') result = kwargs.get('result') db_api.start_tx() try: # TODO(rakhmerov): validate state transition task = db_api.task_get(task_id) workbook = self._get_workbook(task['workbook_name']) wf_trace_msg = "Task '%s' [%s -> %s" % \ (task['name'], task['state'], state) wf_trace_msg += ']' if state == states.ERROR \ else ", result = %s]" % result WORKFLOW_TRACE.info(wf_trace_msg) action_name = wb_task.TaskSpec(task['task_spec'])\ .get_full_action_name() if not a_f.get_action_class(action_name): action = a_f.resolve_adhoc_action_name(workbook, action_name) if not action: msg = 'Unknown action [workbook=%s, action=%s]' % \ (workbook, action_name) raise exc.ActionException(msg) result = a_f.convert_adhoc_action_result(workbook, action_name, result) task_output = data_flow.get_task_output(task, result) # Update task state. task, context = self._update_task(workbook, task, state, task_output) execution = db_api.execution_get(task['execution_id']) self._create_next_tasks(task, workbook) # Determine what tasks need to be started. tasks = db_api.tasks_get(execution_id=task['execution_id']) new_exec_state = self._determine_execution_state(execution, tasks) if execution['state'] != new_exec_state: wf_trace_msg = \ "Execution '%s' [%s -> %s]" % \ (execution['id'], execution['state'], new_exec_state) WORKFLOW_TRACE.info(wf_trace_msg) execution = db_api.execution_update(execution['id'], { "state": new_exec_state }) LOG.info("Changed execution state: %s" % execution) # Create a list of tasks that can be executed immediately (have # their requirements satisfied) along with the list of tasks that # require some delay before they'll be executed. tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) # Populate context with special variables such as `openstack` and # `__execution`. self._add_variables_to_data_flow_context(context, execution) # Update task with new context and params. executables = data_flow.prepare_tasks(tasks_to_start, context, workbook) db_api.commit_tx() except Exception as e: msg = "Failed to create necessary DB objects: %s" % e LOG.exception(msg) raise exc.EngineException(msg) finally: db_api.end_tx() if states.is_stopped_or_finished(execution['state']): return task for task in delayed_tasks: self._schedule_run(workbook, task, context) for task_id, action_name, action_params in executables: self._run_task(task_id, action_name, action_params) return task
def convey_task_result(cls, workbook_name, execution_id, task_id, state, result): db_api.start_tx() try: workbook = cls._get_workbook(workbook_name) #TODO(rakhmerov): validate state transition task = db_api.task_get(workbook_name, execution_id, task_id) wf_trace_msg = "Task '%s' [%s -> %s" % \ (task['name'], task['state'], state) wf_trace_msg += ']' if state == states.ERROR \ else ", result = %s]" % result WORKFLOW_TRACE.info(wf_trace_msg) task_output = data_flow.get_task_output(task, result) # Update task state. task, outbound_context = cls._update_task(workbook, task, state, task_output) execution = db_api.execution_get(workbook_name, execution_id) cls._create_next_tasks(task, workbook) # Determine what tasks need to be started. tasks = db_api.tasks_get(workbook_name, execution_id) new_exec_state = cls._determine_execution_state(execution, tasks) if execution['state'] != new_exec_state: wf_trace_msg = \ "Execution '%s' [%s -> %s]" % \ (execution_id, execution['state'], new_exec_state) WORKFLOW_TRACE.info(wf_trace_msg) execution = \ db_api.execution_update(workbook_name, execution_id, { "state": new_exec_state }) LOG.info("Changed execution state: %s" % execution) tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) cls._add_variables_to_data_flow_context(outbound_context, execution) data_flow.prepare_tasks(tasks_to_start, outbound_context) db_api.commit_tx() except Exception as e: LOG.exception("Failed to create necessary DB objects.") raise exc.EngineException("Failed to create necessary DB objects:" " %s" % e) finally: db_api.end_tx() if states.is_stopped_or_finished(execution["state"]): return task for task in delayed_tasks: cls._schedule_run(workbook, task, outbound_context) if tasks_to_start: cls._run_tasks(tasks_to_start) return task
def convey_task_result(self, cntx, **kwargs): """Conveys task result to Mistral Engine. This method should be used by clients of Mistral Engine to update state of a task once task action has been performed. One of the clients of this method is Mistral REST API server that receives task result from the outside action handlers. Note: calling this method serves an event notifying Mistral that it possibly needs to move the workflow on, i.e. run other workflow tasks for which all dependencies are satisfied. :param cntx: a request context dict :type cntx: dict :param kwargs: a dict of method arguments :type kwargs: dict :return: Task. """ task_id = kwargs.get('task_id') state = kwargs.get('state') result = kwargs.get('result') db_api.start_tx() try: # TODO(rakhmerov): validate state transition task = db_api.task_get(task_id) workbook = self._get_workbook(task['workbook_name']) wf_trace_msg = "Task '%s' [%s -> %s" % \ (task['name'], task['state'], state) wf_trace_msg += ']' if state == states.ERROR \ else ", result = %s]" % result WORKFLOW_TRACE.info(wf_trace_msg) task_output = data_flow.get_task_output(task, result) # Update task state. task, outbound_context = self._update_task(workbook, task, state, task_output) execution = db_api.execution_get(task['execution_id']) self._create_next_tasks(task, workbook) # Determine what tasks need to be started. tasks = db_api.tasks_get(workbook_name=task['workbook_name'], execution_id=task['execution_id']) new_exec_state = self._determine_execution_state(execution, tasks) if execution['state'] != new_exec_state: wf_trace_msg = \ "Execution '%s' [%s -> %s]" % \ (execution['id'], execution['state'], new_exec_state) WORKFLOW_TRACE.info(wf_trace_msg) execution = \ db_api.execution_update(execution['id'], { "state": new_exec_state }) LOG.info("Changed execution state: %s" % execution) tasks_to_start, delayed_tasks = workflow.find_resolved_tasks(tasks) self._add_variables_to_data_flow_context(outbound_context, execution) data_flow.prepare_tasks(tasks_to_start, outbound_context) db_api.commit_tx() except Exception as e: msg = "Failed to create necessary DB objects: %s" % e LOG.exception(msg) raise exc.EngineException(msg) finally: db_api.end_tx() if states.is_stopped_or_finished(execution["state"]): return task for task in delayed_tasks: self._schedule_run(workbook, task, outbound_context) if tasks_to_start: self._run_tasks(tasks_to_start) return task