def test_rerun_sub_workflow(self): wf_service.create_workflows("""--- version: '2.0' wf1: tasks: task1: workflow: wf2 wf2: tasks: task2: workflow: wf3 wf3: tasks: task3: action: std.noop""") # Run workflow and fail task. wf1_ex = self.engine.start_workflow('wf1') self.await_workflow_error(wf1_ex.id) with db_api.transaction(): wf_exs = db_api.get_workflow_executions() task_exs = db_api.get_task_executions() self.assertEqual(3, len(wf_exs), 'The number of workflow executions') self.assertEqual(3, len(task_exs), 'The number of task executions') for wf_ex in wf_exs: self.assertEqual(states.ERROR, wf_ex.state, 'The executions must fail the first time') for task_ex in task_exs: self.assertEqual(states.ERROR, task_ex.state, 'The tasks must fail the first time') wf3_ex = self._assert_single_item(wf_exs, name='wf3') task3_ex = self._assert_single_item(wf3_ex.task_executions, name="task3") self.engine.rerun_workflow(task3_ex.id) self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf_exs = db_api.get_workflow_executions() task_exs = db_api.get_task_executions() self.assertEqual(3, len(wf_exs), 'The number of workflow executions') self.assertEqual(3, len(task_exs), 'The number of task executions') for wf_ex in wf_exs: self.assertEqual(states.SUCCESS, wf_ex.state, 'The executions must success the second time') for task_ex in task_exs: self.assertEqual(states.SUCCESS, task_ex.state, 'The tasks must success the second time')
def test_delete_join_completion_check_on_execution_delete(self): wf_text = """--- version: '2.0' wf: tasks: task1: action: std.noop on-success: join_task task2: description: Never ends action: std.async_noop on-success: join_task join_task: join: all """ wf_service.create_workflows(wf_text) wf_ex = self.engine.start_workflow('wf') tasks = db_api.get_task_executions(workflow_execution_id=wf_ex.id) self.assertGreaterEqual(len(tasks), 2) task1 = self._assert_single_item(tasks, name='task1') self.await_task_success(task1.id) # Once task1 is finished we know that join_task must be created. tasks = db_api.get_task_executions(workflow_execution_id=wf_ex.id) self._assert_single_item( tasks, name='join_task', state=states.WAITING ) calls = db_api.get_delayed_calls() mtd_name = 'mistral.engine.task_handler._refresh_task_state' cnt = sum([1 for c in calls if c.target_method_name == mtd_name]) # There can be 2 calls with different value of 'processing' flag. self.assertTrue(cnt == 1 or cnt == 2) # Stop the workflow. db_api.delete_workflow_execution(wf_ex.id) self._await( lambda: len(db_api.get_delayed_calls(target_method_name=mtd_name)) == 0 )
def test_delete_join_completion_check_on_stop(self): wf_text = """--- version: '2.0' wf: tasks: task1: action: std.noop on-success: join_task task2: description: Never ends action: std.async_noop on-success: join_task join_task: join: all """ wf_service.create_workflows(wf_text) wf_ex = self.engine.start_workflow('wf') tasks = db_api.get_task_executions(workflow_execution_id=wf_ex.id) self.assertGreaterEqual(len(tasks), 2) task1 = self._assert_single_item(tasks, name='task1') self.await_task_success(task1.id) # Once task1 is finished we know that join_task must be created. tasks = db_api.get_task_executions(workflow_execution_id=wf_ex.id) self._assert_single_item( tasks, name='join_task', state=states.WAITING ) # Stop the workflow. self.engine.stop_workflow(wf_ex.id, state=states.CANCELLED) mtd_name = 'mistral.engine.task_handler._refresh_task_state' self._await( lambda: len(db_api.get_delayed_calls(target_method_name=mtd_name)) == 0 )
def test_delete_join_completion_check_on_execution_delete(self): wf_text = """--- version: '2.0' wf: tasks: task1: action: std.noop on-success: join_task task2: description: Never ends action: std.async_noop on-success: join_task join_task: join: all """ wf_service.create_workflows(wf_text) wf_ex = self.engine.start_workflow('wf') tasks = db_api.get_task_executions(workflow_execution_id=wf_ex.id) self.assertGreaterEqual(len(tasks), 2) task1 = self._assert_single_item(tasks, name='task1') self.await_task_success(task1.id) # Once task1 is finished we know that join_task must be created. tasks = db_api.get_task_executions(workflow_execution_id=wf_ex.id) self._assert_single_item( tasks, name='join_task', state=states.WAITING ) # Stop the workflow. db_api.delete_workflow_execution(wf_ex.id) mtd_name = 'mistral.engine.task_handler._refresh_task_state' self._await( lambda: len(db_api.get_delayed_calls(target_method_name=mtd_name)) == 0 )
def test_cascade_delete_deep(self): wf_text = """ version: 2.0 wf: input: - level tasks: initial: action: std.noop on-success: - recurse: <% $.level > 0 %> recurse: workflow: wf input: level: <% $.level - 1 %> """ wf_service.create_workflows(wf_text) wf_ex = self.engine.start_workflow('wf', wf_input={"level": 7}) self.await_workflow_success(wf_ex.id) self.assertEqual(8, len(db_api.get_workflow_executions())) # Now delete the root workflow execution and make sure that # all dependent objects are deleted as well. db_api.delete_workflow_execution(wf_ex.id) self.assertEqual(0, len(db_api.get_workflow_executions())) self.assertEqual(0, len(db_api.get_task_executions())) self.assertEqual(0, len(db_api.get_action_executions()))
def find_task_executions_by_name(wf_ex_id, task_name): """Finds task executions by workflow execution id and task name. :param wf_ex_id: Workflow execution id. :param task_name: Task name. :return: Task executions (possibly a cached value). The returned list may contain task execution clones not bound to the DB session. """ with _TASK_EX_CACHE_LOCK: t_execs = _TASK_EX_CACHE[wf_ex_id].get(task_name) if t_execs: return t_execs t_execs = db_api.get_task_executions( workflow_execution_id=wf_ex_id, name=task_name, sort_keys=[] # disable sorting ) t_execs = [t_ex.get_clone() for t_ex in t_execs] # We can cache only finished tasks because they won't change. all_finished = (t_execs and all( [states.is_completed(t_ex.state) for t_ex in t_execs])) if all_finished: with _TASK_EX_CACHE_LOCK: _TASK_EX_CACHE[wf_ex_id][task_name] = t_execs return t_execs
def find_task_executions_by_name(wf_ex_id, task_name): """Finds task executions by workflow execution id and task name. :param wf_ex_id: Workflow execution id. :param task_name: Task name. :return: Task executions (possibly a cached value). """ cache_key = (wf_ex_id, task_name) with _TASK_EXECUTIONS_CACHE_LOCK: t_execs = _TASK_EXECUTIONS_CACHE.get(cache_key) if t_execs: return t_execs t_execs = db_api.get_task_executions( workflow_execution_id=wf_ex_id, name=task_name ) # We can cache only finished tasks because they won't change. all_finished = ( t_execs and all([states.is_completed(t_ex.state) for t_ex in t_execs]) ) if all_finished: with _TASK_EXECUTIONS_CACHE_LOCK: _TASK_EXECUTIONS_CACHE[cache_key] = t_execs return t_execs
def find_task_executions_by_name(wf_ex_id, task_name): """Finds task executions by workflow execution id and task name. :param wf_ex_id: Workflow execution id. :param task_name: Task name. :return: Task executions (possibly a cached value). """ with _CACHE_LOCK: t_execs = _TASK_EX_CACHE[wf_ex_id].get(task_name) if t_execs: return t_execs t_execs = db_api.get_task_executions(workflow_execution_id=wf_ex_id, name=task_name) # We can cache only finished tasks because they won't change. all_finished = (t_execs and all( [states.is_completed(t_ex.state) for t_ex in t_execs])) if all_finished: with _CACHE_LOCK: _TASK_EX_CACHE[wf_ex_id][task_name] = t_execs return t_execs
def task_(context, task_name=None): # This section may not exist in a context if it's calculated not in # task scope. cur_task = context['__task_execution'] # 1. If task_name is empty it's 'task()' use case, we need to get the # current task. # 2. if task_name is not empty but it's equal to the current task name # we need to take exactly the current instance of this task. Otherwise # there may be ambiguity if there are many tasks with this name. # 3. In other case we just find a task in DB by the given name. if cur_task and (not task_name or cur_task['name'] == task_name): task_ex = db_api.get_task_execution(cur_task['id']) else: task_execs = db_api.get_task_executions( workflow_execution_id=context['__execution']['id'], name=task_name ) # TODO(rakhmerov): Account for multiple executions (i.e. in case of # cycles). task_ex = task_execs[-1] if len(task_execs) > 0 else None if not task_ex: LOG.warning( "Task '%s' not found by the task() expression function", task_name ) return None # We don't use to_dict() db model method because not all fields # make sense for user. return _convert_to_user_model(task_ex)
def task_(context, task_name=None): # This section may not exist in a context if it's calculated not in # task scope. cur_task = context['__task_execution'] # 1. If task_name is empty it's 'task()' use case, we need to get the # current task. # 2. if task_name is not empty but it's equal to the current task name # we need to take exactly the current instance of this task. Otherwise # there may be ambiguity if there are many tasks with this name. # 3. In other case we just find a task in DB by the given name. if cur_task and (not task_name or cur_task['name'] == task_name): task_ex = db_api.get_task_execution(cur_task['id']) else: task_execs = db_api.get_task_executions( workflow_execution_id=context['__execution']['id'], name=task_name ) # TODO(rakhmerov): Account for multiple executions (i.e. in case of # cycles). task_ex = task_execs[-1] if len(task_execs) > 0 else None if not task_ex: return None # We don't use to_dict() db model method because not all fields # make sense for user. return _convert_to_user_model(task_ex)
def test_invalid_workflow_input(self): # Check that in case of invalid input workflow objects aren't even # created. wf_text = """ version: '2.0' wf: input: - param1 - param2 tasks: task1: action: std.noop """ wf_service.create_workflows(wf_text) self.assertRaises( exc.InputException, self.engine.start_workflow, 'wf', '', {'wrong_param': 'some_value'} ) self.assertEqual(0, len(db_api.get_workflow_executions())) self.assertEqual(0, len(db_api.get_task_executions())) self.assertEqual(0, len(db_api.get_action_executions()))
def task_(context, task_name): # Importing data_flow in order to break cycle dependency between modules. from mistral.workflow import data_flow # This section may not exist in a context if it's calculated not in # task scope. cur_task = context['__task_execution'] if cur_task and cur_task['name'] == task_name: task_ex = db_api.get_task_execution(cur_task['id']) else: task_execs = db_api.get_task_executions( workflow_execution_id=context['__execution']['id'], name=task_name) # TODO(rakhmerov): Account for multiple executions (i.e. in case of # cycles). task_ex = task_execs[-1] if len(task_execs) > 0 else None if not task_ex: return None # We don't use to_dict() db model method because not all fields # make sense for user. return { 'id': task_ex.id, 'name': task_ex.name, 'spec': task_ex.spec, 'state': task_ex.state, 'state_info': task_ex.state_info, 'result': data_flow.get_task_execution_result(task_ex), 'published': task_ex.published }
def test_start_task1(self): wf_input = {'param1': 'a', 'param2': 'b'} wf_ex = self.engine.start_workflow('my_wb.wf1', wf_input=wf_input, task_name='task1') # Execution 1. self.assertIsNotNone(wf_ex) self.assertDictEqual(wf_input, wf_ex.input) self.assertDictEqual({ 'task_name': 'task1', 'namespace': '', 'env': {} }, wf_ex.params) # Wait till workflow 'wf1' is completed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(1, len(task_execs)) self.assertEqual(1, len(db_api.get_task_executions())) task_ex = self._assert_single_item(task_execs, name='task1', state=states.SUCCESS) self.assertDictEqual({'result1': 'a'}, task_ex.published)
def test_start_task2(self): wf_input = {'param1': 'a', 'param2': 'b'} wf_ex = self.engine.start_workflow('my_wb.wf1', wf_input, task_name='task2') # Execution 1. self.assertIsNotNone(wf_ex) self.assertDictEqual(wf_input, wf_ex.input) self.assertDictEqual({'task_name': 'task2'}, wf_ex.params) # Wait till workflow 'wf1' is completed. self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(2, len(wf_ex.task_executions)) self.assertEqual(2, len(db_api.get_task_executions())) task1_ex = self._assert_single_item(wf_ex.task_executions, name='task1', state=states.SUCCESS) self.assertDictEqual({'result1': 'a'}, task1_ex.published) task2_ex = self._assert_single_item(wf_ex.task_executions, name='task2', state=states.SUCCESS) self.assertDictEqual({'result2': 'a & b'}, task2_ex.published)
def _build_fail_info_message(wf_ctrl, wf_ex): # Try to find where error is exactly. failed_tasks = [ t_ex for t_ex in db_api.get_task_executions(workflow_execution_id=wf_ex.id, state=states.ERROR, sort_keys=['name']) if not wf_ctrl.is_error_handled_for(t_ex) ] msg = ('Failure caused by error in tasks: %s\n' % ', '.join([t.name for t in failed_tasks])) for t in failed_tasks: msg += '\n %s [task_ex_id=%s] -> %s\n' % (t.name, t.id, t.state_info) for i, ex in enumerate(t.action_executions): if ex.state == states.ERROR: output = (ex.output or dict()).get('result', 'Unknown') msg += (' [action_ex_id=%s, idx=%s]: %s\n' % (ex.id, i, str(output))) for i, ex in enumerate(t.workflow_executions): if ex.state == states.ERROR: output = (ex.output or dict()).get('result', 'Unknown') msg += (' [wf_ex_id=%s, idx=%s]: %s\n' % (ex.id, i, str(output))) return msg
def test_start_task1(self): wf_input = {'param1': 'a', 'param2': 'b'} wf_ex = self.engine.start_workflow( 'my_wb.wf1', wf_input, task_name='task1' ) # Execution 1. self.assertIsNotNone(wf_ex) self.assertDictEqual(wf_input, wf_ex.input) self.assertDictEqual({'task_name': 'task1'}, wf_ex.params) # Wait till workflow 'wf1' is completed. self._await(lambda: self.is_execution_success(wf_ex.id)) wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual(1, len(wf_ex.task_executions)) self.assertEqual(1, len(db_api.get_task_executions())) task_ex = self._assert_single_item( wf_ex.task_executions, name='task1', state=states.SUCCESS ) self.assertDictEqual({'result1': 'a'}, task_ex.published)
def check_and_fix_integrity(wf_ex): check_after_seconds = CONF.engine.execution_integrity_check_delay if check_after_seconds < 0: # Never check integrity if it's a negative value. return # To break cyclic dependency. from mistral.engine import task_handler running_task_execs = db_api.get_task_executions( workflow_execution_id=wf_ex.id, state=states.RUNNING ) for t_ex in running_task_execs: # The idea is that we take the latest known timestamp of the task # execution and consider it eligible for checking and fixing only # if some minimum period of time elapsed since the last update. timestamp = t_ex.updated_at or t_ex.created_at delta = timeutils.delta_seconds(timestamp, timeutils.utcnow()) if delta < check_after_seconds: continue child_executions = t_ex.executions if not child_executions: continue all_finished = all( [states.is_completed(c_ex.state) for c_ex in child_executions] ) if all_finished: # Find the timestamp of the most recently finished child. most_recent_child_timestamp = max( [c_ex.updated_at or c_ex.created_at for c_ex in child_executions] ) interval = timeutils.delta_seconds( most_recent_child_timestamp, timeutils.utcnow() ) if interval > check_after_seconds: # We found a task execution in RUNNING state for which all # child executions are finished. We need to call # "schedule_on_action_complete" on the task handler for any of # the child executions so that the task state is calculated and # updated properly. LOG.warning( "Found a task execution that is likely stuck in RUNNING" " state because all child executions are finished," " will try to recover [task_execution=%s]", t_ex.id ) task_handler.schedule_on_action_complete(child_executions[-1])
def _get_task_resources_with_results(wf_ex_id=None): filters = {} if wf_ex_id: filters['workflow_execution_id'] = wf_ex_id task_exs = db_api.get_task_executions(**filters) tasks = [_get_task_resource_with_result(t_e) for t_e in task_exs] return Tasks(tasks=tasks)
def _build_cancel_info_message(wf_ctrl, wf_ex): # Try to find where cancel is exactly. cancelled_tasks = [ t_ex for t_ex in db_api.get_task_executions(workflow_execution_id=wf_ex.id, state=states.CANCELLED, sort_keys=['name']) ] return ('Cancelled tasks: %s' % ', '.join([t.name for t in cancelled_tasks]))
def test_cascade_delete(self): wf_text = """ version: 2.0 wf: tasks: task1: workflow: sub_wf1 task2: workflow: sub_wf2 sub_wf1: tasks: task1: action: std.noop sub_wf2: tasks: task1: action: std.noop """ wf_service.create_workflows(wf_text) wf_ex = self.engine.start_workflow('wf') self.await_workflow_success(wf_ex.id) self.assertEqual(3, len(db_api.get_workflow_executions())) self.assertEqual(4, len(db_api.get_task_executions())) self.assertEqual(2, len(db_api.get_action_executions())) # Now delete the root workflow execution and make sure that # all dependent objects are deleted as well. db_api.delete_workflow_execution(wf_ex.id) self.assertEqual(0, len(db_api.get_workflow_executions())) self.assertEqual(0, len(db_api.get_task_executions())) self.assertEqual(0, len(db_api.get_action_executions()))
def defer(self): """Defers task. This method puts task to a waiting state. """ # NOTE(rakhmerov): using named locks may cause problems under load # with MySQL that raises a lot of deadlocks in case of high # parallelism so it makes sense to do a fast check if the object # already exists in DB outside of the lock. if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key, state=states.WAITING ) self.task_ex = t_execs[0] if t_execs else None if self.task_ex: return with db_api.named_lock(self.unique_key): if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key ) self.task_ex = t_execs[0] if t_execs else None msg = 'Task is waiting.' if not self.task_ex: self._create_task_execution( state=states.WAITING, state_info=msg ) elif self.task_ex.state != states.WAITING: self.set_state(states.WAITING, msg)
def _get_tasks_from_db(workflow_execution_id=None, recursive=False, state=None, flat=False): task_execs = [] nested_task_exs = [] kwargs = {} if workflow_execution_id: kwargs['workflow_execution_id'] = workflow_execution_id # We can't add state to query if we want to filter by workflow_execution_id # recursively. There might be a workflow_execution in one state with a # nested workflow execution that has a task in the desired state until we # have an optimization for queering all workflow executions under a given # top level workflow execution, this is the way to go. if state and not (workflow_execution_id and recursive): kwargs['state'] = state task_execs.extend(db_api.get_task_executions(**kwargs)) # If it is not recursive no need to check nested workflows. # If there is no workflow execution id, we already have all we need, and # doing more queries will just create duplication in the results. if recursive and workflow_execution_id: for t in task_execs: if t.type == utils.WORKFLOW_TASK_TYPE: # Get nested workflow execution that matches the task. nested_workflow_executions = db_api.get_workflow_executions( task_execution_id=t.id ) # There might be zero nested executions. for nested_workflow_execution in nested_workflow_executions: nested_task_exs.extend( _get_tasks_from_db( nested_workflow_execution.id, recursive, state, flat ) ) if state or flat: # Filter by state and flat. task_execs = [ t for t in task_execs if _should_pass_filter(t, state, flat) ] # The nested tasks were already filtered, since this is a recursion. task_execs.extend(nested_task_exs) return task_execs
def _build_cancel_info_message(wf_ctrl, wf_ex): # Try to find where cancel is exactly. cancelled_tasks = [ t_ex for t_ex in db_api.get_task_executions( workflow_execution_id=wf_ex.id, state=states.CANCELLED, sort_keys=['name'] ) ] return ( 'Cancelled tasks: %s' % ', '.join([t.name for t in cancelled_tasks]) )
def test_start_task2(self): wf_input = { 'param1': 'a', 'param2': 'b' } wf_ex = self.engine.start_workflow( 'my_wb.wf1', wf_input=wf_input, task_name='task2' ) # Execution 1. self.assertIsNotNone(wf_ex) self.assertDictEqual(wf_input, wf_ex.input) self.assertDictEqual( { 'task_name': 'task2', 'namespace': '', 'env': {} }, wf_ex.params ) # Wait till workflow 'wf1' is completed. self.await_workflow_success(wf_ex.id) with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) task_execs = wf_ex.task_executions self.assertEqual(2, len(task_execs)) self.assertEqual(2, len(db_api.get_task_executions())) task1_ex = self._assert_single_item( task_execs, name='task1', state=states.SUCCESS ) self.assertDictEqual({'result1': 'a'}, task1_ex.published) task2_ex = self._assert_single_item( task_execs, name='task2', state=states.SUCCESS ) self.assertDictEqual({'result2': 'a & b'}, task2_ex.published)
def test_one_line_requires_syntax(self): wf_input = {'param1': 'a', 'param2': 'b'} wf_ex = self.engine.start_workflow('my_wb.wf1', wf_input=wf_input, task_name='task4') self.await_workflow_success(wf_ex.id) tasks = db_api.get_task_executions() self.assertEqual(2, len(tasks)) self._assert_single_item(tasks, name='task4', state=states.SUCCESS) self._assert_single_item(tasks, name='task3', state=states.SUCCESS)
def test_one_line_requires_syntax(self): wf_input = {'param1': 'a', 'param2': 'b'} wf_ex = self.engine.start_workflow( 'my_wb.wf1', wf_input, task_name='task4' ) self._await(lambda: self.is_execution_success(wf_ex.id)) tasks = db_api.get_task_executions() self.assertEqual(2, len(tasks)) self._assert_single_item(tasks, name='task4', state=states.SUCCESS) self._assert_single_item(tasks, name='task3', state=states.SUCCESS)
def _get_task_resources_with_results(wf_ex_id=None): filters = {} if wf_ex_id: filters['workflow_execution_id'] = wf_ex_id tasks = [] task_execs = db_api.get_task_executions(**filters) for task_ex in task_execs: task = Task.from_dict(task_ex.to_dict()) task.result = json.dumps( data_flow.get_task_execution_result(task_ex) ) tasks += [task] return Tasks(tasks=tasks)
def _get_task_resources_with_results(wf_ex_id=None): filters = {} if wf_ex_id: filters['workflow_execution_id'] = wf_ex_id tasks = [] with db_api.transaction(): task_execs = db_api.get_task_executions(**filters) for task_ex in task_execs: task = Task.from_dict(task_ex.to_dict()) task.result = json.dumps( data_flow.get_task_execution_result(task_ex) ) tasks += [task] return Tasks(tasks=tasks)
def defer(self): """Defers task. This method puts task to a waiting state. """ if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, name=self.task_spec.get_name()) self.task_ex = t_execs[0] if t_execs else None if not self.task_ex: self._create_task_execution() if self.task_ex: self.set_state(states.WAITING, 'Task is deferred.') self.waiting = True
def defer(self): """Defers task. This method puts task to a waiting state. """ with db_api.named_lock(self.unique_key): if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key) self.task_ex = t_execs[0] if t_execs else None msg = 'Task is waiting.' if not self.task_ex: self._create_task_execution(state=states.WAITING, state_info=msg) elif self.task_ex.state != states.WAITING: self.set_state(states.WAITING, msg)
def _build_fail_info_message(wf_ctrl, wf_ex): # Try to find where error is exactly. failed_tasks = [ t_ex for t_ex in db_api.get_task_executions( workflow_execution_id=wf_ex.id, state=states.ERROR, sort_keys=['name'] ) if not wf_ctrl.is_error_handled_for(t_ex) ] msg = ('Failure caused by error in tasks: %s\n' % ', '.join([t.name for t in failed_tasks])) for t in failed_tasks: msg += '\n %s [task_ex_id=%s] -> %s\n' % (t.name, t.id, t.state_info) for i, ex in enumerate(t.action_executions): if ex.state == states.ERROR: output = (ex.output or dict()).get('result', 'Unknown') msg += ( ' [action_ex_id=%s, idx=%s]: %s\n' % ( ex.id, i, str(output) ) ) for i, ex in enumerate(t.workflow_executions): if ex.state == states.ERROR: output = (ex.output or dict()).get('result', 'Unknown') msg += ( ' [wf_ex_id=%s, idx=%s]: %s\n' % ( ex.id, i, str(output) ) ) return msg
def defer(self): """Defers task. This method puts task to a waiting state. """ with db_api.named_lock(self.unique_key): if not self.task_ex: t_execs = db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, unique_key=self.unique_key ) self.task_ex = t_execs[0] if t_execs else None msg = 'Task is waiting.' if not self.task_ex: self._create_task_execution( state=states.WAITING, state_info=msg ) elif self.task_ex.state != states.WAITING: self.set_state(states.WAITING, msg)
def task_(context, task_name): # This section may not exist in a context if it's calculated not in # task scope. cur_task = context['__task_execution'] if cur_task and cur_task['name'] == task_name: task_ex = db_api.get_task_execution(cur_task['id']) else: task_execs = db_api.get_task_executions( workflow_execution_id=context['__execution']['id'], name=task_name ) # TODO(rakhmerov): Account for multiple executions (i.e. in case of # cycles). task_ex = task_execs[-1] if len(task_execs) > 0 else None if not task_ex: return None # We don't use to_dict() db model method because not all fields # make sense for user. return _convert_to_user_model(task_ex)
def _test_subworkflow(self, env): wf2_ex = self.engine.start_workflow('my_wb.wf2', env=env) # Execution of 'wf2'. self.assertIsNotNone(wf2_ex) self.assertDictEqual({}, wf2_ex.input) self._await(lambda: len(db_api.get_workflow_executions()) == 2, 0.5, 5) wf_execs = db_api.get_workflow_executions() self.assertEqual(2, len(wf_execs)) # Execution of 'wf1'. wf2_ex = self._assert_single_item(wf_execs, name='my_wb.wf2') wf1_ex = self._assert_single_item(wf_execs, name='my_wb.wf1') expected_wf1_input = { 'param1': 'Bonnie', 'param2': 'Clyde' } self.assertIsNotNone(wf1_ex.task_execution_id) self.assertDictEqual(wf1_ex.input, expected_wf1_input) # Wait till workflow 'wf1' is completed. self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) self.assertDictEqual( {'final_result': "'Bonnie & Clyde'"}, wf1_ex.output ) # Wait till workflow 'wf2' is completed. self.await_workflow_success(wf2_ex.id) with db_api.transaction(): wf2_ex = db_api.get_workflow_execution(wf2_ex.id) self.assertDictEqual( {'slogan': "'Bonnie & Clyde' is a cool movie!\n"}, wf2_ex.output ) with db_api.transaction(): # Check if target is resolved. wf1_task_execs = db_api.get_task_executions( workflow_execution_id=wf1_ex.id ) self._assert_single_item(wf1_task_execs, name='task1') self._assert_single_item(wf1_task_execs, name='task2') for t_ex in wf1_task_execs: a_ex = t_ex.action_executions[0] callback_url = '/v2/action_executions/%s' % a_ex.id r_exe.RemoteExecutor.run_action.assert_any_call( a_ex.id, 'mistral.actions.std_actions.EchoAction', {}, a_ex.input, False, { 'task_execution_id': t_ex.id, 'callback_url': callback_url, 'workflow_execution_id': wf1_ex.id, 'workflow_name': wf1_ex.name, 'action_execution_id': a_ex.id, }, target=TARGET, timeout=None )
def test_notify_rerun_nested_workflow(self): wf_def = """ wf_1: tasks: wf_1_t1: workflow: wf_2 on-success: - wf_1_t2 wf_1_t2: action: std.noop version: '2.0' wf_2: tasks: wf_2_t1: action: std.noop on-success: - wf_2_t2 wf_2_t2: action: std.noop """ wf_svc.create_workflows(wf_def) notify_options = [{'type': 'webhook'}] params = {'notify': notify_options} wf_1_ex = self.engine.start_workflow('wf_1', '', **params) self.await_workflow_error(wf_1_ex.id) with db_api.transaction(): wf_exs = db_api.get_workflow_executions() self._assert_single_item(wf_exs, name='wf_1', state=states.ERROR) self._assert_single_item(wf_exs, name='wf_2', state=states.ERROR) task_exs = db_api.get_task_executions() self._assert_single_item(task_exs, name='wf_1_t1', state=states.ERROR) wf_2_t1 = self._assert_single_item(task_exs, name='wf_2_t1', state=states.ERROR) self.assertEqual(2, len(task_exs)) self.assertEqual(2, len(wf_exs)) self.engine.rerun_workflow(wf_2_t1.id) self.await_workflow_success(wf_1_ex.id) with db_api.transaction(): wf_exs = db_api.get_workflow_executions() wf_1_ex = self._assert_single_item(wf_exs, name='wf_1', state=states.SUCCESS) wf_2_ex = self._assert_single_item(wf_exs, name='wf_2', state=states.SUCCESS) task_wf_1_exs = wf_1_ex.task_executions wf_1_t1 = self._assert_single_item(task_wf_1_exs, name='wf_1_t1', state=states.SUCCESS) wf_1_t2 = self._assert_single_item(task_wf_1_exs, name='wf_1_t2', state=states.SUCCESS) task_wf_2_exs = wf_2_ex.task_executions wf_2_t1 = self._assert_single_item(task_wf_2_exs, name='wf_2_t1', state=states.SUCCESS) wf_2_t2 = self._assert_single_item(task_wf_2_exs, name='wf_2_t2', state=states.SUCCESS) self.assertEqual(2, len(task_wf_1_exs)) self.assertEqual(2, len(task_wf_2_exs)) self.assertEqual(2, len(wf_exs)) expected_order = [ (wf_1_ex.id, events.WORKFLOW_LAUNCHED), (wf_1_t1.id, events.TASK_LAUNCHED), (wf_2_ex.id, events.WORKFLOW_LAUNCHED), (wf_2_t1.id, events.TASK_LAUNCHED), (wf_2_t1.id, events.TASK_FAILED), (wf_2_ex.id, events.WORKFLOW_FAILED), (wf_1_t1.id, events.TASK_FAILED), (wf_1_ex.id, events.WORKFLOW_FAILED), # rerun (wf_2_ex.id, events.WORKFLOW_RERUN), (wf_1_ex.id, events.WORKFLOW_RERUN), (wf_1_t1.id, events.TASK_RERUN), (wf_2_t1.id, events.TASK_RERUN), (wf_2_t1.id, events.TASK_SUCCEEDED), (wf_2_t2.id, events.TASK_LAUNCHED), (wf_2_t2.id, events.TASK_SUCCEEDED), (wf_2_ex.id, events.WORKFLOW_SUCCEEDED), (wf_1_t1.id, events.TASK_SUCCEEDED), (wf_1_t2.id, events.TASK_LAUNCHED), (wf_1_t2.id, events.TASK_SUCCEEDED), (wf_1_ex.id, events.WORKFLOW_SUCCEEDED), ] self.assertTrue(self.publishers['wbhk'].publish.called) self.assertListEqual(expected_order, EVENT_LOGS)
def test_subworkflow_success(self): wf2_ex = self.engine.start_workflow('wb1.wf2', None) project_id = auth_context.ctx().project_id # Execution of 'wf2'. self.assertEqual(project_id, wf2_ex.project_id) self.assertIsNotNone(wf2_ex) self.assertDictEqual({}, wf2_ex.input) self.assertDictEqual({}, wf2_ex.params) self._await(lambda: len(db_api.get_workflow_executions()) == 2, 0.5, 5) wf_execs = db_api.get_workflow_executions() self.assertEqual(2, len(wf_execs)) # Execution of 'wf2'. wf1_ex = self._assert_single_item(wf_execs, name='wb1.wf1') wf2_ex = self._assert_single_item(wf_execs, name='wb1.wf2') self.assertEqual(project_id, wf1_ex.project_id) self.assertIsNotNone(wf1_ex.task_execution_id) self.assertDictContainsSubset( { 'task_name': 'task2', 'task_execution_id': wf1_ex.task_execution_id }, wf1_ex.params) self.assertDictEqual({ 'param1': 'Bonnie', 'param2': 'Clyde' }, wf1_ex.input) # Wait till workflow 'wf1' is completed. self.await_workflow_success(wf1_ex.id) wf1_ex = db_api.get_workflow_execution(wf1_ex.id) self.assertDictEqual({'final_result': "'Bonnie & Clyde'"}, wf1_ex.output) # Wait till workflow 'wf2' is completed. self.await_workflow_success(wf2_ex.id, timeout=4) wf2_ex = db_api.get_workflow_execution(wf2_ex.id) self.assertDictEqual({'slogan': "'Bonnie & Clyde' is a cool movie!"}, wf2_ex.output) # Check project_id in tasks. wf1_task_execs = db_api.get_task_executions( workflow_execution_id=wf1_ex.id) wf2_task_execs = db_api.get_task_executions( workflow_execution_id=wf2_ex.id) wf2_task1_ex = self._assert_single_item(wf1_task_execs, name='task1') wf1_task1_ex = self._assert_single_item(wf2_task_execs, name='task1') wf1_task2_ex = self._assert_single_item(wf1_task_execs, name='task2') self.assertEqual(project_id, wf2_task1_ex.project_id) self.assertEqual(project_id, wf1_task1_ex.project_id) self.assertEqual(project_id, wf1_task2_ex.project_id)
def _get_task_executions(self, **kwargs): return db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, sort_keys=[], # disable sorting **kwargs )
def _get_task_executions(self, **kwargs): return db_api.get_task_executions( workflow_execution_id=self.wf_ex.id, sort_keys=[], # disable sorting **kwargs)
def _test_subworkflow(self, env): wf2_ex = self.engine.start_workflow('my_wb.wf2', {}, env=env) # Execution of 'wf2'. self.assertIsNotNone(wf2_ex) self.assertDictEqual({}, wf2_ex.input) self.assertDictContainsSubset({'env': env}, wf2_ex.params) self._await(lambda: len(db_api.get_workflow_executions()) == 2, 0.5, 5) wf_execs = db_api.get_workflow_executions() self.assertEqual(2, len(wf_execs)) # Execution of 'wf1'. wf2_ex = self._assert_single_item(wf_execs, name='my_wb.wf2') wf1_ex = self._assert_single_item(wf_execs, name='my_wb.wf1') expected_start_params = { 'task_name': 'task2', 'task_execution_id': wf1_ex.task_execution_id, 'env': env } expected_wf1_input = {'param1': 'Bonnie', 'param2': 'Clyde'} self.assertIsNotNone(wf1_ex.task_execution_id) self.assertDictContainsSubset(expected_start_params, wf1_ex.params) self.assertDictEqual(wf1_ex.input, expected_wf1_input) # Wait till workflow 'wf1' is completed. self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) self.assertDictEqual({'final_result': "'Bonnie & Clyde'"}, wf1_ex.output) # Wait till workflow 'wf2' is completed. self.await_workflow_success(wf2_ex.id) with db_api.transaction(): wf2_ex = db_api.get_workflow_execution(wf2_ex.id) self.assertDictEqual( {'slogan': "'Bonnie & Clyde' is a cool movie!\n"}, wf2_ex.output) with db_api.transaction(): # Check if target is resolved. wf1_task_execs = db_api.get_task_executions( workflow_execution_id=wf1_ex.id) self._assert_single_item(wf1_task_execs, name='task1') self._assert_single_item(wf1_task_execs, name='task2') for t_ex in wf1_task_execs: a_ex = t_ex.action_executions[0] rpc.ExecutorClient.run_action.assert_any_call( a_ex.id, 'mistral.actions.std_actions.EchoAction', {}, a_ex.input, TARGET, safe_rerun=False)
def _num_of_tasks(): return len( db_api.get_task_executions(workflow_execution_id=wf_ex.id) )
def find_task_executions_with_state(wf_ex_id, state): return db_api.get_task_executions( workflow_execution_id=wf_ex_id, state=state )
def find_task_executions_with_state(wf_ex_id, state): return db_api.get_task_executions(workflow_execution_id=wf_ex_id, state=state)
def test_subworkflow_success(self): wf2_ex = self.engine.start_workflow('wb1.wf2') project_id = auth_context.ctx().project_id # Execution of 'wf2'. self.assertEqual(project_id, wf2_ex.project_id) self.assertIsNotNone(wf2_ex) self.assertDictEqual({}, wf2_ex.input) self.assertDictEqual({'namespace': '', 'env': {}}, wf2_ex.params) self._await(lambda: len(db_api.get_workflow_executions()) == 2, 0.5, 5) wf_execs = db_api.get_workflow_executions() self.assertEqual(2, len(wf_execs)) # Execution of 'wf2'. wf1_ex = self._assert_single_item(wf_execs, name='wb1.wf1') wf2_ex = self._assert_single_item(wf_execs, name='wb1.wf2') self.assertEqual(project_id, wf1_ex.project_id) self.assertIsNotNone(wf1_ex.task_execution_id) self.assertDictContainsSubset( { 'task_name': 'task2', 'task_execution_id': wf1_ex.task_execution_id }, wf1_ex.params ) self.assertDictEqual( { 'param1': 'Bonnie', 'param2': 'Clyde' }, wf1_ex.input ) # Wait till workflow 'wf1' is completed. self.await_workflow_success(wf1_ex.id) with db_api.transaction(): wf1_ex = db_api.get_workflow_execution(wf1_ex.id) wf1_output = wf1_ex.output self.assertDictEqual( {'final_result': "'Bonnie & Clyde'"}, wf1_output ) # Wait till workflow 'wf2' is completed. self.await_workflow_success(wf2_ex.id, timeout=4) with db_api.transaction(): wf2_ex = db_api.get_workflow_execution(wf2_ex.id) wf2_output = wf2_ex.output self.assertDictEqual( {'slogan': "'Bonnie & Clyde' is a cool movie!"}, wf2_output ) # Check project_id in tasks. wf1_task_execs = db_api.get_task_executions( workflow_execution_id=wf1_ex.id ) wf2_task_execs = db_api.get_task_executions( workflow_execution_id=wf2_ex.id ) wf2_task1_ex = self._assert_single_item(wf1_task_execs, name='task1') wf1_task1_ex = self._assert_single_item(wf2_task_execs, name='task1') wf1_task2_ex = self._assert_single_item(wf1_task_execs, name='task2') self.assertEqual(project_id, wf2_task1_ex.project_id) self.assertEqual(project_id, wf1_task1_ex.project_id) self.assertEqual(project_id, wf1_task2_ex.project_id)
def _check_and_fix_integrity(wf_ex_id): check_after_seconds = CONF.engine.execution_integrity_check_delay if check_after_seconds < 0: # Never check integrity if it's a negative value. return # To break cyclic dependency. from mistral.engine import task_handler with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex_id) if states.is_completed(wf_ex.state): return _schedule_check_and_fix_integrity(wf_ex, delay=120) running_task_execs = db_api.get_task_executions( workflow_execution_id=wf_ex.id, state=states.RUNNING, limit=CONF.engine.execution_integrity_check_batch_size ) for t_ex in running_task_execs: # The idea is that we take the latest known timestamp of the task # execution and consider it eligible for checking and fixing only # if some minimum period of time elapsed since the last update. timestamp = t_ex.updated_at or t_ex.created_at delta = timeutils.delta_seconds(timestamp, timeutils.utcnow()) if delta < check_after_seconds: continue child_executions = t_ex.executions if not child_executions: continue all_finished = all( [states.is_completed(c_ex.state) for c_ex in child_executions] ) if all_finished: # Find the timestamp of the most recently finished child. most_recent_child_timestamp = max( [c_ex.updated_at or c_ex.created_at for c_ex in child_executions] ) interval = timeutils.delta_seconds( most_recent_child_timestamp, timeutils.utcnow() ) if interval > check_after_seconds: # We found a task execution in RUNNING state for which all # child executions are finished. We need to call # "schedule_on_action_complete" on the task handler for # any of the child executions so that the task state is # calculated and updated properly. LOG.warning( "Found a task execution that is likely stuck in" " RUNNING state because all child executions are" " finished, will try to recover [task_execution=%s]", t_ex.id ) task_handler.schedule_on_action_complete( child_executions[-1] )
def _test_subworkflow(self, env): wf2_ex = self.engine.start_workflow( 'my_wb.wf2', {}, env=env ) # Execution of 'wf2'. self.assertIsNotNone(wf2_ex) self.assertDictEqual({}, wf2_ex.input) self.assertDictContainsSubset({'env': env}, wf2_ex.params) self._await(lambda: len(db_api.get_workflow_executions()) == 2, 0.5, 5) wf_execs = db_api.get_workflow_executions() self.assertEqual(2, len(wf_execs)) # Execution of 'wf1'. wf2_ex = self._assert_single_item(wf_execs, name='my_wb.wf2') wf1_ex = self._assert_single_item(wf_execs, name='my_wb.wf1') expected_start_params = { 'task_name': 'task2', 'task_execution_id': wf1_ex.task_execution_id, 'env': env } expected_wf1_input = { 'param1': 'Bonnie', 'param2': 'Clyde' } self.assertIsNotNone(wf1_ex.task_execution_id) self.assertDictContainsSubset(expected_start_params, wf1_ex.params) self.assertDictEqual(wf1_ex.input, expected_wf1_input) # Wait till workflow 'wf1' is completed. self.await_execution_success(wf1_ex.id) wf1_ex = db_api.get_workflow_execution(wf1_ex.id) expected_wf1_output = {'final_result': "'Bonnie & Clyde'"} self.assertDictEqual(wf1_ex.output, expected_wf1_output) # Wait till workflow 'wf2' is completed. self.await_execution_success(wf2_ex.id) wf2_ex = db_api.get_workflow_execution(wf2_ex.id) expected_wf2_output = {'slogan': "'Bonnie & Clyde' is a cool movie!\n"} self.assertDictEqual(wf2_ex.output, expected_wf2_output) # Check if target is resolved. wf1_task_execs = db_api.get_task_executions( workflow_execution_id=wf1_ex.id ) self._assert_single_item(wf1_task_execs, name='task1') self._assert_single_item(wf1_task_execs, name='task2') for t_ex in wf1_task_execs: a_ex = t_ex.executions[0] rpc.ExecutorClient.run_action.assert_any_call( a_ex.id, 'mistral.actions.std_actions.EchoAction', {}, a_ex.input, TARGET )