def get_controller(wf_ex, wf_spec=None): """Gets a workflow controller instance by given workflow execution object. :param wf_ex: Workflow execution object. :param wf_spec: Workflow specification object. If passed, the method works faster. :returns: Workflow controller class. """ if not wf_spec: wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id) wf_type = wf_spec.get_type() ctrl_cls = None for cls in u.iter_subclasses(WorkflowController): if cls.__workflow_type__ == wf_type: ctrl_cls = cls break if not ctrl_cls: raise exc.MistralError( 'Failed to find a workflow controller [type=%s]' % wf_type ) return ctrl_cls(wf_ex, wf_spec)
def _build_action(action_ex): if isinstance(action_ex, models.WorkflowExecution): return actions.WorkflowAction(None, action_ex=action_ex) wf_name = None wf_spec_name = None if action_ex.workflow_name: wf_name = action_ex.workflow_name wf_spec = spec_parser.get_workflow_spec_by_execution_id( action_ex.task_execution.workflow_execution_id) wf_spec_name = wf_spec.get_name() adhoc_action_name = action_ex.runtime_context.get('adhoc_action_name') if adhoc_action_name: action_def = actions.resolve_action_definition(adhoc_action_name, wf_name, wf_spec_name) return actions.AdHocAction(action_def, action_ex=action_ex) action_def = actions.resolve_action_definition(action_ex.name, wf_name, wf_spec_name) return actions.PythonAction(action_def, action_ex=action_ex)
def schedule(self, input_dict, target, index=0, desc='', safe_rerun=False): assert not self.action_ex parent_wf_ex = self.task_ex.workflow_execution parent_wf_spec = spec_parser.get_workflow_spec_by_execution_id( parent_wf_ex.id) task_spec = spec_parser.get_task_spec(self.task_ex.spec) wf_spec_name = task_spec.get_workflow_name() wf_def = e_utils.resolve_workflow_definition( parent_wf_ex.workflow_name, parent_wf_spec.get_name(), wf_spec_name) wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at) wf_params = {'task_execution_id': self.task_ex.id, 'index': index} if 'env' in parent_wf_ex.params: wf_params['env'] = parent_wf_ex.params['env'] for k, v in list(input_dict.items()): if k not in wf_spec.get_input(): wf_params[k] = v del input_dict[k] wf_handler.start_workflow(wf_def.id, input_dict, "sub-workflow execution", wf_params)
def _build_action(action_ex): if isinstance(action_ex, models.WorkflowExecution): return actions.WorkflowAction(None, action_ex=action_ex) wf_name = None wf_spec_name = None if action_ex.workflow_name: wf_name = action_ex.workflow_name wf_spec = spec_parser.get_workflow_spec_by_execution_id( action_ex.task_execution.workflow_execution_id ) wf_spec_name = wf_spec.get_name() adhoc_action_name = action_ex.runtime_context.get('adhoc_action_name') if adhoc_action_name: action_def = actions.resolve_action_definition( adhoc_action_name, wf_name, wf_spec_name ) return actions.AdHocAction(action_def, action_ex=action_ex) action_def = actions.resolve_action_definition( action_ex.name, wf_name, wf_spec_name ) return actions.PythonAction(action_def, action_ex=action_ex)
def _on_action_complete(action_ex): """Handles action completion event. :param action_ex: Action execution. """ task_ex = action_ex.task_execution if not task_ex: return task_spec = spec_parser.get_task_spec(task_ex.spec) wf_ex = task_ex.workflow_execution task = _create_task( wf_ex, spec_parser.get_workflow_spec_by_execution_id(wf_ex.id), task_spec, task_ex.in_context, task_ex) try: task.on_action_complete(action_ex) except exc.MistralException as e: wf_ex = task_ex.workflow_execution msg = ("Failed to handle action completion [error=%s, wf=%s, task=%s," " action=%s]:\n%s" % (e, wf_ex.name, task_ex.name, action_ex.name, tb.format_exc())) LOG.error(msg) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(wf_ex, msg) return
def get_controller(wf_ex, wf_spec=None): """Gets a workflow controller instance by given workflow execution object. :param wf_ex: Workflow execution object. :param wf_spec: Workflow specification object. If passed, the method works faster. :returns: Workflow controller class. """ if not wf_spec: wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id) wf_type = wf_spec.get_type() ctrl_cls = None for cls in u.iter_subclasses(WorkflowController): if cls.__workflow_type__ == wf_type: ctrl_cls = cls break if not ctrl_cls: raise exc.MistralError( 'Failed to find a workflow controller [type=%s]' % wf_type) return ctrl_cls(wf_ex, wf_spec)
def continue_task(task_ex): wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id ) task = _build_task_from_execution(wf_spec, task_ex) try: task.set_state(states.RUNNING, None) task.run() except exc.MistralException as e: wf_ex = task_ex.workflow_execution msg = ( "Failed to run task [error=%s, wf=%s, task=%s]:\n%s" % (e, wf_ex, task_ex.name, tb.format_exc()) ) LOG.error(msg) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(wf_ex, msg) return
def continue_task(task_ex): wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id ) task = _build_task_from_execution(wf_spec, task_ex) try: task.set_state(states.RUNNING, None) task.run() except exc.MistralException as e: wf_ex = task_ex.workflow_execution msg = ( "Failed to run task [wf=%s, task=%s]: %s\n%s" % (wf_ex, task_ex.name, e, tb.format_exc()) ) LOG.error(msg) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(wf_ex, msg) return
def fail_task(task_ex, msg): wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id) task = _build_task_from_execution(wf_spec, task_ex) task.set_state(states.ERROR, msg) wf_handler.fail_workflow(task_ex.workflow_execution, msg)
def __init__(self, wf_ex=None): self.wf_ex = wf_ex if wf_ex: # We're processing a workflow that's already in progress. self.wf_spec = spec_parser.get_workflow_spec_by_execution_id( wf_ex.id) else: self.wf_spec = None
def __init__(self, wf_def, wf_ex=None): self.wf_def = wf_def self.wf_ex = wf_ex if wf_ex: # We're processing a workflow that's already in progress. self.wf_spec = spec_parser.get_workflow_spec_by_execution_id( wf_ex.id) else: # New workflow execution. self.wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at)
def __init__(self, wf_ex, wf_spec=None): """Creates a new workflow controller. :param wf_ex: Workflow execution. :param wf_spec: Workflow specification. """ self.wf_ex = wf_ex if wf_spec is None: wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id) self.wf_spec = wf_spec
def __init__(self, wf_ex, wf_spec=None): """Creates a new workflow controller. :param wf_ex: Workflow execution. :param wf_spec: Workflow specification. """ self.wf_ex = wf_ex if wf_spec is None: wf_spec = spec_parser.get_workflow_spec_by_execution_id(wf_ex.id) self.wf_spec = wf_spec
def __init__(self, wf_def, wf_ex=None): self.wf_def = wf_def self.wf_ex = wf_ex if wf_ex: # We're processing a workflow that's already in progress. self.wf_spec = spec_parser.get_workflow_spec_by_execution_id( wf_ex.id ) else: # New workflow execution. self.wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at )
def _refresh_task_state(task_ex_id): with db_api.transaction(): task_ex = db_api.load_task_execution(task_ex_id) if not task_ex: return wf_ex = task_ex.workflow_execution if states.is_completed(wf_ex.state): return wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id ) wf_ctrl = wf_base.get_controller(wf_ex, wf_spec) state, state_info, cardinality = wf_ctrl.get_logical_task_state( task_ex ) if state == states.RUNNING: continue_task(task_ex) elif state == states.ERROR: task = _build_task_from_execution(wf_spec, task_ex) task.complete(state, state_info) elif state == states.WAITING: # Let's assume that a task takes 0.01 sec in average to complete # and based on this assumption calculate a time of the next check. # The estimation is very rough, of course, but this delay will be # decreasing as task preconditions will be completing which will # give a decent asymptotic approximation. # For example, if a 'join' task has 100 inbound incomplete tasks # then the next 'refresh_task_state' call will happen in 10 # seconds. For 500 tasks it will be 50 seconds. The larger the # workflow is, the more beneficial this mechanism will be. delay = int(cardinality * 0.01) _schedule_refresh_task_state(task_ex, max(1, delay)) else: # Must never get here. raise RuntimeError( 'Unexpected logical task state [task_ex=%s, state=%s]' % (task_ex, state) )
def _refresh_task_state(task_ex_id): with db_api.transaction(): task_ex = db_api.load_task_execution(task_ex_id) if not task_ex: return wf_ex = task_ex.workflow_execution if states.is_completed(wf_ex.state): return wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id ) wf_ctrl = wf_base.get_controller(wf_ex, wf_spec) state, state_info, cardinality = wf_ctrl.get_logical_task_state( task_ex ) if state == states.RUNNING: continue_task(task_ex) elif state == states.ERROR: task = _build_task_from_execution(wf_spec, task_ex) task.complete(state, state_info) elif state == states.WAITING: # Let's assume that a task takes 0.01 sec in average to complete # and based on this assumption calculate a time of the next check. # The estimation is very rough, of course, but this delay will be # decreasing as task preconditions will be completing which will # give a decent asymptotic approximation. # For example, if a 'join' task has 100 inbound incomplete tasks # then the next 'refresh_task_state' call will happen in 10 # seconds. For 500 tasks it will be 50 seconds. The larger the # workflow is, the more beneficial this mechanism will be. delay = int(cardinality * 0.01) _schedule_refresh_task_state(task_ex, max(1, delay)) else: # Must never get here. raise RuntimeError( 'Unexpected logical task state [task_ex=%s, state=%s]' % (task_ex, state) )
def complete_task(task_ex, state, state_info): wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id) task = _build_task_from_execution(wf_spec, task_ex) try: task.complete(state, state_info) except exc.MistralException as e: wf_ex = task_ex.workflow_execution msg = ("Failed to complete task [error=%s, wf=%s, task=%s]:\n%s" % (e, wf_ex, task_ex.name, tb.format_exc())) LOG.error(msg) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(wf_ex, msg) return
def force_fail_task(task_ex, msg): """Forces the given task to fail. This method implements the 'forced' task fail without giving a chance to a workflow controller to handle the error. Its main purpose is to reflect errors caused by workflow structure (errors 'publish', 'on-xxx' clauses etc.) rather than failed actions. If such an error happens we should also force the entire workflow to fail. I.e., this kind of error must be propagated to a higher level, to the workflow. :param task_ex: Task execution. :param msg: Error message. """ wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id) task = _build_task_from_execution(wf_spec, task_ex) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(task_ex.workflow_execution, msg)
def _on_action_complete(action_ex): """Handles action completion event. :param action_ex: Action execution. """ task_ex = action_ex.task_execution if not task_ex: return task_spec = spec_parser.get_task_spec(task_ex.spec) wf_ex = task_ex.workflow_execution task = _create_task( wf_ex, spec_parser.get_workflow_spec_by_execution_id(wf_ex.id), task_spec, task_ex.in_context, task_ex ) try: task.on_action_complete(action_ex) except exc.MistralException as e: wf_ex = task_ex.workflow_execution msg = ("Failed to handle action completion [error=%s, wf=%s, task=%s," " action=%s]:\n%s" % (e, wf_ex.name, task_ex.name, action_ex.name, tb.format_exc())) LOG.error(msg) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(wf_ex, msg) return
def force_fail_task(task_ex, msg): """Forces the given task to fail. This method implements the 'forced' task fail without giving a chance to a workflow controller to handle the error. Its main purpose is to reflect errors caused by workflow structure (errors 'publish', 'on-xxx' clauses etc.) rather than failed actions. If such an error happens we should also force the entire workflow to fail. I.e., this kind of error must be propagated to a higher level, to the workflow. :param task_ex: Task execution. :param msg: Error message. """ wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id ) task = _build_task_from_execution(wf_spec, task_ex) task.set_state(states.ERROR, msg) wf_handler.force_fail_workflow(task_ex.workflow_execution, msg)
def _refresh_task_state(task_ex_id): with db_api.transaction(): task_ex = db_api.get_task_execution(task_ex_id) wf_spec = spec_parser.get_workflow_spec_by_execution_id( task_ex.workflow_execution_id) wf_ctrl = wf_base.get_controller(task_ex.workflow_execution, wf_spec) state, state_info = wf_ctrl.get_logical_task_state(task_ex) if state == states.RUNNING: continue_task(task_ex) elif state == states.ERROR: fail_task(task_ex, state_info) elif state == states.WAITING: # TODO(rakhmerov): Algorithm for increasing rescheduling delay. _schedule_refresh_task_state(task_ex, 1) else: # Must never get here. raise RuntimeError( 'Unexpected logical task state [task_ex=%s, state=%s]' % (task_ex, state))
def test_cache_workflow_spec_by_execution_id(self): wf_text = """ version: '2.0' wf: tasks: task1: action: std.echo output="Echo" """ wfs = wf_service.create_workflows(wf_text) self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size()) self.assertEqual(0, spec_parser.get_wf_definition_spec_cache_size()) wf_def = wfs[0] wf_spec = spec_parser.get_workflow_spec_by_definition_id( wf_def.id, wf_def.updated_at ) self.assertEqual(1, len(wf_spec.get_tasks())) self.assertEqual(0, spec_parser.get_wf_execution_spec_cache_size()) self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size()) wf_ex = db_api.create_workflow_execution({ 'id': '1-2-3-4', 'name': 'wf', 'workflow_id': wf_def.id, 'spec': wf_spec.to_dict(), 'state': states.RUNNING }) # Check that we can get a valid spec by execution id. wf_spec_by_exec_id = spec_parser.get_workflow_spec_by_execution_id( wf_ex.id ) self.assertEqual(1, len(wf_spec_by_exec_id.get_tasks())) # Now update workflow definition and check that cache is updated too. wf_text = """ version: '2.0' wf: tasks: task1: action: std.echo output="1" task2: action: std.echo output="2" """ wfs = wf_service.update_workflows(wf_text) self.assertEqual(1, spec_parser.get_wf_definition_spec_cache_size()) wf_spec = spec_parser.get_workflow_spec_by_definition_id( wfs[0].id, wfs[0].updated_at ) self.assertEqual(2, len(wf_spec.get_tasks())) self.assertEqual(2, spec_parser.get_wf_definition_spec_cache_size()) self.assertEqual(1, spec_parser.get_wf_execution_spec_cache_size()) # Now finally update execution cache and check that we can # get a valid spec by execution id. spec_parser.cache_workflow_spec_by_execution_id(wf_ex.id, wf_spec) wf_spec_by_exec_id = spec_parser.get_workflow_spec_by_execution_id( wf_ex.id ) self.assertEqual(2, len(wf_spec_by_exec_id.get_tasks()))