def rerun_workflow(wf_ex, task_ex, reset=True, env=None): if wf_ex.state == states.PAUSED: return wf_ex.get_clone() wf = workflows.Workflow(wf_ex=wf_ex) wf.rerun(task_ex, reset=reset, env=env)
def resume_workflow(wf_ex, env=None): if not states.is_paused_or_idle(wf_ex.state): return wf_ex.get_clone() wf = workflows.Workflow(wf_ex=wf_ex) wf.resume(env=env)
def _check_and_complete(wf_ex_id): # Note: This method can only be called via scheduler. with db_api.transaction(): wf_ex = db_api.load_workflow_execution(wf_ex_id) if not wf_ex or states.is_completed(wf_ex.state): return wf = workflows.Workflow(wf_ex=wf_ex) try: incomplete_tasks_count = wf.check_and_complete() except exc.MistralException as e: msg = ("Failed to check and complete [wf_ex=%s]:" " %s\n%s" % (wf_ex, e, tb.format_exc())) LOG.error(msg) force_fail_workflow(wf.wf_ex, msg) return if not states.is_completed(wf_ex.state): # Let's assume that a task takes 0.01 sec in average to complete # and based on this assumption calculate a time of the next check. # The estimation is very rough but this delay will be decreasing # as tasks will be completing which will give a decent # approximation. # For example, if a workflow has 100 incomplete tasks then the # next check call will happen in 10 seconds. For 500 tasks it will # be 50 seconds. The larger the workflow is, the more beneficial # this mechanism will be. delay = int(incomplete_tasks_count * 0.01) _schedule_check_and_complete(wf_ex, delay)
def _on_task_complete(task_ex_id): # Note: This method can only be called via scheduler. with db_api.transaction(): task_ex = db_api.get_task_execution(task_ex_id) wf_ex = task_ex.workflow_execution wf = workflows.Workflow(db_api.get_workflow_definition( wf_ex.workflow_id), wf_ex=wf_ex) try: wf.on_task_complete(task_ex) except exc.MistralException as e: msg = ("Failed to handle task completion [wf_ex=%s, task_ex=%s]:" " %s\n%s" % (wf_ex, task_ex, e, tb.format_exc())) LOG.error(msg) fail_workflow(wf.wf_ex, msg) return if not states.is_completed(wf_ex.state): # TODO(rakhmerov): Moving forward we can implement some more fancy # algorithm for increasing delay for rescheduling so that we don't # put too serious load onto scheduler. delay = 1 schedule_on_task_complete(task_ex, delay)
def test_set_state(self): wf_text = """ version: '2.0' wf: tasks: task1: action: std.echo output="Echo" on-success: - task2 task2: action: std.noop """ wf_service.create_workflows(wf_text) wf_ex = self.engine.start_workflow('wf') self.await_workflow_success(wf_ex.id) # The state in db is SUCCESS, but wf_ex still contains outdated info. self.assertEqual("RUNNING", wf_ex.state) wf = workflows.Workflow(wf_ex) # Trying to change the status of succeed execution. There is no error, # only warning message that state has been changed in db. wf.set_state("ERROR") with db_api.transaction(): wf_ex = db_api.get_workflow_execution(wf_ex.id) self.assertEqual("SUCCESS", wf_ex.state)
def pause_workflow(wf_ex, msg=None): wf = workflows.Workflow( db_api.get_workflow_definition(wf_ex.workflow_id), wf_ex=wf_ex ) wf.set_state(states.PAUSED, msg)
def start_workflow(wf_identifier, wf_input, desc, params): wf = workflows.Workflow( db_api.get_workflow_definition(wf_identifier) ) wf.start(wf_input, desc=desc, params=params) return wf.wf_ex
def resume_workflow(wf_ex, env=None): if not states.is_paused_or_idle(wf_ex.state): return wf_ex.get_clone() wf = workflows.Workflow(db_api.get_workflow_definition(wf_ex.workflow_id), wf_ex=wf_ex) wf.resume(env=env)
def start_workflow(wf_identifier, wf_input, desc, params): wf = workflows.Workflow(db_api.get_workflow_definition(wf_identifier)) wf.start(wf_input, desc=desc, params=params) _schedule_check_and_complete(wf.wf_ex) return wf.wf_ex
def stop_workflow(wf_ex, state, msg=None): wf = workflows.Workflow( db_api.get_workflow_definition(wf_ex.workflow_id), wf_ex=wf_ex ) # In this case we should not try to handle possible errors. Instead, # we need to let them pop up since the typical way of failing objects # doesn't work here. Failing a workflow is the same as stopping it # with ERROR state. wf.stop(state, msg)
def rerun_workflow(wf_ex, task_ex, reset=True, env=None): if wf_ex.state == states.PAUSED: return wf_ex.get_clone() wf = workflows.Workflow(wf_ex=wf_ex) wf.rerun(task_ex, reset=reset, env=env) _schedule_check_and_complete(wf_ex) if wf_ex.task_execution_id: _schedule_check_and_complete(wf_ex.task_execution.workflow_execution)
def start_workflow(wf_identifier, wf_namespace, wf_input, desc, params): wf = workflows.Workflow() wf_def = db_api.get_workflow_definition(wf_identifier, wf_namespace) if 'namespace' not in params: params['namespace'] = wf_def.namespace wf.start(wf_def=wf_def, input_dict=wf_input, desc=desc, params=params) _schedule_check_and_complete(wf.wf_ex) return wf.wf_ex
def rerun_workflow(wf_ex, task_ex, reset=True, env=None): if wf_ex.state == states.PAUSED: return wf_ex.get_clone() wf = workflows.Workflow(wf_ex=wf_ex) wf.rerun(task_ex, reset=reset, env=env) _schedule_check_and_fix_integrity( wf_ex, delay=CONF.engine.execution_integrity_check_delay) if wf_ex.task_execution_id: _schedule_check_and_fix_integrity( wf_ex.task_execution.workflow_execution, delay=CONF.engine.execution_integrity_check_delay)
def pause_workflow(wf_ex, msg=None): # Pause subworkflows first. for task_ex in wf_ex.task_executions: sub_wf_exs = db_api.get_workflow_executions( task_execution_id=task_ex.id) for sub_wf_ex in sub_wf_exs: if not states.is_completed(sub_wf_ex.state): pause_workflow(sub_wf_ex, msg=msg) # If all subworkflows paused successfully, pause the main workflow. # If any subworkflows failed to pause for temporary reason, this # allows pause to be executed again on the main workflow. wf = workflows.Workflow(wf_ex=wf_ex) wf.pause(msg=msg)
def on_task_complete(task_ex): wf_ex = task_ex.workflow_execution wf = workflows.Workflow(db_api.get_workflow_definition(wf_ex.workflow_id), wf_ex=wf_ex) try: wf.on_task_complete(task_ex) except exc.MistralException as e: msg = ( "Failed to handle task completion [wf_ex=%s, task_ex=%s]: %s\n%s" % (wf_ex, task_ex, e, tb.format_exc())) LOG.error(msg) fail_workflow(wf.wf_ex, msg)
def check_and_complete(wf_ex_id): wf_ex = db_api.load_workflow_execution(wf_ex_id) if not wf_ex or states.is_completed(wf_ex.state): return wf = workflows.Workflow(wf_ex=wf_ex) try: wf.check_and_complete() except exc.MistralException as e: msg = ("Failed to check and complete [wf_ex_id=%s, wf_name=%s]:" " %s\n%s" % (wf_ex_id, wf_ex.name, e, tb.format_exc())) LOG.error(msg) force_fail_workflow(wf.wf_ex, msg)
def resume_workflow(wf_ex, env=None): if not states.is_paused_or_idle(wf_ex.state): return wf_ex.get_clone() # Resume subworkflows first. for task_ex in wf_ex.task_executions: sub_wf_exs = db_api.get_workflow_executions( task_execution_id=task_ex.id) for sub_wf_ex in sub_wf_exs: if not states.is_completed(sub_wf_ex.state): resume_workflow(sub_wf_ex) # Resume current workflow here so to trigger continue workflow only # after all other subworkflows are placed back in running state. wf = workflows.Workflow(wf_ex=wf_ex) wf.resume(env=env)
def stop_workflow(wf_ex, state, msg=None): wf = workflows.Workflow(wf_ex=wf_ex) # In this case we should not try to handle possible errors. Instead, # we need to let them pop up since the typical way of failing objects # doesn't work here. Failing a workflow is the same as stopping it # with ERROR state. wf.stop(state, msg) # Cancels subworkflows. if state == states.CANCELLED: for task_ex in wf_ex.task_executions: sub_wf_exs = db_api.get_workflow_executions( task_execution_id=task_ex.id) for sub_wf_ex in sub_wf_exs: if not states.is_completed(sub_wf_ex.state): stop_workflow(sub_wf_ex, state, msg=msg)
def rerun_workflow(wf_ex, task_ex, reset=True, env=None): if wf_ex.state == states.PAUSED: return wf_ex.get_clone() # To break cyclic dependency. from mistral.engine import task_handler wf = workflows.Workflow(wf_ex=wf_ex) task = task_handler.build_task_from_execution(wf.wf_spec, task_ex) wf.rerun(task, reset=reset, env=env) _schedule_check_and_fix_integrity( wf_ex, delay=CONF.engine.execution_integrity_check_delay) if wf_ex.task_execution_id: _schedule_check_and_fix_integrity( wf_ex.task_execution.workflow_execution, delay=CONF.engine.execution_integrity_check_delay)
def _check_and_complete(wf_ex_id): # Note: This method can only be called via scheduler. with db_api.transaction(): wf_ex = db_api.load_workflow_execution(wf_ex_id) if not wf_ex or states.is_completed(wf_ex.state): return wf = workflows.Workflow(wf_ex=wf_ex) try: check_and_fix_integrity(wf_ex) num_incomplete_tasks = wf.check_and_complete() if not states.is_completed(wf_ex.state): delay = ( 2 + int(num_incomplete_tasks * 0.1) if num_incomplete_tasks else 4 ) # Rescheduling this check may not happen if errors are # raised in the business logic. If the error is DB related # and not considered fatal (e.g. disconnect, deadlock), the # retry annotation around the method will ensure that the # whole method is retried in a new transaction. On fatal # errors, the check should not be rescheduled as it could # result in undesired consequences. # In case there are some errors that should not be # considered fatal, those should be handled explicitly. _schedule_check_and_complete(wf_ex, delay) except exc.MistralException as e: msg = ( "Failed to check and complete [wf_ex_id=%s, wf_name=%s]:" " %s\n%s" % (wf_ex_id, wf_ex.name, e, tb.format_exc()) ) LOG.error(msg) force_fail_workflow(wf.wf_ex, msg)
def pause_workflow(wf_ex, msg=None): wf = workflows.Workflow(wf_ex=wf_ex) wf.set_state(states.PAUSED, msg)