def rerun_workflow(wf_ex, task_ex, reset=True, env=None):
if wf_ex.state == states.PAUSED:
return wf_ex.get_clone()
wf = workflows.Workflow(wf_ex=wf_ex)
wf.rerun(task_ex, reset=reset, env=env)
def resume_workflow(wf_ex, env=None):
if not states.is_paused_or_idle(wf_ex.state):
return wf_ex.get_clone()
wf = workflows.Workflow(wf_ex=wf_ex)
wf.resume(env=env)
def _check_and_complete(wf_ex_id):
# Note: This method can only be called via scheduler.
with db_api.transaction():
wf_ex = db_api.load_workflow_execution(wf_ex_id)
if not wf_ex or states.is_completed(wf_ex.state):
return
wf = workflows.Workflow(wf_ex=wf_ex)
try:
incomplete_tasks_count = wf.check_and_complete()
except exc.MistralException as e:
msg = ("Failed to check and complete [wf_ex=%s]:"
" %s\n%s" % (wf_ex, e, tb.format_exc()))
LOG.error(msg)
force_fail_workflow(wf.wf_ex, msg)
return
if not states.is_completed(wf_ex.state):
# Let's assume that a task takes 0.01 sec in average to complete
# and based on this assumption calculate a time of the next check.
# The estimation is very rough but this delay will be decreasing
# as tasks will be completing which will give a decent
# approximation.
# For example, if a workflow has 100 incomplete tasks then the
# next check call will happen in 10 seconds. For 500 tasks it will
# be 50 seconds. The larger the workflow is, the more beneficial
# this mechanism will be.
delay = int(incomplete_tasks_count * 0.01)
_schedule_check_and_complete(wf_ex, delay)
def _on_task_complete(task_ex_id):
# Note: This method can only be called via scheduler.
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
wf_ex = task_ex.workflow_execution
wf = workflows.Workflow(db_api.get_workflow_definition(
wf_ex.workflow_id),
wf_ex=wf_ex)
try:
wf.on_task_complete(task_ex)
except exc.MistralException as e:
msg = ("Failed to handle task completion [wf_ex=%s, task_ex=%s]:"
" %s\n%s" % (wf_ex, task_ex, e, tb.format_exc()))
LOG.error(msg)
fail_workflow(wf.wf_ex, msg)
return
if not states.is_completed(wf_ex.state):
# TODO(rakhmerov): Moving forward we can implement some more fancy
# algorithm for increasing delay for rescheduling so that we don't
# put too serious load onto scheduler.
delay = 1
schedule_on_task_complete(task_ex, delay)
def test_set_state(self):
wf_text = """
version: '2.0'
wf:
tasks:
task1:
action: std.echo output="Echo"
on-success:
- task2
task2:
action: std.noop
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_success(wf_ex.id)
# The state in db is SUCCESS, but wf_ex still contains outdated info.
self.assertEqual("RUNNING", wf_ex.state)
wf = workflows.Workflow(wf_ex)
# Trying to change the status of succeed execution. There is no error,
# only warning message that state has been changed in db.
wf.set_state("ERROR")
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual("SUCCESS", wf_ex.state)
def pause_workflow(wf_ex, msg=None):
wf = workflows.Workflow(
db_api.get_workflow_definition(wf_ex.workflow_id),
wf_ex=wf_ex
)
wf.set_state(states.PAUSED, msg)
def start_workflow(wf_identifier, wf_input, desc, params):
wf = workflows.Workflow(
db_api.get_workflow_definition(wf_identifier)
)
wf.start(wf_input, desc=desc, params=params)
return wf.wf_ex
def resume_workflow(wf_ex, env=None):
if not states.is_paused_or_idle(wf_ex.state):
return wf_ex.get_clone()
wf = workflows.Workflow(db_api.get_workflow_definition(wf_ex.workflow_id),
wf_ex=wf_ex)
wf.resume(env=env)
def start_workflow(wf_identifier, wf_input, desc, params):
wf = workflows.Workflow(db_api.get_workflow_definition(wf_identifier))
wf.start(wf_input, desc=desc, params=params)
_schedule_check_and_complete(wf.wf_ex)
return wf.wf_ex
def stop_workflow(wf_ex, state, msg=None):
wf = workflows.Workflow(
db_api.get_workflow_definition(wf_ex.workflow_id),
wf_ex=wf_ex
)
# In this case we should not try to handle possible errors. Instead,
# we need to let them pop up since the typical way of failing objects
# doesn't work here. Failing a workflow is the same as stopping it
# with ERROR state.
wf.stop(state, msg)
def rerun_workflow(wf_ex, task_ex, reset=True, env=None):
if wf_ex.state == states.PAUSED:
return wf_ex.get_clone()
wf = workflows.Workflow(wf_ex=wf_ex)
wf.rerun(task_ex, reset=reset, env=env)
_schedule_check_and_complete(wf_ex)
if wf_ex.task_execution_id:
_schedule_check_and_complete(wf_ex.task_execution.workflow_execution)
def start_workflow(wf_identifier, wf_namespace, wf_input, desc, params):
wf = workflows.Workflow()
wf_def = db_api.get_workflow_definition(wf_identifier, wf_namespace)
if 'namespace' not in params:
params['namespace'] = wf_def.namespace
wf.start(wf_def=wf_def, input_dict=wf_input, desc=desc, params=params)
_schedule_check_and_complete(wf.wf_ex)
return wf.wf_ex
def rerun_workflow(wf_ex, task_ex, reset=True, env=None):
if wf_ex.state == states.PAUSED:
return wf_ex.get_clone()
wf = workflows.Workflow(wf_ex=wf_ex)
wf.rerun(task_ex, reset=reset, env=env)
_schedule_check_and_fix_integrity(
wf_ex, delay=CONF.engine.execution_integrity_check_delay)
if wf_ex.task_execution_id:
_schedule_check_and_fix_integrity(
wf_ex.task_execution.workflow_execution,
delay=CONF.engine.execution_integrity_check_delay)
def pause_workflow(wf_ex, msg=None):
# Pause subworkflows first.
for task_ex in wf_ex.task_executions:
sub_wf_exs = db_api.get_workflow_executions(
task_execution_id=task_ex.id)
for sub_wf_ex in sub_wf_exs:
if not states.is_completed(sub_wf_ex.state):
pause_workflow(sub_wf_ex, msg=msg)
# If all subworkflows paused successfully, pause the main workflow.
# If any subworkflows failed to pause for temporary reason, this
# allows pause to be executed again on the main workflow.
wf = workflows.Workflow(wf_ex=wf_ex)
wf.pause(msg=msg)
def on_task_complete(task_ex):
wf_ex = task_ex.workflow_execution
wf = workflows.Workflow(db_api.get_workflow_definition(wf_ex.workflow_id),
wf_ex=wf_ex)
try:
wf.on_task_complete(task_ex)
except exc.MistralException as e:
msg = (
"Failed to handle task completion [wf_ex=%s, task_ex=%s]: %s\n%s" %
(wf_ex, task_ex, e, tb.format_exc()))
LOG.error(msg)
fail_workflow(wf.wf_ex, msg)
def check_and_complete(wf_ex_id):
wf_ex = db_api.load_workflow_execution(wf_ex_id)
if not wf_ex or states.is_completed(wf_ex.state):
return
wf = workflows.Workflow(wf_ex=wf_ex)
try:
wf.check_and_complete()
except exc.MistralException as e:
msg = ("Failed to check and complete [wf_ex_id=%s, wf_name=%s]:"
" %s\n%s" % (wf_ex_id, wf_ex.name, e, tb.format_exc()))
LOG.error(msg)
force_fail_workflow(wf.wf_ex, msg)
def resume_workflow(wf_ex, env=None):
if not states.is_paused_or_idle(wf_ex.state):
return wf_ex.get_clone()
# Resume subworkflows first.
for task_ex in wf_ex.task_executions:
sub_wf_exs = db_api.get_workflow_executions(
task_execution_id=task_ex.id)
for sub_wf_ex in sub_wf_exs:
if not states.is_completed(sub_wf_ex.state):
resume_workflow(sub_wf_ex)
# Resume current workflow here so to trigger continue workflow only
# after all other subworkflows are placed back in running state.
wf = workflows.Workflow(wf_ex=wf_ex)
wf.resume(env=env)
def stop_workflow(wf_ex, state, msg=None):
wf = workflows.Workflow(wf_ex=wf_ex)
# In this case we should not try to handle possible errors. Instead,
# we need to let them pop up since the typical way of failing objects
# doesn't work here. Failing a workflow is the same as stopping it
# with ERROR state.
wf.stop(state, msg)
# Cancels subworkflows.
if state == states.CANCELLED:
for task_ex in wf_ex.task_executions:
sub_wf_exs = db_api.get_workflow_executions(
task_execution_id=task_ex.id)
for sub_wf_ex in sub_wf_exs:
if not states.is_completed(sub_wf_ex.state):
stop_workflow(sub_wf_ex, state, msg=msg)
def rerun_workflow(wf_ex, task_ex, reset=True, env=None):
if wf_ex.state == states.PAUSED:
return wf_ex.get_clone()
# To break cyclic dependency.
from mistral.engine import task_handler
wf = workflows.Workflow(wf_ex=wf_ex)
task = task_handler.build_task_from_execution(wf.wf_spec, task_ex)
wf.rerun(task, reset=reset, env=env)
_schedule_check_and_fix_integrity(
wf_ex, delay=CONF.engine.execution_integrity_check_delay)
if wf_ex.task_execution_id:
_schedule_check_and_fix_integrity(
wf_ex.task_execution.workflow_execution,
delay=CONF.engine.execution_integrity_check_delay)
def _check_and_complete(wf_ex_id):
# Note: This method can only be called via scheduler.
with db_api.transaction():
wf_ex = db_api.load_workflow_execution(wf_ex_id)
if not wf_ex or states.is_completed(wf_ex.state):
return
wf = workflows.Workflow(wf_ex=wf_ex)
try:
check_and_fix_integrity(wf_ex)
num_incomplete_tasks = wf.check_and_complete()
if not states.is_completed(wf_ex.state):
delay = (
2 + int(num_incomplete_tasks * 0.1) if num_incomplete_tasks
else 4
)
# Rescheduling this check may not happen if errors are
# raised in the business logic. If the error is DB related
# and not considered fatal (e.g. disconnect, deadlock), the
# retry annotation around the method will ensure that the
# whole method is retried in a new transaction. On fatal
# errors, the check should not be rescheduled as it could
# result in undesired consequences.
# In case there are some errors that should not be
# considered fatal, those should be handled explicitly.
_schedule_check_and_complete(wf_ex, delay)
except exc.MistralException as e:
msg = (
"Failed to check and complete [wf_ex_id=%s, wf_name=%s]:"
" %s\n%s" % (wf_ex_id, wf_ex.name, e, tb.format_exc())
)
LOG.error(msg)
force_fail_workflow(wf.wf_ex, msg)
def pause_workflow(wf_ex, msg=None):
wf = workflows.Workflow(wf_ex=wf_ex)
wf.set_state(states.PAUSED, msg)
