def _check_and_complete(wf_ex_id):
# Note: This method can only be called via scheduler.
with db_api.transaction():
wf_ex = db_api.load_workflow_execution(wf_ex_id)
if not wf_ex or states.is_completed(wf_ex.state):
return
wf = workflows.Workflow(wf_ex=wf_ex)
try:
incomplete_tasks_count = wf.check_and_complete()
except exc.MistralException as e:
msg = ("Failed to check and complete [wf_ex=%s]:"
" %s\n%s" % (wf_ex, e, tb.format_exc()))
LOG.error(msg)
force_fail_workflow(wf.wf_ex, msg)
return
if not states.is_completed(wf_ex.state):
# Let's assume that a task takes 0.01 sec in average to complete
# and based on this assumption calculate a time of the next check.
# The estimation is very rough but this delay will be decreasing
# as tasks will be completing which will give a decent
# approximation.
# For example, if a workflow has 100 incomplete tasks then the
# next check call will happen in 10 seconds. For 500 tasks it will
# be 50 seconds. The larger the workflow is, the more beneficial
# this mechanism will be.
delay = int(incomplete_tasks_count * 0.01)
_schedule_check_and_complete(wf_ex, delay)
def update(self, state, state_info=None):
"""Update task and set specified state.
Method sets specified task state.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Ignore if task already completed.
if states.is_completed(self.task_ex.state):
return
# Update only if state transition is valid.
if not states.is_valid_transition(self.task_ex.state, state):
return
# We can't set the task state to RUNNING if some other
# child executions are paused.
child_states = [a_ex.state for a_ex in self.task_ex.executions]
if state == states.RUNNING and states.PAUSED in child_states:
return
self.set_state(state, state_info)
if states.is_completed(self.task_ex.state):
self.register_workflow_completion_check()
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.load_task_execution(task_ex_id)
if not task_ex:
return
if (states.is_completed(task_ex.state)
or task_ex.state == states.RUNNING):
return
wf_ex = task_ex.workflow_execution
if states.is_completed(wf_ex.state):
return
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
wf_ctrl = wf_base.get_controller(wf_ex, wf_spec)
with db_api.named_lock(task_ex.id):
# NOTE: we have to use this lock to prevent two (or more) such
# methods from changing task state and starting its action or
# workflow. Checking task state outside of this section is a
# performance optimization because locking is pretty expensive.
db_api.refresh(task_ex)
if (states.is_completed(task_ex.state)
or task_ex.state == states.RUNNING):
return
log_state = wf_ctrl.get_logical_task_state(task_ex)
state = log_state.state
state_info = log_state.state_info
# Update 'triggered_by' because it could have changed.
task_ex.runtime_context['triggered_by'] = log_state.triggered_by
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
complete_task(task_ex, state, state_info)
elif state == states.WAITING:
LOG.info(
"Task execution is still in WAITING state"
" [task_ex_id=%s, task_name=%s]",
task_ex_id,
task_ex.name
)
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex_id=%s, '
'task_name=%s, state=%s]' %
(task_ex_id, task_ex.name, state)
)
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.load_task_execution(task_ex_id)
if not task_ex:
return
if (states.is_completed(task_ex.state)
or task_ex.state == states.RUNNING):
return
wf_ex = task_ex.workflow_execution
if states.is_completed(wf_ex.state):
return
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id
)
wf_ctrl = wf_base.get_controller(wf_ex, wf_spec)
with db_api.named_lock(task_ex.id):
# NOTE: we have to use this lock to prevent two (or more) such
# methods from changing task state and starting its action or
# workflow. Checking task state outside of this section is a
# performance optimization because locking is pretty expensive.
db_api.refresh(task_ex)
if (states.is_completed(task_ex.state)
or task_ex.state == states.RUNNING):
return
log_state = wf_ctrl.get_logical_task_state(task_ex)
state = log_state.state
state_info = log_state.state_info
# Update 'triggered_by' because it could have changed.
task_ex.runtime_context['triggered_by'] = log_state.triggered_by
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
complete_task(task_ex, state, state_info)
elif state == states.WAITING:
LOG.info(
"Task execution is still in WAITING state"
" [task_ex_id=%s, task_name=%s]",
task_ex_id,
task_ex.name
)
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex_id=%s, '
'task_name=%s, state=%s]' %
(task_ex_id, task_ex.name, state)
)
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(
id=self.wf_ex.id,
cur_state=cur_state,
state=state
)
if wf_ex is None:
# Do nothing because the state was updated previously.
return
self.wf_ex = wf_ex
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex,
"Workflow '%s' [%s -> %s, msg=%s]"
% (self.wf_ex.workflow_name, cur_state, state, state_info)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
# No need to keep task executions of this workflow in the
# lookup cache anymore.
lookup_utils.invalidate_cached_task_executions(self.wf_ex.id)
triggers.on_workflow_complete(self.wf_ex)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id
)
parent_wf = Workflow(wf_ex=parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def delete(self, id, force=False):
"""Delete the specified Execution.
:param id: UUID of execution to delete.
:param force: Optional. Force the deletion of unfinished executions.
Default: false. While the api is backward compatible
the behaviour is not the same. The new default is the
safer option
"""
acl.enforce('executions:delete', context.ctx())
LOG.debug("Delete execution [id=%s]", id)
if not force:
state = db_api.get_workflow_execution(
id,
fields=(db_models.WorkflowExecution.state,)
)[0]
if not states.is_completed(state):
raise exc.NotAllowedException(
"Only completed executions can be deleted. "
"Use --force to override this. "
"Execution {} is in {} state".format(id, state)
)
return rest_utils.rest_retry_on_db_error(
db_api.delete_workflow_execution
)(id)
def _find_next_task_names(self, task_ex, ctx):
t_state = task_ex.state
t_name = task_ex.name
t_names = []
if states.is_completed(t_state):
t_names += self._find_next_task_names_for_clause(
self.get_on_complete_clause(t_name),
ctx
)
if t_state == states.ERROR:
t_names += self._find_next_task_names_for_clause(
self.get_on_error_clause(t_name),
ctx
)
elif t_state == states.SUCCESS:
t_names += self._find_next_task_names_for_clause(
self.get_on_success_clause(t_name),
ctx
)
return t_names
def _find_next_tasks(self, task_ex, ctx=None):
t_state = task_ex.state
t_name = task_ex.name
ctx_view = data_flow.ContextView(
ctx or data_flow.evaluate_task_outbound_context(task_ex),
self.wf_ex.context, self.wf_ex.input)
# [(task_name, params, 'on-success'|'on-error'|'on-complete'), ...]
result = []
def process_clause(clause, event_name):
task_tuples = self._find_next_tasks_for_clause(clause, ctx_view)
for t in task_tuples:
result.append((t[0], t[1], event_name))
if t_state == states.SUCCESS:
process_clause(self.wf_spec.get_on_success_clause(t_name),
'on-success')
elif t_state == states.ERROR:
process_clause(self.wf_spec.get_on_error_clause(t_name),
'on-error')
if states.is_completed(t_state) and not states.is_cancelled(t_state):
process_clause(self.wf_spec.get_on_complete_clause(t_name),
'on-complete')
return result
def find_task_executions_by_name(wf_ex_id, task_name):
"""Finds task executions by workflow execution id and task name.
:param wf_ex_id: Workflow execution id.
:param task_name: Task name.
:return: Task executions (possibly a cached value).
"""
cache_key = (wf_ex_id, task_name)
with _TASK_EXECUTIONS_CACHE_LOCK:
t_execs = _TASK_EXECUTIONS_CACHE.get(cache_key)
if t_execs:
return t_execs
t_execs = db_api.get_task_executions(
workflow_execution_id=wf_ex_id,
name=task_name
)
# We can cache only finished tasks because they won't change.
all_finished = (
t_execs and
all([states.is_completed(t_ex.state) for t_ex in t_execs])
)
if all_finished:
with _TASK_EXECUTIONS_CACHE_LOCK:
_TASK_EXECUTIONS_CACHE[cache_key] = t_execs
return t_execs
def _continue_workflow(self, task_ex=None, reset=True, env=None):
wf_ctrl = wf_base.get_controller(self.wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset, env=env)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
cmds = list(
filter(lambda c: not isinstance(c, commands.PauseWorkflow), cmds)
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in self.wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
if not cmds:
self._check_and_complete()
def delete(self, id, force=False):
"""Delete the specified Execution.
:param id: UUID of execution to delete.
:param force: Optional. Force the deletion of unfinished executions.
Default: false. While the api is backward compatible
the behaviour is not the same. The new default is the
safer option
"""
acl.enforce('executions:delete', context.ctx())
LOG.debug("Delete execution [id=%s]", id)
if not force:
state = db_api.get_workflow_execution(
id, fields=(db_models.WorkflowExecution.state, ))[0]
if not states.is_completed(state):
raise exc.NotAllowedException(
"Only completed executions can be deleted. "
"Use --force to override this. "
"Execution {} is in {} state".format(id, state))
return rest_utils.rest_retry_on_db_error(
db_api.delete_workflow_execution)(id)
def _continue_workflow(self, task_ex=None, reset=True, env=None):
wf_ctrl = wf_base.get_controller(self.wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset, env=env)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was paused.
cmds = list(
filter(lambda c: not isinstance(c, commands.PauseWorkflow), cmds)
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in self.wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
if not cmds:
self._check_and_complete()
def delete(self, id):
"""Delete the specified action_execution.
:param id: UUID of action execution to delete
"""
acl.enforce('action_executions:delete', context.ctx())
LOG.debug("Delete action_execution [id=%s]", id)
if not cfg.CONF.api.allow_action_execution_deletion:
raise exc.NotAllowedException("Action execution deletion is not "
"allowed.")
with db_api.transaction():
action_ex = db_api.get_action_execution(id)
if action_ex.task_execution_id:
raise exc.NotAllowedException(
"Only ad-hoc action execution can be deleted."
)
if not states.is_completed(action_ex.state):
raise exc.NotAllowedException(
"Only completed action execution can be deleted."
)
return db_api.delete_action_execution(id)
def _continue_workflow(self, wf_ex, task_ex=None, reset=True, env=None):
wf_ex = wf_service.update_workflow_execution_env(wf_ex, env)
wf_handler.set_execution_state(wf_ex,
states.RUNNING,
set_upstream=True)
wf_ctrl = wf_base.WorkflowController.get_controller(wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset, env=env)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was pause.
cmds = list(
filter(lambda c: not isinstance(c, commands.PauseWorkflow), cmds))
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
self._dispatch_workflow_commands(wf_ex, cmds)
if not cmds:
if not wf_utils.find_incomplete_task_executions(wf_ex):
wf_handler.succeed_workflow(
wf_ex, wf_ctrl.evaluate_workflow_final_context())
return wf_ex.get_clone()
def _on_task_state_change(self, task_ex, wf_ex, task_state=states.SUCCESS):
task_spec = spec_parser.get_task_spec(task_ex.spec)
wf_spec = spec_parser.get_workflow_spec(wf_ex.spec)
# We must be sure that if task is completed,
# it was also completed in previous transaction.
if (task_handler.is_task_completed(task_ex, task_spec)
and states.is_completed(task_state)):
task_handler.after_task_complete(task_ex, task_spec, wf_spec)
# Ignore DELAYED state.
if task_ex.state == states.RUNNING_DELAYED:
return
wf_ctrl = wf_base.WorkflowController.get_controller(wf_ex, wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow()
task_ex.processed = True
self._dispatch_workflow_commands(wf_ex, cmds)
self._check_workflow_completion(wf_ex, wf_ctrl)
elif task_handler.need_to_continue(task_ex, task_spec):
# Re-run existing task.
cmds = [commands.RunExistingTask(task_ex, reset=False)]
self._dispatch_workflow_commands(wf_ex, cmds)
def _dispatch_workflow_commands(self, wf_ex, wf_cmds):
if not wf_cmds:
return
for cmd in wf_cmds:
if isinstance(cmd, commands.RunTask) and cmd.is_waiting():
task_handler.defer_task(cmd)
elif isinstance(cmd, commands.RunTask):
task_handler.run_new_task(cmd)
elif isinstance(cmd, commands.RunExistingTask):
task_handler.run_existing_task(
cmd.task_ex.id,
reset=cmd.reset
)
elif isinstance(cmd, commands.SetWorkflowState):
if states.is_completed(cmd.new_state):
self._stop_workflow(cmd.wf_ex, cmd.new_state, cmd.msg)
else:
wf_handler.set_execution_state(wf_ex, cmd.new_state)
elif isinstance(cmd, commands.Noop):
# Do nothing.
pass
else:
raise RuntimeError('Unsupported workflow command: %s' % cmd)
if wf_ex.state != states.RUNNING:
break
def delete(self, id):
"""Delete the specified action_execution.
:param id: UUID of action execution to delete
"""
acl.enforce('action_executions:delete', context.ctx())
LOG.debug("Delete action_execution [id=%s]", id)
if not cfg.CONF.api.allow_action_execution_deletion:
raise exc.NotAllowedException("Action execution deletion is not "
"allowed.")
with db_api.transaction():
action_ex = db_api.get_action_execution(id)
if action_ex.task_execution_id:
raise exc.NotAllowedException(
"Only ad-hoc action execution can be deleted.")
if not states.is_completed(action_ex.state):
raise exc.NotAllowedException(
"Only completed action execution can be deleted.")
return db_api.delete_action_execution(id)
def _find_next_tasks(self, task_ex, ctx):
t_n = task_ex.name
t_s = task_ex.state
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex), ctx,
data_flow.get_workflow_environment_dict(self.wf_ex),
self.wf_ex.context, self.wf_ex.input)
# [(task_name, params, 'on-success'|'on-error'|'on-complete'), ...]
result = []
if t_s == states.ERROR:
for name, cond, params in self.wf_spec.get_on_error_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-error'))
if t_s == states.SUCCESS:
for name, cond, params in self.wf_spec.get_on_success_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-success'))
if states.is_completed(t_s) and not states.is_cancelled(t_s):
for name, cond, params in self.wf_spec.get_on_complete_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-complete'))
return result
def dispatch_workflow_commands(wf_ex, wf_cmds):
# TODO(rakhmerov): I don't like these imports but otherwise we have
# import cycles.
from mistral.engine import task_handler
from mistral.engine import workflow_handler as wf_handler
if not wf_cmds:
return
for cmd in wf_cmds:
if isinstance(cmd, (commands.RunTask, commands.RunExistingTask)):
task_handler.run_task(cmd)
elif isinstance(cmd, commands.SetWorkflowState):
# TODO(rakhmerov): Make just a single call to workflow_handler
if states.is_completed(cmd.new_state):
wf_handler.stop_workflow(cmd.wf_ex, cmd.new_state, cmd.msg)
else:
wf_handler.set_workflow_state(wf_ex, cmd.new_state, cmd.msg)
elif isinstance(cmd, commands.Noop):
# Do nothing.
pass
else:
raise exc.MistralError('Unsupported workflow command: %s' % cmd)
if wf_ex.state != states.RUNNING:
break
def find_task_executions_by_name(wf_ex_id, task_name):
"""Finds task executions by workflow execution id and task name.
:param wf_ex_id: Workflow execution id.
:param task_name: Task name.
:return: Task executions (possibly a cached value).
"""
with _CACHE_LOCK:
t_execs = _TASK_EX_CACHE[wf_ex_id].get(task_name)
if t_execs:
return t_execs
t_execs = db_api.get_task_executions(workflow_execution_id=wf_ex_id,
name=task_name)
# We can cache only finished tasks because they won't change.
all_finished = (t_execs and all(
[states.is_completed(t_ex.state) for t_ex in t_execs]))
if all_finished:
with _CACHE_LOCK:
_TASK_EX_CACHE[wf_ex_id][task_name] = t_execs
return t_execs
def dispatch_workflow_commands(wf_ex, wf_cmds):
# TODO(rakhmerov): I don't like these imports but otherwise we have
# import cycles.
from mistral.engine import task_handler
from mistral.engine import workflow_handler as wf_handler
if not wf_cmds:
return
for cmd in wf_cmds:
if isinstance(cmd, (commands.RunTask, commands.RunExistingTask)):
task_handler.run_task(cmd)
elif isinstance(cmd, commands.SetWorkflowState):
# TODO(rakhmerov): Make just a single call to workflow_handler
if states.is_completed(cmd.new_state):
wf_handler.stop_workflow(cmd.wf_ex, cmd.new_state, cmd.msg)
else:
wf_handler.set_workflow_state(wf_ex, cmd.new_state, cmd.msg)
elif isinstance(cmd, commands.Noop):
# Do nothing.
pass
else:
raise exc.MistralError('Unsupported workflow command: %s' % cmd)
if wf_ex.state != states.RUNNING:
break
def find_task_executions_by_name(wf_ex_id, task_name):
"""Finds task executions by workflow execution id and task name.
:param wf_ex_id: Workflow execution id.
:param task_name: Task name.
:return: Task executions (possibly a cached value). The returned list
may contain task execution clones not bound to the DB session.
"""
with _TASK_EX_CACHE_LOCK:
t_execs = _TASK_EX_CACHE[wf_ex_id].get(task_name)
if t_execs:
return t_execs
t_execs = db_api.get_task_executions(
workflow_execution_id=wf_ex_id,
name=task_name,
sort_keys=[] # disable sorting
)
t_execs = [t_ex.get_clone() for t_ex in t_execs]
# We can cache only finished tasks because they won't change.
all_finished = (t_execs and all(
[states.is_completed(t_ex.state) for t_ex in t_execs]))
if all_finished:
with _TASK_EX_CACHE_LOCK:
_TASK_EX_CACHE[wf_ex_id][task_name] = t_execs
return t_execs
def put(self, id, action_ex):
"""Update the specified action_execution.
:param id: UUID of action execution to update
:param action_ex: Action execution for update
"""
acl.enforce('action_executions:update', context.ctx())
LOG.debug("Update action_execution [id=%s, action_execution=%s]", id,
action_ex)
if action_ex.state not in SUPPORTED_TRANSITION_STATES:
raise exc.InvalidResultException(
"Error. Expected one of %s, actual: %s" %
(SUPPORTED_TRANSITION_STATES, action_ex.state))
if states.is_completed(action_ex.state):
output = action_ex.output
if action_ex.state == states.SUCCESS:
result = ml_actions.Result(data=output)
elif action_ex.state == states.ERROR:
if not output:
output = 'Unknown error'
result = ml_actions.Result(error=output)
elif action_ex.state == states.CANCELLED:
result = ml_actions.Result(cancel=True)
values = rpc.get_engine_client().on_action_complete(id, result)
if action_ex.state in [states.PAUSED, states.RUNNING]:
state = action_ex.state
values = rpc.get_engine_client().on_action_update(id, state)
return resources.ActionExecution.from_dict(values)
def _find_next_task_names(self, task_ex):
t_state = task_ex.state
t_name = task_ex.name
ctx = data_flow.evaluate_task_outbound_context(task_ex)
t_names = []
if states.is_completed(t_state):
t_names += self._find_next_task_names_for_clause(
self.wf_spec.get_on_complete_clause(t_name),
ctx
)
if t_state == states.ERROR:
t_names += self._find_next_task_names_for_clause(
self.wf_spec.get_on_error_clause(t_name),
ctx
)
elif t_state == states.SUCCESS:
t_names += self._find_next_task_names_for_clause(
self.wf_spec.get_on_success_clause(t_name),
ctx
)
return t_names
def _find_next_tasks(self, task_ex):
t_state = task_ex.state
t_name = task_ex.name
ctx = data_flow.evaluate_task_outbound_context(task_ex)
t_names_and_params = []
if states.is_completed(t_state):
t_names_and_params += (
self._find_next_tasks_for_clause(
self.wf_spec.get_on_complete_clause(t_name),
ctx
)
)
if t_state == states.ERROR:
t_names_and_params += (
self._find_next_tasks_for_clause(
self.wf_spec.get_on_error_clause(t_name),
ctx
)
)
elif t_state == states.SUCCESS:
t_names_and_params += (
self._find_next_tasks_for_clause(
self.wf_spec.get_on_success_clause(t_name),
ctx
)
)
return t_names_and_params
def _get_induced_join_state(self, in_task_spec, in_task_ex, join_task_spec,
t_execs_cache):
join_task_name = join_task_spec.get_name()
if not in_task_ex:
possible, depth = self._possible_route(in_task_spec, t_execs_cache)
if possible:
return states.WAITING, depth, None
else:
return states.ERROR, depth, 'impossible route'
if not states.is_completed(in_task_ex.state):
return states.WAITING, 1, None
if self._is_conditional_transition(in_task_ex, in_task_spec) and \
not hasattr(in_task_ex, "in_context"):
in_task_ex = db_api.get_task_execution(in_task_ex.id)
# [(task name, params, event name), ...]
next_tasks_tuples = self._find_next_tasks(in_task_ex)
next_tasks_dict = {tup[0]: tup[2] for tup in next_tasks_tuples}
if join_task_name not in next_tasks_dict:
return states.ERROR, 1, "not triggered"
return states.RUNNING, 1, next_tasks_dict[join_task_name]
def _continue_workflow(self, wf_ex, task_ex=None, reset=True):
wf_handler.set_execution_state(wf_ex, states.RUNNING)
wf_ctrl = wf_base.WorkflowController.get_controller(wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow(task_ex=task_ex, reset=reset)
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was pause.
cmds = filter(
lambda c: not isinstance(c, commands.PauseWorkflow),
cmds
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
self._dispatch_workflow_commands(wf_ex, cmds)
if not cmds:
if not wf_utils.find_incomplete_task_executions(wf_ex):
wf_handler.succeed_workflow(
wf_ex,
wf_ctrl.evaluate_workflow_final_context()
)
return wf_ex.get_clone()
def _on_task_complete(task_ex_id):
# Note: This method can only be called via scheduler.
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
wf_ex = task_ex.workflow_execution
wf = workflows.Workflow(db_api.get_workflow_definition(
wf_ex.workflow_id),
wf_ex=wf_ex)
try:
wf.on_task_complete(task_ex)
except exc.MistralException as e:
msg = ("Failed to handle task completion [wf_ex=%s, task_ex=%s]:"
" %s\n%s" % (wf_ex, task_ex, e, tb.format_exc()))
LOG.error(msg)
fail_workflow(wf.wf_ex, msg)
return
if not states.is_completed(wf_ex.state):
# TODO(rakhmerov): Moving forward we can implement some more fancy
# algorithm for increasing delay for rescheduling so that we don't
# put too serious load onto scheduler.
delay = 1
schedule_on_task_complete(task_ex, delay)
def _possible_route(self, task_spec, t_execs_cache, depth=1):
in_task_specs = self.wf_spec.find_inbound_task_specs(task_spec)
if not in_task_specs:
return True, depth
for t_s in in_task_specs:
if t_s.get_name() not in t_execs_cache:
t_execs_cache.update(
self._prepare_task_executions_cache(task_spec))
t_ex = t_execs_cache.get(t_s.get_name())
if not t_ex:
possible, depth = self._possible_route(t_s, t_execs_cache,
depth + 1)
if possible:
return True, depth
else:
t_name = task_spec.get_name()
if not states.is_completed(t_ex.state):
return True, depth
if t_name in [t[0] for t in t_ex.next_tasks]:
return True, depth
return False, depth
def _get_induced_join_state(self, in_task_spec, in_task_ex,
join_task_spec, t_execs_cache):
join_task_name = join_task_spec.get_name()
if not in_task_ex:
possible, depth = self._possible_route(
in_task_spec,
t_execs_cache
)
if possible:
return states.WAITING, depth, None
else:
return states.ERROR, depth, 'impossible route'
if not states.is_completed(in_task_ex.state):
return states.WAITING, 1, None
if self._is_conditional_transition(in_task_ex, in_task_spec) and \
not hasattr(in_task_ex, "in_context"):
in_task_ex = db_api.get_task_execution(in_task_ex.id)
# [(task name, params, event name), ...]
next_tasks_tuples = self._find_next_tasks(in_task_ex)
next_tasks_dict = {tup[0]: tup[2] for tup in next_tasks_tuples}
if join_task_name not in next_tasks_dict:
return states.ERROR, 1, "not triggered"
return states.RUNNING, 1, next_tasks_dict[join_task_name]
def check_and_fix_integrity(wf_ex):
check_after_seconds = CONF.engine.execution_integrity_check_delay
if check_after_seconds < 0:
# Never check integrity if it's a negative value.
return
# To break cyclic dependency.
from mistral.engine import task_handler
running_task_execs = db_api.get_task_executions(
workflow_execution_id=wf_ex.id,
state=states.RUNNING
)
for t_ex in running_task_execs:
# The idea is that we take the latest known timestamp of the task
# execution and consider it eligible for checking and fixing only
# if some minimum period of time elapsed since the last update.
timestamp = t_ex.updated_at or t_ex.created_at
delta = timeutils.delta_seconds(timestamp, timeutils.utcnow())
if delta < check_after_seconds:
continue
child_executions = t_ex.executions
if not child_executions:
continue
all_finished = all(
[states.is_completed(c_ex.state) for c_ex in child_executions]
)
if all_finished:
# Find the timestamp of the most recently finished child.
most_recent_child_timestamp = max(
[c_ex.updated_at or c_ex.created_at for c_ex in
child_executions]
)
interval = timeutils.delta_seconds(
most_recent_child_timestamp,
timeutils.utcnow()
)
if interval > check_after_seconds:
# We found a task execution in RUNNING state for which all
# child executions are finished. We need to call
# "schedule_on_action_complete" on the task handler for any of
# the child executions so that the task state is calculated and
# updated properly.
LOG.warning(
"Found a task execution that is likely stuck in RUNNING"
" state because all child executions are finished,"
" will try to recover [task_execution=%s]", t_ex.id
)
task_handler.schedule_on_action_complete(child_executions[-1])
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = "retry_task_policy"
runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause, data_flow.evaluate_task_outbound_context(task_ex)
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if "retry_no" in policy_context:
retry_no = policy_context["retry_no"]
del policy_context["retry_no"]
retries_remain = retry_no + 1 < self.count
stop_continue_flag = task_ex.state == states.SUCCESS and not self._continue_on_clause
stop_continue_flag = stop_continue_flag or (self._continue_on_clause and not continue_on_evaluation)
break_triggered = task_ex.state == states.ERROR and self.break_on
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.DELAYED
policy_context["retry_no"] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)
def set_workflow_state(wf_ex, state, msg=None):
if states.is_completed(state):
stop_workflow(wf_ex, state, msg)
elif states.is_paused(state):
pause_workflow(wf_ex, msg)
else:
raise exc.MistralError('Invalid workflow state [wf_ex=%s, state=%s]' %
(wf_ex, state))
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.load_task_execution(task_ex_id)
if not task_ex:
return
wf_ex = task_ex.workflow_execution
if states.is_completed(wf_ex.state):
return
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id)
wf_ctrl = wf_base.get_controller(wf_ex, wf_spec)
with db_api.named_lock(task_ex.id):
db_api.refresh(task_ex)
if (states.is_completed(task_ex.state)
or task_ex.state == states.RUNNING):
return
log_state = wf_ctrl.get_logical_task_state(task_ex)
state = log_state.state
state_info = log_state.state_info
# Update 'triggered_by' because it could have changed.
task_ex.runtime_context['triggered_by'] = log_state.triggered_by
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
complete_task(task_ex, state, state_info)
elif state == states.WAITING:
LOG.info(
"Task execution is still in WAITING state"
" [task_ex_id=%s, task_name=%s]", task_ex_id, task_ex.name)
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex_id=%s, '
'task_name=%s, state=%s]' %
(task_ex_id, task_ex.name, state))
def _get_unaccepted_act_exs(task_ex):
# Choose only if not accepted but completed.
return list(
filter(
lambda x: not x.accepted and states.is_completed(x.state),
task_ex.executions
)
)
def _is_upstream_task_execution(self, t_spec, t_ex_candidate):
if not states.is_completed(t_ex_candidate.state):
return False
if not t_spec.get_join():
return not t_ex_candidate.processed
return self._triggers_join(t_spec, self.wf_spec.get_tasks()[t_ex_candidate.name])
def on_action_complete(action_ex, wf_spec, result):
"""Handles event of action result arrival.
Given action result this method changes corresponding task execution
object. This method must never be called for the case of individual
action which is not associated with any tasks.
:param action_ex: Action execution objects the result belongs to.
:param wf_spec: Workflow specification.
:param result: Task action/workflow output wrapped into
mistral.workflow.utils.Result instance.
:return Task execution object.
"""
task_ex = action_ex.task_execution
# Ignore if action already completed.
if (states.is_completed(action_ex.state)
and not isinstance(action_ex, models.WorkflowExecution)):
return task_ex
task_spec = wf_spec.get_tasks()[task_ex.name]
try:
result = action_handler.transform_result(result, task_ex, task_spec)
except exc.YaqlEvaluationException as e:
err_msg = str(e)
LOG.error(
'YAQL error while transforming action result'
' [action_execution_id=%s, result=%s]: %s', action_ex.id, result,
err_msg)
result = wf_utils.Result(error=err_msg)
# Ignore workflow executions because they're handled during
# workflow completion.
if not isinstance(action_ex, models.WorkflowExecution):
action_handler.store_action_result(action_ex, result)
if result.is_success():
task_state = states.SUCCESS
task_state_info = None
else:
task_state = states.ERROR
task_state_info = result.error
if not task_spec.get_with_items():
_complete_task(task_ex, task_spec, task_state, task_state_info)
else:
with_items.increase_capacity(task_ex)
if with_items.is_completed(task_ex):
_complete_task(task_ex, task_spec,
with_items.get_final_state(task_ex),
task_state_info)
return task_ex
def set_workflow_state(wf_ex, state, msg=None):
if states.is_completed(state):
stop_workflow(wf_ex, state, msg)
elif states.is_paused(state):
pause_workflow(wf_ex, msg)
else:
raise exc.MistralError(
'Invalid workflow execution state [wf_ex_id=%s, wf_name=%s, '
'state=%s]' % (wf_ex.id, wf_ex.name, state))
def set_workflow_state(wf_ex, state, msg=None):
if states.is_completed(state):
stop_workflow(wf_ex, state, msg)
elif states.is_paused(state):
pause_workflow(wf_ex, msg)
else:
raise exc.MistralError(
'Invalid workflow state [wf_ex=%s, state=%s]' % (wf_ex, state)
)
def set_state(self, state, state_info=None):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(
id=self.wf_ex.id,
cur_state=cur_state,
state=state
)
if wf_ex is None:
# Do nothing because the state was updated previously.
return False
self.wf_ex = wf_ex
self.wf_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
wf_trace.info(
self.wf_ex,
"Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name,
cur_state,
state,
self.wf_ex.state_info)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
triggers.on_workflow_complete(self.wf_ex)
return True
def _is_upstream_task_execution(self, t_spec, t_ex_candidate):
if not states.is_completed(t_ex_candidate.state):
return False
if not t_spec.get_join():
return not t_ex_candidate.processed
return self._triggers_join(
t_spec,
self.wf_spec.get_tasks()[t_ex_candidate.name])
def _fail_task_if_incomplete(task_ex_id, timeout):
from mistral.engine import task_handler
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = 'Task timed out [timeout(s)=%s].' % timeout
task_handler.complete_task(task_ex, states.ERROR, msg)
def set_workflow_state(wf_ex, state, msg=None):
if states.is_completed(state):
stop_workflow(wf_ex, state, msg)
elif states.is_paused(state):
pause_workflow(wf_ex, msg)
else:
raise exc.MistralError(
'Invalid workflow execution state [wf_ex_id=%s, wf_name=%s, '
'state=%s]' % (wf_ex.id, wf_ex.name, state)
)
def _fail_task_if_incomplete(task_ex_id, timeout):
from mistral.engine import task_handler
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = 'Task timed out [timeout(s)=%s].' % timeout
task_handler.complete_task(task_ex, states.ERROR, msg)
def set_state(self, state, state_info=None):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(
id=self.wf_ex.id,
cur_state=cur_state,
state=state
)
if wf_ex is None:
# Do nothing because the state was updated previously.
return False
self.wf_ex = wf_ex
self.wf_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
wf_trace.info(
self.wf_ex,
"Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name,
cur_state,
state,
self.wf_ex.state_info)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
triggers.on_workflow_complete(self.wf_ex)
return True
def _triggers_join(self, join_task_spec, inbound_task_spec):
in_t_ex = wf_utils.find_task_execution(self.wf_ex, inbound_task_spec)
if not in_t_ex or not states.is_completed(in_t_ex.state):
return False
return filter(
lambda t_name: join_task_spec.get_name() == t_name,
self._find_next_task_names(
in_t_ex, data_flow.evaluate_task_outbound_context(in_t_ex)))
def _triggers_join(self, join_task_spec, inbound_task_spec):
in_t_ex = wf_utils.find_task_execution(self.wf_ex, inbound_task_spec)
if not in_t_ex or not states.is_completed(in_t_ex.state):
return False
return filter(
lambda t_name: join_task_spec.get_name() == t_name,
self._find_next_task_names(in_t_ex, data_flow.evaluate_task_outbound_context(in_t_ex)),
)
def fail_task_if_incomplete(task_ex_id, timeout):
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = "Task timed out [id=%s, timeout(s)=%s]." % (task_ex_id, timeout)
wf_trace.info(task_ex, msg)
wf_trace.info(task_ex, "Task '%s' [%s -> ERROR]" % (task_ex.name, task_ex.state))
rpc.get_engine_client().on_task_state_change(task_ex_id, states.ERROR)
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
:return: True if the state was changed as a result of this call,
False otherwise.
"""
assert self.task_ex
cur_state = self.task_ex.state
if cur_state != state or self.task_ex.state_info != state_info:
task_ex = db_api.update_task_execution_state(id=self.task_ex.id,
cur_state=cur_state,
state=state)
if task_ex is None:
# Do nothing because the update query did not change the DB.
return False
self.task_ex = task_ex
self.task_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
self.state_changed = True
# Recalculating "started_at" timestamp only if the state
# was WAITING (all preconditions are satisfied and it's
# ready to start) or IDLE, or the task is being rerun. So
# we treat all iterations of "retry" policy as one run.
if state == states.RUNNING and \
(cur_state in (None, states.WAITING) or self.rerun):
self.task_ex.started_at = utils.utc_now_sec()
if states.is_completed(state):
self.task_ex.finished_at = utils.utc_now_sec()
if self.rerun:
self.task_ex.finished_at = None
if processed is not None:
self.task_ex.processed = processed
self._notify(cur_state, state)
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name, self.task_ex.id, cur_state, state,
self.task_ex.state_info))
return True
def _check_and_complete(wf_ex_id):
# Note: This method can only be called via scheduler.
with db_api.transaction():
wf_ex = db_api.load_workflow_execution(wf_ex_id)
if not wf_ex or states.is_completed(wf_ex.state):
return
wf = workflows.Workflow(
db_api.get_workflow_definition(wf_ex.workflow_id),
wf_ex=wf_ex
)
try:
incomplete_tasks_count = wf.check_and_complete()
except exc.MistralException as e:
msg = (
"Failed to check and complete [wf_ex=%s]:"
" %s\n%s" % (wf_ex, e, tb.format_exc())
)
LOG.error(msg)
force_fail_workflow(wf.wf_ex, msg)
return
if not states.is_completed(wf_ex.state):
# Let's assume that a task takes 0.01 sec in average to complete
# and based on this assumption calculate a time of the next check.
# The estimation is very rough but this delay will be decreasing
# as tasks will be completing which will give a decent
# approximation.
# For example, if a workflow has 100 incomplete tasks then the
# next check call will happen in 10 seconds. For 500 tasks it will
# be 50 seconds. The larger the workflow is, the more beneficial
# this mechanism will be.
delay = int(incomplete_tasks_count * 0.01)
_schedule_check_and_complete(wf_ex, delay)
def update(self, state, state_info=None):
"""Update task and set specified state.
Method sets specified task state.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Record the current task state.
old_task_state = self.task_ex.state
# Ignore if task already completed.
if states.is_completed(self.task_ex.state):
# Publish task event again so subscribers know
# task completed state is being processed again.
self.notify(old_task_state, self.task_ex.state)
return
# Update only if state transition is valid.
if not states.is_valid_transition(self.task_ex.state, state):
return
# We can't set the task state to RUNNING if some other
# child executions are paused.
child_states = [a_ex.state for a_ex in self.task_ex.executions]
if state == states.RUNNING and states.PAUSED in child_states:
return
self.set_state(state, state_info)
if states.is_completed(self.task_ex.state):
self.register_workflow_completion_check()
# Publish event.
self.notify(old_task_state, self.task_ex.state)
def _is_upstream_task_execution(self, t_spec, t_ex_candidate):
if not states.is_completed(t_ex_candidate.state):
return False
if not t_spec.get_join():
return t_ex_candidate.processed
induced_state, _ = self._get_induced_join_state(
self.wf_spec.get_tasks()[t_ex_candidate.name],
t_spec
)
return induced_state == states.RUNNING
def resume_workflow(self, execution_id):
try:
with db_api.transaction():
# Must be before loading the object itself (see method doc).
self._lock_workflow_execution(execution_id)
wf_ex = db_api.get_workflow_execution(execution_id)
if wf_ex.state != states.PAUSED:
return
wf_handler.set_execution_state(wf_ex, states.RUNNING)
wf_ctrl = wf_base.WorkflowController.get_controller(wf_ex)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow()
# When resuming a workflow we need to ignore all 'pause'
# commands because workflow controller takes tasks that
# completed within the period when the workflow was pause.
cmds = filter(
lambda c: not isinstance(c, commands.PauseWorkflow),
cmds
)
# Since there's no explicit task causing the operation
# we need to mark all not processed tasks as processed
# because workflow controller takes only completed tasks
# with flag 'processed' equal to False.
for t_ex in wf_ex.task_executions:
if states.is_completed(t_ex.state) and not t_ex.processed:
t_ex.processed = True
self._dispatch_workflow_commands(wf_ex, cmds)
if not cmds:
if not wf_utils.find_incomplete_tasks(wf_ex):
wf_handler.succeed_workflow(
wf_ex,
wf_ctrl.evaluate_workflow_final_context()
)
return wf_ex
except Exception as e:
LOG.error(
"Failed to resume execution id=%s: %s\n%s",
execution_id, e, traceback.format_exc()
)
self._fail_workflow(execution_id, e)
raise e
def complete(self, result):
assert self.action_ex
if states.is_completed(self.action_ex.state):
return
prev_state = self.action_ex.state
self.action_ex.state = (states.SUCCESS if result.is_success()
else states.ERROR)
self.action_ex.output = self._prepare_output(result)
self.action_ex.accepted = True
self._log_result(prev_state, result)
def _find_next_commands(self):
cmds = super(DirectWorkflowController, self)._find_next_commands()
if not self.wf_ex.task_executions:
return self._find_start_commands()
task_execs = [
t_ex for t_ex in self.wf_ex.task_executions if states.is_completed(t_ex.state) and not t_ex.processed
]
for t_ex in task_execs:
cmds.extend(self._find_next_commands_for_task(t_ex))
return cmds
def _complete_task(task_ex, task_spec, state):
# Ignore if task already completed.
if states.is_completed(task_ex.state):
return []
_set_task_state(task_ex, state)
data_flow.publish_variables(
task_ex,
task_spec
)
if not task_spec.get_keep_result():
data_flow.destroy_task_result(task_ex)
def test_resume_different_task_states(self):
wb_service.create_workbook_v2(WORKBOOK_DIFFERENT_TASK_STATES)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
self.await_workflow_paused(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.PAUSED, wf_ex.state)
self.assertEqual(3, len(task_execs))
task2_ex = self._assert_single_item(task_execs, name='task2')
# Task2 is not finished yet.
self.assertFalse(states.is_completed(task2_ex.state))
wf_ex = self.engine.resume_workflow(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
# Wait for task3 to be processed.
task3_ex = self._assert_single_item(task_execs, name='task3')
self.await_task_success(task3_ex.id)
self.await_task_processed(task3_ex.id)
# Finish task2.
task2_action_ex = db_api.get_action_executions(
task_execution_id=task2_ex.id
)[0]
self.engine.on_action_complete(task2_action_ex.id, utils.Result())
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(states.SUCCESS, wf_ex.state, wf_ex.state_info)
self.assertEqual(4, len(task_execs))
def complete(self, state, state_info=None):
"""Complete task and set specified state.
Method sets specified task state and runs all necessary post
completion logic such as publishing workflow variables and
scheduling new workflow commands.
:param state: New task state.
:param state_info: New state information (i.e. error message).
"""
assert self.task_ex
# Ignore if task already completed.
if states.is_completed(self.task_ex.state):
return
self.set_state(state, state_info)
data_flow.publish_variables(self.task_ex, self.task_spec)
if not self.task_spec.get_keep_result():
# Destroy task result.
for ex in self.task_ex.action_executions:
if hasattr(ex, 'output'):
ex.output = {}
self._after_task_complete()
# Ignore DELAYED state.
if self.task_ex.state == states.RUNNING_DELAYED:
return
# If workflow is paused we shouldn't schedule new commands
# and mark task as processed.
if states.is_paused(self.wf_ex.state):
return
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
# Calculate commands to process next.
cmds = wf_ctrl.continue_workflow()
# Mark task as processed after all decisions have been made
# upon its completion.
self.task_ex.processed = True
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def pause_workflow(wf_ex, msg=None):
# Pause subworkflows first.
for task_ex in wf_ex.task_executions:
sub_wf_exs = db_api.get_workflow_executions(
task_execution_id=task_ex.id
)
for sub_wf_ex in sub_wf_exs:
if not states.is_completed(sub_wf_ex.state):
pause_workflow(sub_wf_ex, msg=msg)
# If all subworkflows paused successfully, pause the main workflow.
# If any subworkflows failed to pause for temporary reason, this
# allows pause to be executed again on the main workflow.
wf = workflows.Workflow(wf_ex=wf_ex)
wf.pause(msg=msg)
