def on_task_state_change(self, task_ex_id, state):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked. But by the moment this method is called the
# last execution object may have changed. It's a race condition.
execution = task_ex.executions[-1]
wf_ex_id = task_ex.workflow_execution_id
# Must be before loading the object itself (see method doc).
self._lock_workflow_execution(wf_ex_id)
wf_ex = task_ex.workflow_execution
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, task_ex.state, state)
)
task_ex.state = state
self._on_task_state_change(task_ex, wf_ex, action_ex=execution)
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
"""
assert self.task_ex
if (self.task_ex.state != state or
self.task_ex.state_info != state_info):
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name,
self.task_ex.id,
self.task_ex.state,
state,
state_info)
)
self.state_changed = True
self.task_ex.state = state
self.task_ex.state_info = state_info
if processed is not None:
self.task_ex.processed = processed
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = "wait_before_policy"
runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get("skip"):
# Unset state 'DELAYED'.
wf_trace.info(task_ex, "Task '%s' [%s -> %s]" % (task_ex.name, states.DELAYED, states.RUNNING))
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({"skip": True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)
def start(self, input_dict, desc='', params=None):
"""Start workflow.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
"""
assert not self.wf_ex
wf_trace.info(self.wf_ex, "Starting workflow: %s" % self.wf_def)
# TODO(rakhmerov): This call implicitly changes input_dict! Fix it!
# After fix we need to move validation after adding risky fields.
eng_utils.validate_input(self.wf_def, input_dict, self.wf_spec)
self._create_execution(input_dict, desc, params)
self.set_state(states.RUNNING)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
cmds = wf_ctrl.continue_workflow()
dispatcher.dispatch_workflow_commands(self.wf_ex, cmds)
def _log_action_result(action_ex, from_state, to_state, result):
def _result_msg():
if action_ex.state == states.ERROR:
return "error = %s" % utils.cut(result.error)
return "result = %s" % utils.cut(result.data)
wf_trace.info(None, "Action execution '%s' [%s -> %s, %s]" % (action_ex.name, from_state, to_state, _result_msg()))
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
No need to move to next iteration.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if state != states.ERROR:
return
wf_trace.info(
task_ex,
"Task '%s' [%s -> ERROR]"
% (task_ex.name, task_ex.state)
)
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
if not retries_remain or self.break_on:
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def _set_task_state(task_ex, state):
# TODO(rakhmerov): How do we log task result?
wf_trace.info(
task_ex.workflow_execution,
"Task execution '%s' [%s -> %s]" %
(task_ex.name, task_ex.state, state)
)
task_ex.state = state
def fail_task_if_incomplete(task_ex_id, timeout):
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = "Task timed out [id=%s, timeout(s)=%s]." % (task_ex_id, timeout)
wf_trace.info(task_ex, msg)
wf_trace.info(task_ex, "Task '%s' [%s -> ERROR]" % (task_ex.name, task_ex.state))
rpc.get_engine_client().on_task_state_change(task_ex_id, states.ERROR)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
scheduler.schedule_call(
None,
"mistral.engine.policies.fail_task_if_incomplete",
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay,
)
wf_trace.info(task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." % (task_ex.id, self.delay))
def set_task_state(task_ex, state, state_info, processed=None):
wf_trace.info(
task_ex.workflow_execution,
"Task execution '%s' [%s -> %s]" %
(task_ex.name, task_ex.state, state)
)
task_ex.state = state
task_ex.state_info = state_info
if processed is not None:
task_ex.processed = processed
def before_task_start(self, task_ex, task_spec):
super(PauseBeforePolicy, self).before_task_start(task_ex, task_spec)
if not self.expr:
return
wf_trace.info(
task_ex,
"Workflow paused before task '%s' [%s -> %s]" %
(task_ex.name, task_ex.workflow_execution.state, states.PAUSED)
)
task_ex.workflow_execution.state = states.PAUSED
task_ex.state = states.IDLE
def _log_result(self, prev_state, result):
state = self.action_ex.state
def _result_msg():
if state == states.ERROR:
return "error = %s" % utils.cut(result.error)
return "result = %s" % utils.cut(result.data)
wf_trace.info(
None,
"Action execution '%s' [%s -> %s, %s]" %
(self.action_ex.name, prev_state, state, _result_msg())
)
def _log_result(self, prev_state, result):
state = self.action_ex.state
if prev_state != state:
wf_trace.info(
None,
"Action '%s' (%s)(task=%s) [%s -> %s, %s]" %
(self.action_ex.name,
self.action_ex.id,
self.task_ex.name if self.task_ex else None,
prev_state,
state,
result.cut_repr())
)
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
:return: True if the state was changed as a result of this call,
False otherwise.
"""
assert self.task_ex
cur_state = self.task_ex.state
# Set initial started_at in case of waiting => running.
# We can't set this just in run_existing, because task retries
# will update started_at, which is incorrect.
if cur_state == states.WAITING and state == states.RUNNING:
self.save_started_time()
if cur_state != state or self.task_ex.state_info != state_info:
task_ex = db_api.update_task_execution_state(
id=self.task_ex.id,
cur_state=cur_state,
state=state
)
if task_ex is None:
# Do nothing because the update query did not change the DB.
return False
self.task_ex = task_ex
self.task_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
self.state_changed = True
if processed is not None:
self.task_ex.processed = processed
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name,
self.task_ex.id,
cur_state,
state,
self.task_ex.state_info)
)
return True
def before_task_start(self, task_ex, task_spec):
super(PauseBeforePolicy, self).before_task_start(task_ex, task_spec)
if not self.expr:
return
wf_trace.info(
task_ex,
"Workflow paused before task '%s' [%s -> %s]" %
(task_ex.name, task_ex.workflow_execution.state, states.PAUSED)
)
task_ex.state = states.IDLE
wf_handler.pause_workflow(task_ex.workflow_execution)
def _log_result(self, prev_state, result):
state = self.action_ex.state
def _result_msg():
if state == states.ERROR:
return "error = %s" % utils.cut(result.error)
return "result = %s" % utils.cut(result.data)
if prev_state != state:
wf_trace.info(
None, "Action '%s' (%s)(task=%s) [%s -> %s, %s]" %
(self.action_ex.name, self.action_ex.id, self.task_ex.name
if self.task_ex else None, prev_state, state, _result_msg()))
def _log_result(self, prev_state, result):
state = self.action_ex.state
def _result_msg():
if state == states.ERROR:
return "error = %s" % utils.cut(result.error)
return "result = %s" % utils.cut(result.data)
wf_trace.info(
None,
"Action execution '%s' [%s -> %s, %s]" %
(self.action_ex.name, prev_state, state, _result_msg())
)
def _run_action_or_workflow(task_ex, task_spec, input_dict, index, wf_spec):
t_name = task_ex.name
if task_spec.get_action_name():
wf_trace.info(
task_ex, "Task '%s' is RUNNING [action_name = %s]" %
(t_name, task_spec.get_action_name()))
_schedule_run_action(task_ex, task_spec, input_dict, index, wf_spec)
elif task_spec.get_workflow_name():
wf_trace.info(
task_ex, "Task '%s' is RUNNING [workflow_name = %s]" %
(t_name, task_spec.get_workflow_name()))
_schedule_run_workflow(task_ex, task_spec, input_dict, index, wf_spec)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
# No need to wait for a task if delay is 0
if self.delay == 0:
return
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'RUNNING_DELAYED'.
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, states.RUNNING_DELAYED, states.RUNNING)
)
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
sched = sched_base.get_system_scheduler()
job = sched_base.SchedulerJob(
run_after=self.delay,
func_name=_CONTINUE_TASK_PATH,
func_args={
'task_ex_id': task_ex.id
}
)
sched.schedule(job)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
scheduler.schedule_call(
None,
'mistral.engine.policies._fail_task_if_incomplete',
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay
)
wf_trace.info(
task_ex,
"Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay)
)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
# No timeout if delay is 0
if self.delay == 0:
return
scheduler.schedule_call(None,
_FAIL_IF_INCOMPLETE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay)
wf_trace.info(
task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay))
def start(self, wf_def, wf_ex_id, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param wf_ex_id: Workflow execution id.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
:raises
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
wf_trace.info(
self.wf_ex,
'Starting workflow [name=%s, input=%s]' %
(wf_def.name, utils.cut(input_dict))
)
self.validate_input(input_dict)
self._create_execution(
wf_def,
wf_ex_id,
self.prepare_input(input_dict),
desc,
params
)
self.set_state(states.RUNNING)
# Publish event as soon as state is set to running.
self.notify(events.WORKFLOW_LAUNCHED)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
dispatcher.dispatch_workflow_commands(
self.wf_ex,
wf_ctrl.continue_workflow()
)
def start(self, wf_def, wf_ex_id, input_dict, desc='', params=None):
"""Start workflow.
:param wf_def: Workflow definition.
:param wf_ex_id: Workflow execution id.
:param input_dict: Workflow input.
:param desc: Workflow execution description.
:param params: Workflow type specific parameters.
:raises
"""
assert not self.wf_ex
# New workflow execution.
self.wf_spec = spec_parser.get_workflow_spec_by_definition_id(
wf_def.id,
wf_def.updated_at
)
wf_trace.info(
self.wf_ex,
'Starting workflow [name=%s, input=%s]' %
(wf_def.name, utils.cut(input_dict))
)
self.validate_input(input_dict)
self._create_execution(
wf_def,
wf_ex_id,
self.prepare_input(input_dict),
desc,
params
)
self.set_state(states.RUNNING)
# Publish event as soon as state is set to running.
self.notify(events.WORKFLOW_LAUNCHED)
wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec)
dispatcher.dispatch_workflow_commands(
self.wf_ex,
wf_ctrl.continue_workflow()
)
def on_task_state_change(self, task_ex_id, state, state_info=None):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked.
wf_ex_id = task_ex.workflow_execution_id
wf_ex = wf_handler.lock_workflow_execution(wf_ex_id)
wf_trace.info(
task_ex, "Task '%s' [%s -> %s] state_info : %s" %
(task_ex.name, task_ex.state, state, state_info))
task_ex.state = state
task_ex.state_info = state_info
self._on_task_state_change(task_ex, wf_ex)
def _run_action_or_workflow(task_ex, task_spec, input_dict, index):
t_name = task_ex.name
if task_spec.get_action_name():
wf_trace.info(
task_ex,
"Task '%s' is RUNNING [action_name = %s]" %
(t_name, task_spec.get_action_name())
)
_schedule_run_action(task_ex, task_spec, input_dict, index)
elif task_spec.get_workflow_name():
wf_trace.info(
task_ex,
"Task '%s' is RUNNING [workflow_name = %s]" %
(t_name, task_spec.get_workflow_name()))
_schedule_run_workflow(task_ex, task_spec, input_dict, index)
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
:return: True if the state was changed as a result of this call,
False otherwise.
"""
assert self.task_ex
cur_state = self.task_ex.state
# Set initial started_at in case of waiting => running.
# We can't set this just in run_existing, because task retries
# will update started_at, which is incorrect.
if cur_state == states.WAITING and state == states.RUNNING:
self.save_started_time()
if cur_state != state or self.task_ex.state_info != state_info:
task_ex = db_api.update_task_execution_state(id=self.task_ex.id,
cur_state=cur_state,
state=state)
if task_ex is None:
# Do nothing because the update query did not change the DB.
return False
self.task_ex = task_ex
self.task_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
self.state_changed = True
if processed is not None:
self.task_ex.processed = processed
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name, self.task_ex.id, cur_state, state,
self.task_ex.state_info))
return True
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
:return: True if the state was changed as a result of this call,
False otherwise.
"""
assert self.task_ex
cur_state = self.task_ex.state
if cur_state != state or self.task_ex.state_info != state_info:
task_ex = db_api.update_task_execution_state(
id=self.task_ex.id,
cur_state=cur_state,
state=state
)
if task_ex is None:
# Do nothing because the update query did not change the DB.
return False
self.task_ex = task_ex
self.task_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
self.state_changed = True
if processed is not None:
self.task_ex.processed = processed
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name,
self.task_ex.id,
cur_state,
state,
self.task_ex.state_info)
)
return True
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
self.wf_ex.state = state
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex, "Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name, cur_state, state, state_info))
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
# No need to keep task executions of this workflow in the
# lookup cache anymore.
lookup_utils.invalidate_cached_task_executions(self.wf_ex.id)
triggers.on_workflow_complete(self.wf_ex)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id)
parent_wf = Workflow(wf_ex=parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
"""
wf_trace.info(
self.task_ex.workflow_execution,
"Task execution '%s' [%s -> %s]: %s" %
(self.task_ex.id, self.task_ex.state, state, state_info)
)
self.task_ex.state = state
self.task_ex.state_info = state_info
if processed is not None:
self.task_ex.processed = processed
def set_state(self, state, state_info=None):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(
id=self.wf_ex.id,
cur_state=cur_state,
state=state
)
if wf_ex is None:
# Do nothing because the state was updated previously.
return False
self.wf_ex = wf_ex
self.wf_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
wf_trace.info(
self.wf_ex,
"Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name,
cur_state,
state,
self.wf_ex.state_info)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
triggers.on_workflow_complete(self.wf_ex)
return True
def set_execution_state(wf_ex, state, state_info=None, set_upstream=False):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex,
"Execution of workflow '%s' [%s -> %s]"
% (wf_ex.workflow_name, cur_state, state)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
wf_ex.accepted = wf_ex.state in (states.SUCCESS, states.ERROR)
# If specified, then recursively set the state of the parent workflow
# executions to the same state. Only changing state to RUNNING is
# supported.
if set_upstream and state == states.RUNNING and wf_ex.task_execution_id:
task_ex = db_api.get_task_execution(wf_ex.task_execution_id)
parent_wf_ex = lock_workflow_execution(task_ex.workflow_execution_id)
set_execution_state(
parent_wf_ex,
state,
state_info=state_info,
set_upstream=set_upstream
)
task_handler.set_task_state(
task_ex,
state,
state_info=None,
processed=False
)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
# No need to wait for a task if delay is 0
if self.delay == 0:
return
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'RUNNING_DELAYED'.
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, states.RUNNING_DELAYED, states.RUNNING)
)
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def set_workflow_state(wf_ex, state, state_info=None, set_upstream=False):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex, "Execution of workflow '%s' [%s -> %s]" %
(wf_ex.workflow_name, cur_state, state))
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
wf_ex.accepted = wf_ex.state in (states.SUCCESS, states.ERROR)
# If specified, then recursively set the state of the parent workflow
# executions to the same state. Only changing state to RUNNING is
# supported.
# TODO(rakhmerov): I don't like this hardcoded special case. It's
# used only to continue the workflow (rerun) but at the first glance
# seems like a generic behavior. Need to handle it differently.
if set_upstream and state == states.RUNNING and wf_ex.task_execution_id:
task_ex = db_api.get_task_execution(wf_ex.task_execution_id)
parent_wf_ex = lock_workflow_execution(task_ex.workflow_execution_id)
set_workflow_state(parent_wf_ex,
state,
state_info=state_info,
set_upstream=set_upstream)
# TODO(rakhmerov): How do we need to set task state properly?
# It doesn't seem right to intervene into the parent workflow
# internals. We just need to communicate changes back to parent
# worklfow and it should do what's needed itself.
task_ex.state = state
task_ex.state_info = None
task_ex.processed = False
def create_workflow_execution(wf_identifier, wf_input, description, params):
params = canonize_workflow_params(params)
wf_def = db_api.get_workflow_definition(wf_identifier)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = _create_workflow_execution(
wf_def,
wf_spec,
wf_input,
description,
params
)
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_identifier)
return wf_ex.id
def set_state(self, state, state_info=None):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(
id=self.wf_ex.id,
cur_state=cur_state,
state=state
)
if wf_ex is None:
# Do nothing because the state was updated previously.
return False
self.wf_ex = wf_ex
self.wf_ex.state_info = json.dumps(state_info) \
if isinstance(state_info, dict) else state_info
wf_trace.info(
self.wf_ex,
"Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name,
cur_state,
state,
self.wf_ex.state_info)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
triggers.on_workflow_complete(self.wf_ex)
return True
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
self.wf_ex.state = state
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex,
"Execution of workflow '%s' [%s -> %s]"
% (self.wf_ex.workflow_name, cur_state, state)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = state in (states.SUCCESS, states.ERROR)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id
)
parent_wf = Workflow(
db_api.get_workflow_definition(parent_task_ex.workflow_id),
parent_task_ex.workflow_execution
)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
# No need to wait for a task if delay is 0
if self.delay == 0:
return
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'RUNNING_DELAYED'.
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, states.RUNNING_DELAYED, states.RUNNING)
)
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def set_execution_state(wf_ex, state, state_info=None):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex, "Execution of workflow '%s' [%s -> %s]" %
(wf_ex.workflow_name, cur_state, state))
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully.
wf_ex.accepted = wf_ex.state == states.SUCCESS
def before_task_start(self, task):
super(PauseBeforePolicy, self).before_task_start(task)
if not self.expr:
return
wf_trace.info(
task.task_ex,
"Workflow paused before task '%s' [%s -> %s]" %
(
task.get_name(),
task.wf_ex.state,
states.PAUSED
)
)
task.set_state(states.IDLE, "Set by 'pause-before' policy")
wf_handler.pause_workflow(task.wf_ex)
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
self.wf_ex.state = state
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex,
"Execution of workflow '%s' [%s -> %s]"
% (self.wf_ex.workflow_name, cur_state, state)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id
)
parent_wf = Workflow(
db_api.get_workflow_definition(parent_task_ex.workflow_id),
parent_task_ex.workflow_execution
)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def _log_result(self, prev_state, result):
state = self.action_ex.state
def _result_msg():
if state == states.ERROR:
return "error = %s" % utils.cut(result.error)
return "result = %s" % utils.cut(result.data)
if prev_state != state:
wf_trace.info(
None,
"Action '%s' (%s)(task=%s) [%s -> %s, %s]" %
(self.action_ex.name,
self.action_ex.id,
self.task_ex.name if self.task_ex else None,
prev_state,
state,
_result_msg())
)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
# No timeout if delay is 0
if self.delay == 0:
return
scheduler.schedule_call(
None,
_FAIL_IF_INCOMPLETE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay
)
wf_trace.info(
task_ex,
"Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay)
)
def create_workflow_execution(wf_identifier,
wf_input,
description,
params,
wf_spec=None):
params = canonize_workflow_params(params)
wf_def = db_api.get_workflow_definition(wf_identifier)
if wf_spec is None:
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = _create_workflow_execution(wf_def, wf_spec, wf_input, description,
params)
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_identifier)
return wf_ex, wf_spec
def start_workflow(self, wf_name, wf_input, description='', **params):
wf_exec_id = None
try:
params = self._canonize_workflow_params(params)
with db_api.transaction():
wf_def = db_api.get_workflow_definition(wf_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
eng_utils.validate_input(wf_def, wf_input, wf_spec)
wf_ex = self._create_workflow_execution(
wf_def,
wf_spec,
wf_input,
description,
params
)
wf_exec_id = wf_ex.id
wf_trace.info(wf_ex, "Starting workflow: '%s'" % wf_name)
wf_ctrl = wf_base.WorkflowController.get_controller(
wf_ex,
wf_spec
)
self._dispatch_workflow_commands(
wf_ex,
wf_ctrl.continue_workflow()
)
return wf_ex.get_clone()
except Exception as e:
LOG.error(
"Failed to start workflow '%s' id=%s: %s\n%s",
wf_name, wf_exec_id, e, traceback.format_exc()
)
self._fail_workflow(wf_exec_id, e)
raise e
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'RUNNING_DELAYED'.
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, states.RUNNING_DELAYED, states.RUNNING)
)
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
# TODO(rakhmerov): This is wrong as task handler doesn't manage
# transactions and hence it can't be called explicitly.
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def on_task_state_change(self, task_ex_id, state, state_info=None):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked.
wf_ex_id = task_ex.workflow_execution_id
wf_ex = wf_handler.lock_workflow_execution(wf_ex_id)
wf_spec = spec_parser.get_workflow_spec(wf_ex.spec)
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s] state_info : %s"
% (task_ex.name, task_ex.state, state, state_info)
)
task_ex.state = state
task_ex.state_info = state_info
self._on_task_state_change(task_ex, wf_ex, wf_spec)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
# No timeout if delay is 0
if self.delay == 0:
return
sched = sched_base.get_system_scheduler()
job = sched_base.SchedulerJob(run_after=self.delay,
func_name=_FAIL_IF_INCOMPLETE_TASK_PATH,
func_args={
'task_ex_id': task_ex.id,
'timeout': self.delay
})
sched.schedule(job)
wf_trace.info(
task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay))
def set_execution_state(wf_ex, state, state_info=None):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex,
"Execution of workflow '%s' [%s -> %s]"
% (wf_ex.workflow_name, cur_state, state)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully.
wf_ex.accepted = wf_ex.state == states.SUCCESS
def _run_new(self):
if self.waiting:
self.defer()
return
self._create_task_execution()
# Add state change log
# self.set_state(states.RUNNING, None, processed=False)
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name,
self.task_ex.id,
states.IDLE,
self.task_ex.state,
None)
)
LOG.debug(
'Starting task [workflow=%s, task=%s, init_state=%s]',
self.wf_ex.name,
self.task_spec.get_name(),
self.task_ex.state
)
self._before_task_start()
# Policies could possibly change task state.
if self.task_ex.state != states.RUNNING:
return
# Add Kafka log trace
kfk_trace.log(kfk_etypes.TASK_START, None, states.RUNNING,
self.wf_ex.workflow_id, self.wf_ex.id,
self.task_ex.id, self.task_ex.name,
None, None, None)
self._schedule_actions()
def on_task_state_change(self, task_ex_id, state):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked. But by the moment this method is called the
# last execution object may have changed. It's a race condition.
execution = task_ex.executions[-1]
wf_ex_id = task_ex.workflow_execution_id
# Must be before loading the object itself (see method doc).
self._lock_workflow_execution(wf_ex_id)
wf_ex = task_ex.workflow_execution
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]" % (task_ex.name, task_ex.state, state))
task_ex.state = state
self._on_task_state_change(task_ex, wf_ex, action_ex=execution)
def set_execution_state(wf_ex, state, state_info=None, set_upstream=False):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex, "Execution of workflow '%s' [%s -> %s]" %
(wf_ex.workflow_name, cur_state, state))
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully.
wf_ex.accepted = wf_ex.state == states.SUCCESS
# If specified, then recursively set the state of the parent workflow
# executions to the same state. Only changing state to RUNNING is
# supported.
if set_upstream and state == states.RUNNING and wf_ex.task_execution_id:
task_ex = db_api.get_task_execution(wf_ex.task_execution_id)
parent_wf_ex = lock_workflow_execution(task_ex.workflow_execution_id)
set_execution_state(parent_wf_ex,
state,
state_info=state_info,
set_upstream=set_upstream)
task_handler.set_task_state(task_ex,
state,
state_info=None,
processed=False)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'RUNNING_DELAYED'.
wf_trace.info(
task_ex, "Task '%s' [%s -> %s]" %
(task_ex.name, states.RUNNING_DELAYED, states.RUNNING))
task_ex.state = states.RUNNING
return
if task_ex.state != states.IDLE:
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
# TODO(rakhmerov): This is wrong as task handler doesn't manage
# transactions and hence it can't be called explicitly.
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(id=self.wf_ex.id,
cur_state=cur_state,
state=state)
if wf_ex is None:
# Do nothing because the state was updated previously.
return
self.wf_ex = wf_ex
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex, "Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name, cur_state, state, state_info))
# Add kafka log trace, only record the changed state
if state != cur_state:
kfk_trace.log(kfk_etypes.wf_parse(cur_state, state),
None,
state,
self.wf_ex.workflow_id,
self.wf_ex.id,
None,
None,
self.wf_ex.input,
self.wf_ex.output,
triggered_by=None)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
# No need to keep task executions of this workflow in the
# lookup cache anymore.
lookup_utils.invalidate_cached_task_executions(self.wf_ex.id)
triggers.on_workflow_complete(self.wf_ex)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id)
parent_wf = Workflow(wf_ex=parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def set_state(self, state, state_info, processed=None):
"""Sets task state without executing post completion logic.
:param state: New task state.
:param state_info: New state information (i.e. error message).
:param processed: New "processed" flag value.
:return True if the state was changed as a result of this call,
False otherwise.
"""
assert self.task_ex
cur_state = self.task_ex.state
if cur_state != state or self.task_ex.state_info != state_info:
task_ex = db_api.update_task_execution_state(
id=self.task_ex.id,
cur_state=cur_state,
state=state
)
if task_ex is None:
# Do nothing because the update query did not change the DB.
return False
self.task_ex = task_ex
self.task_ex.state_info = state_info
self.state_changed = True
if processed is not None:
self.task_ex.processed = processed
wf_trace.info(
self.task_ex.workflow_execution,
"Task '%s' (%s) [%s -> %s, msg=%s]" %
(self.task_ex.name,
self.task_ex.id,
cur_state,
state,
state_info)
)
# Add kafka log trace
task_input = None
task_output = None
actions = self.task_ex.action_executions
if actions:
if len(actions) > 1:
actions = sorted(actions, key=lambda a : a.created_at, reverse=False)
# klog.d("GET TASK action_executions", actions)
task_input = actions[-1].input
task_output = actions[-1].output
atom_id = None
if task_input and isinstance(task_input, dict) and task_input.has_key('atom'):
atom_id = task_input['atom']
trigger = {}
if 'triggered_by' in self.task_ex.runtime_context:
trigger = self.task_ex.runtime_context['triggered_by']
kfk_trace.log(kfk_etypes.tk_parse(cur_state, state), atom_id, state,
self.wf_ex.workflow_id, self.wf_ex.id,
self.task_ex.id, self.task_ex.name,
task_input, task_output, trigger)
return True
def _log_task_delay(task_ex, delay_sec):
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s, delay = %s sec]" %
(task_ex.name, task_ex.state, states.RUNNING_DELAYED, delay_sec)
)
def _log_task_delay(task_ex, delay_sec):
wf_trace.info(
task_ex, "Task '%s' [%s -> %s, delay = %s sec]" %
(task_ex.name, task_ex.state, states.DELAYED, delay_sec))
