def _set_task_state(task_ex, state):
# TODO(rakhmerov): How do we log task result?
wf_trace.info(
task_ex.workflow_execution, "Task execution '%s' [%s -> %s]" %
(task_ex.name, task_ex.state, state))
task_ex.state = state
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'DELAYED'.
wf_trace.info(
task_ex, "Task '%s' [%s -> %s]" %
(task_ex.name, states.DELAYED, states.RUNNING))
task_ex.state = states.RUNNING
return
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
No need to move to next iteration.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if state != states.ERROR:
return
wf_trace.info(
task_ex,
"Task '%s' [%s -> ERROR]"
% (task_ex.name, task_ex.state)
)
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
if not retries_remain or self.break_on:
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def _log_action_result(wf_ex, action_ex, from_state, to_state, result):
def _result_msg():
if action_ex.state == states.ERROR:
return "error = %s" % utils.cut(result.error)
return "result = %s" % utils.cut(result.data)
wf_trace.info(
wf_ex, "Action execution '%s' [%s -> %s, %s]" %
(action_ex.name, from_state, to_state, _result_msg()))
def fail_task_if_incomplete(task_ex_id, timeout):
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = "Task timed out [id=%s, timeout(s)=%s]." % (task_ex_id, timeout)
wf_trace.info(task_ex, msg)
wf_trace.info(
task_ex, "Task '%s' [%s -> ERROR]" % (task_ex.name, task_ex.state))
rpc.get_engine_client().on_task_state_change(task_ex_id, states.ERROR)
def before_task_start(self, task_ex, task_spec):
super(PauseBeforePolicy, self).before_task_start(task_ex, task_spec)
if not self.expr:
return
wf_trace.info(
task_ex, "Workflow paused before task '%s' [%s -> %s]" %
(task_ex.name, task_ex.workflow_execution.state, states.PAUSED))
task_ex.workflow_execution.state = states.PAUSED
task_ex.state = states.IDLE
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
No need to move to next iteration.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
task_ex.runtime_context = runtime_context
state = task_ex.state
if state != states.ERROR:
return
wf_trace.info(
task_ex, "Task '%s' [%s -> ERROR]" % (task_ex.name, task_ex.state))
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
if not retries_remain or self.break_on:
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
scheduler.schedule_call(
None,
'mistral.engine.policies.fail_task_if_incomplete',
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay)
wf_trace.info(
task_ex, "Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay))
def before_task_start(self, task_ex, task_spec):
super(PauseBeforePolicy, self).before_task_start(task_ex, task_spec)
if not self.expr:
return
wf_trace.info(
task_ex,
"Workflow paused before task '%s' [%s -> %s]" %
(task_ex.name, task_ex.workflow_execution.state, states.PAUSED)
)
task_ex.workflow_execution.state = states.PAUSED
task_ex.state = states.IDLE
def _run_action_or_workflow(task_ex, task_spec, input_dict, index):
t_name = task_ex.name
if task_spec.get_action_name():
wf_trace.info(
task_ex, "Task '%s' is RUNNING [action_name = %s]" %
(t_name, task_spec.get_action_name()))
_schedule_run_action(task_ex, task_spec, input_dict, index)
elif task_spec.get_workflow_name():
wf_trace.info(
task_ex, "Task '%s' is RUNNING [workflow_name = %s]" %
(t_name, task_spec.get_workflow_name()))
_schedule_run_workflow(task_ex, task_spec, input_dict, index)
def before_task_start(self, task_ex, task_spec):
super(TimeoutPolicy, self).before_task_start(task_ex, task_spec)
scheduler.schedule_call(
None,
'mistral.engine.policies.fail_task_if_incomplete',
self.delay,
task_ex_id=task_ex.id,
timeout=self.delay
)
wf_trace.info(
task_ex,
"Timeout check scheduled [task=%s, timeout(s)=%s]." %
(task_ex.id, self.delay)
)
def fail_task_if_incomplete(task_ex_id, timeout):
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = "Task timed out [id=%s, timeout(s)=%s]." % (task_ex_id, timeout)
wf_trace.info(task_ex, msg)
wf_trace.info(
task_ex,
"Task '%s' [%s -> ERROR]" % (task_ex.name, task_ex.state)
)
rpc.get_engine_client().on_task_state_change(
task_ex_id,
states.ERROR
)
def before_task_start(self, task_ex, task_spec):
super(WaitBeforePolicy, self).before_task_start(task_ex, task_spec)
context_key = 'wait_before_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
task_ex.runtime_context = runtime_context
policy_context = runtime_context[context_key]
if policy_context.get('skip'):
# Unset state 'DELAYED'.
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, states.DELAYED, states.RUNNING)
)
task_ex.state = states.RUNNING
return
policy_context.update({'skip': True})
_log_task_delay(task_ex, self.delay)
task_ex.state = states.DELAYED
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def _log_task_delay(task_ex, delay_sec):
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s, delay = %s sec]" %
(task_ex.name, task_ex.state, states.DELAYED, delay_sec)
)
def _log_task_delay(task_ex, delay_sec):
wf_trace.info(
task_ex, "Task '%s' [%s -> %s, delay = %s sec]" %
(task_ex.name, task_ex.state, states.DELAYED, delay_sec))