def _evaluate_expression(self, expression, ctx=None):
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(self.task_ex),
data_flow.get_workflow_environment_dict(self.wf_ex), ctx
or self.ctx, self.wf_ex.context, self.wf_ex.input)
return expr.evaluate_recursively(expression, ctx_view)
def _find_next_tasks(self, task_ex, ctx=None):
t_state = task_ex.state
t_name = task_ex.name
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex), ctx
or data_flow.evaluate_task_outbound_context(task_ex),
data_flow.get_workflow_environment_dict(self.wf_ex),
self.wf_ex.context, self.wf_ex.input)
# [(task_name, params, 'on-success'|'on-error'|'on-complete'), ...]
result = []
def process_clause(clause, event_name):
task_tuples = self._find_next_tasks_for_clause(clause, ctx_view)
for t in task_tuples:
result.append((t[0], t[1], event_name))
if t_state == states.SUCCESS:
process_clause(self.wf_spec.get_on_success_clause(t_name),
'on-success')
elif t_state == states.ERROR:
process_clause(self.wf_spec.get_on_error_clause(t_name),
'on-error')
if states.is_completed(t_state) and not states.is_cancelled(t_state):
process_clause(self.wf_spec.get_on_complete_clause(t_name),
'on-complete')
return result
def _find_next_tasks(self, task_ex, ctx):
t_n = task_ex.name
t_s = task_ex.state
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex), ctx,
data_flow.get_workflow_environment_dict(self.wf_ex),
self.wf_ex.context, self.wf_ex.input)
# [(task_name, params, 'on-success'|'on-error'|'on-complete'), ...]
result = []
if t_s == states.ERROR:
for name, cond, params in self.wf_spec.get_on_error_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-error'))
if t_s == states.SUCCESS:
for name, cond, params in self.wf_spec.get_on_success_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-success'))
if states.is_completed(t_s) and not states.is_cancelled(t_s):
for name, cond, params in self.wf_spec.get_on_complete_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-complete'))
return result
def _evaluate_expression(self, expression, ctx=None):
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(self.task_ex),
data_flow.get_workflow_environment_dict(self.wf_ex),
ctx or self.ctx,
self.wf_ex.context,
self.wf_ex.input
)
return expr.evaluate_recursively(expression, ctx_view)
def get_expression_context(self, ctx=None):
assert self.task_ex
return data_flow.ContextView(
data_flow.get_current_task_dict(self.task_ex),
data_flow.get_workflow_environment_dict(self.wf_ex),
ctx or {},
self.task_ex.in_context,
self.wf_ex.context,
self.wf_ex.input,
)
def all_errors_handled(self):
for t_ex in lookup_utils.find_error_task_executions(self.wf_ex.id):
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(t_ex),
data_flow.evaluate_task_outbound_context(t_ex),
data_flow.get_workflow_environment_dict(self.wf_ex),
self.wf_ex.context, self.wf_ex.input)
tasks_on_error = self._find_next_tasks_for_clause(
self.wf_spec.get_on_error_clause(t_ex.name), ctx_view)
if not tasks_on_error:
return False
return True
def get_published_global(task_ex, wf_ex=None):
if task_ex.state not in [states.SUCCESS, states.ERROR]:
return
if wf_ex is None:
wf_ex = task_ex.workflow_execution
expr_ctx = ContextView(get_current_task_dict(task_ex), task_ex.in_context,
get_workflow_environment_dict(wf_ex), wf_ex.context,
wf_ex.input)
task_spec = spec_parser.get_task_spec(task_ex.spec)
publish_spec = task_spec.get_publish(task_ex.state)
if not publish_spec:
return
global_vars = publish_spec.get_global()
return expr.evaluate_recursively(global_vars, expr_ctx)
def after_task_complete(self, task_ex, task_spec):
"""Called right after task completes.
:param task_ex: Completed task DB model.
:param task_spec: Completed task specification.
"""
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
task_ex.in_context,
data_flow.get_current_task_dict(task_ex),
data_flow.get_workflow_environment_dict(wf_ex),
wf_ex.context,
wf_ex.input
)
data_flow.evaluate_object_fields(self, ctx_view)
self._validate()
def before_task_start(self, task_ex, task_spec):
"""Called right before task start.
:param task_ex: DB model for task that is about to start.
:param task_spec: Task specification.
"""
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
task_ex.in_context,
data_flow.get_current_task_dict(task_ex),
data_flow.get_workflow_environment_dict(wf_ex),
wf_ex.context,
wf_ex.input
)
data_flow.evaluate_object_fields(self, ctx_view)
self._validate()
def _find_next_tasks(self, task_ex, ctx=None):
t_state = task_ex.state
t_name = task_ex.name
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
ctx or data_flow.evaluate_task_outbound_context(task_ex),
data_flow.get_workflow_environment_dict(self.wf_ex),
self.wf_ex.context,
self.wf_ex.input
)
# [(task_name, params, 'on-success'|'on-error'|'on-complete'), ...]
result = []
def process_clause(clause, event_name):
task_tuples = self._find_next_tasks_for_clause(clause, ctx_view)
for t in task_tuples:
result.append((t[0], t[1], event_name))
if t_state == states.SUCCESS:
process_clause(
self.wf_spec.get_on_success_clause(t_name),
'on-success'
)
elif t_state == states.ERROR:
process_clause(
self.wf_spec.get_on_error_clause(t_name),
'on-error'
)
if states.is_completed(t_state) and not states.is_cancelled(t_state):
process_clause(
self.wf_spec.get_on_complete_clause(t_name),
'on-complete'
)
return result
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
data_flow.evaluate_task_outbound_context(task_ex),
wf_ex.context,
wf_ex.input
)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
ctx_view
)
break_on_evaluation = expressions.evaluate(
self._break_on_clause,
ctx_view
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (
task_ex.state == states.SUCCESS and
not self._continue_on_clause
)
stop_continue_flag = (
stop_continue_flag or
(self._continue_on_clause and not continue_on_evaluation)
)
stop_continue_flag = (
stop_continue_flag or
_has_incomplete_inbound_tasks(task_ex)
)
break_triggered = (
task_ex.state == states.ERROR and
break_on_evaluation
)
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.RUNNING_DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
data_flow.evaluate_task_outbound_context(task_ex), wf_ex.context,
wf_ex.input)
continue_on_evaluation = expressions.evaluate(self._continue_on_clause,
ctx_view)
break_on_evaluation = expressions.evaluate(self._break_on_clause,
ctx_view)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (task_ex.state == states.SUCCESS
and not self._continue_on_clause)
stop_continue_flag = (stop_continue_flag
or (self._continue_on_clause
and not continue_on_evaluation))
break_triggered = (task_ex.state == states.ERROR
and break_on_evaluation)
if not retries_remain or break_triggered or stop_continue_flag:
return
data_flow.invalidate_task_execution_result(task_ex)
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
# NOTE(vgvoleg): join tasks in direct workflows can't be
# retried as-is, because these tasks can't start without
# a correct logical state.
if hasattr(task_spec, "get_join") and task_spec.get_join():
from mistral.engine import task_handler as t_h
_log_task_delay(task_ex, self.delay, states.WAITING)
task_ex.state = states.WAITING
t_h._schedule_refresh_task_state(task_ex.id, self.delay)
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
sched = sched_base.get_system_scheduler()
job = sched_base.SchedulerJob(run_after=self.delay,
func_name=_CONTINUE_TASK_PATH,
func_args={'task_ex_id': task_ex.id})
sched.schedule(job)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
data_flow.evaluate_task_outbound_context(task_ex),
wf_ex.context,
wf_ex.input
)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
ctx_view
)
break_on_evaluation = expressions.evaluate(
self._break_on_clause,
ctx_view
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (
task_ex.state == states.SUCCESS and
not self._continue_on_clause
)
stop_continue_flag = (
stop_continue_flag or
(self._continue_on_clause and not continue_on_evaluation)
)
break_triggered = (
task_ex.state == states.ERROR and
break_on_evaluation
)
if not retries_remain or break_triggered or stop_continue_flag:
return
data_flow.invalidate_task_execution_result(task_ex)
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
# NOTE(vgvoleg): join tasks in direct workflows can't be
# retried as is, because this tasks can't start without
# the correct logical state.
if hasattr(task_spec, "get_join") and task_spec.get_join():
from mistral.engine import task_handler as t_h
_log_task_delay(task_ex, self.delay, states.WAITING)
task_ex.state = states.WAITING
t_h._schedule_refresh_task_state(task_ex.id, self.delay)
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
