def _find_next_tasks(self, task_ex, ctx):
t_n = task_ex.name
t_s = task_ex.state
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex), ctx,
data_flow.get_workflow_environment_dict(self.wf_ex),
self.wf_ex.context, self.wf_ex.input)
# [(task_name, params, 'on-success'|'on-error'|'on-complete'), ...]
result = []
if t_s == states.ERROR:
for name, cond, params in self.wf_spec.get_on_error_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-error'))
if t_s == states.SUCCESS:
for name, cond, params in self.wf_spec.get_on_success_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-success'))
if states.is_completed(t_s) and not states.is_cancelled(t_s):
for name, cond, params in self.wf_spec.get_on_complete_clause(t_n):
if not cond or expr.evaluate(cond, ctx_view):
params = expr.evaluate_recursively(params, ctx_view)
result.append((name, params, 'on-complete'))
return result
def test_evaluate_mixing_jinja_and_yaql(self):
actual = expr.evaluate('<% $.a %>{{ _.a }}', {'a': 'b'})
self.assertEqual('<% $.a %>b', actual)
actual = expr.evaluate('{{ _.a }}<% $.a %>', {'a': 'b'})
self.assertEqual('b<% $.a %>', actual)
def test_evaluate_mixing_jinja_and_yaql(self):
actual = expr.evaluate('<% $.a %>{{ _.a }}', {'a': 'b'})
self.assertEqual('<% $.a %>b', actual)
actual = expr.evaluate('{{ _.a }}<% $.a %>', {'a': 'b'})
self.assertEqual('b<% $.a %>', actual)
def test_context_view(self):
ctx = data_flow.ContextView({
'k1': 'v1',
'k11': 'v11',
'k3': 'v3'
}, {
'k2': 'v2',
'k21': 'v21',
'k3': 'v32'
})
self.assertIsInstance(ctx, dict)
self.assertEqual(5, len(ctx))
self.assertIn('k1', ctx)
self.assertIn('k11', ctx)
self.assertIn('k3', ctx)
self.assertIn('k2', ctx)
self.assertIn('k21', ctx)
self.assertEqual('v1', ctx['k1'])
self.assertEqual('v1', ctx.get('k1'))
self.assertEqual('v11', ctx['k11'])
self.assertEqual('v11', ctx.get('k11'))
self.assertEqual('v3', ctx['k3'])
self.assertEqual('v2', ctx['k2'])
self.assertEqual('v2', ctx.get('k2'))
self.assertEqual('v21', ctx['k21'])
self.assertEqual('v21', ctx.get('k21'))
self.assertIsNone(ctx.get('Not existing key'))
self.assertRaises(exc.MistralError, ctx.update)
self.assertRaises(exc.MistralError, ctx.clear)
self.assertRaises(exc.MistralError, ctx.pop, 'k1')
self.assertRaises(exc.MistralError, ctx.popitem)
self.assertRaises(exc.MistralError, ctx.__setitem__, 'k5', 'v5')
self.assertRaises(exc.MistralError, ctx.__delitem__, 'k2')
self.assertEqual('v1', expr.evaluate('<% $.k1 %>', ctx))
self.assertEqual('v2', expr.evaluate('<% $.k2 %>', ctx))
self.assertEqual('v3', expr.evaluate('<% $.k3 %>', ctx))
# Now change the order of dictionaries and make sure to have
# a different for key 'k3'.
ctx = data_flow.ContextView({
'k2': 'v2',
'k21': 'v21',
'k3': 'v32'
}, {
'k1': 'v1',
'k11': 'v11',
'k3': 'v3'
})
self.assertEqual('v32', expr.evaluate('<% $.k3 %>', ctx))
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = "retry_task_policy"
runtime_context = _ensure_context_has_key(task_ex.runtime_context, context_key)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause, data_flow.evaluate_task_outbound_context(task_ex)
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if "retry_no" in policy_context:
retry_no = policy_context["retry_no"]
del policy_context["retry_no"]
retries_remain = retry_no + 1 < self.count
stop_continue_flag = task_ex.state == states.SUCCESS and not self._continue_on_clause
stop_continue_flag = stop_continue_flag or (self._continue_on_clause and not continue_on_evaluation)
break_triggered = task_ex.state == states.ERROR and self.break_on
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.DELAYED
policy_context["retry_no"] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(None, _RUN_EXISTING_TASK_PATH, self.delay, task_ex_id=task_ex.id)
def test_context_view_eval_root_with_yaql(self):
ctx = data_flow.ContextView({'k1': 'v1'}, {'k2': 'v2'})
res = expr.evaluate('<% $ %>', ctx)
self.assertIsNotNone(res)
self.assertIsInstance(res, dict)
self.assertEqual(2, len(res))
def test_context_view_eval_values(self):
ctx = data_flow.ContextView({'k1': 'v1'}, {'k2': 'v2'})
res = expr.evaluate('<% $.values() %>', ctx)
self.assertIsNotNone(res)
self.assertIsInstance(res, list)
self.assertEqual(2, len(res))
self.assertIn('v1', res)
self.assertIn('v2', res)
def test_context_view_eval_root_with_yaql(self):
ctx = data_flow.ContextView(
{'k1': 'v1'},
{'k2': 'v2'}
)
res = expr.evaluate('<% $ %>', ctx)
self.assertIsNotNone(res)
self.assertIsInstance(res, dict)
self.assertEqual(2, len(res))
def test_context_view_eval_values(self):
ctx = data_flow.ContextView(
{'k1': 'v1'},
{'k2': 'v2'}
)
res = expr.evaluate('<% $.values() %>', ctx)
self.assertIsNotNone(res)
self.assertIsInstance(res, list)
self.assertEqual(2, len(res))
self.assertIn('v1', res)
self.assertIn('v2', res)
def _find_next_task_names_for_clause(clause, ctx):
"""Finds next task(command) names base on given {name: condition}
dictionary.
:param clause: Dictionary {task_name: condition} taken from
'on-complete', 'on-success' or 'on-error' clause.
:param ctx: Context that clause expressions should be evaluated
against of.
:return: List of task(command) names.
"""
if not clause:
return []
return [t_name for t_name, condition in clause if not condition or expr.evaluate(condition, ctx)]
def get_task_runtime(task_spec, state=states.IDLE, outbound_context=None,
task_runtime_context=None):
"""
Computes the state and exec_flow_context runtime properties for a task
based on the supplied properties. This method takes the retry nature of a
task into consideration.
:param task_spec: specification of the task
:param state: suggested next state
:param outbound_context: outbound_context to be used for computation
:param task_runtime_context: current flow context
:return: state, exec_flow_context tuple. Sample scenarios are,
1. state = SUCCESS
No need to move to next iteration.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
if not (state == states.ERROR and task_spec.is_retry_task()):
return state, task_runtime_context
if task_runtime_context is None:
task_runtime_context = {}
if outbound_context is None:
outbound_context = {}
wf_trace_msg = "Task '%s' [%s -> " % (task_spec.name, state)
retry_no = -1
if "retry_no" in task_runtime_context:
retry_no = task_runtime_context["retry_no"]
retry_count, break_on, delay = task_spec.get_retry_parameters()
retries_remain = retry_no + 1 < retry_count
break_early = expressions.evaluate(break_on, outbound_context) if \
break_on and outbound_context else False
if retries_remain and not break_early:
state = states.DELAYED if delay > 0 else states.IDLE
retry_no += 1
WORKFLOW_TRACE.info(wf_trace_msg + "%s, delay = %s sec]" % (state, delay))
task_runtime_context["retry_no"] = retry_no
return state, task_runtime_context
def _find_next_task_names_for_clause(clause, ctx):
"""Finds next task(command) names base on given {name: condition}
dictionary.
:param clause: Dictionary {task_name: condition} taken from
'on-complete', 'on-success' or 'on-error' clause.
:param ctx: Context that clause expressions should be evaluated
against of.
:return: List of task(command) names.
"""
if not clause:
return []
return [
t_name for t_name, condition in clause
if not condition or expr.evaluate(condition, ctx)
]
def _find_next_tasks_for_clause(clause, ctx):
"""Finds next tasks names.
This method finds next task(command) base on given {name: condition}
dictionary.
:param clause: Tuple (task_name, condition, parameters) taken from
'on-complete', 'on-success' or 'on-error' clause.
:param ctx: Context that clause expressions should be evaluated
against of.
:return: List of task(command) names.
"""
if not clause:
return []
return [(t_name, expr.evaluate_recursively(params, ctx))
for t_name, condition, params in clause
if not condition or expr.evaluate(condition, ctx)]
def _find_next_tasks_for_clause(clause, ctx):
"""Finds next tasks names.
This method finds next tasks(commands) base on given {name: condition}
dictionary.
:param clause: Tuple (task_name, condition, parameters) taken from
'on-complete', 'on-success' or 'on-error' clause.
:param ctx: Context that clause expressions should be evaluated
against of.
:return: List of task(command) names.
"""
if not clause:
return []
return [
(t_name, expr.evaluate_recursively(params, ctx))
for t_name, condition, params in clause
if not condition or expr.evaluate(condition, ctx)
]
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
data_flow.evaluate_task_outbound_context(task_ex), wf_ex.context,
wf_ex.input)
continue_on_evaluation = expressions.evaluate(self._continue_on_clause,
ctx_view)
break_on_evaluation = expressions.evaluate(self._break_on_clause,
ctx_view)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (task_ex.state == states.SUCCESS
and not self._continue_on_clause)
stop_continue_flag = (stop_continue_flag
or (self._continue_on_clause
and not continue_on_evaluation))
break_triggered = (task_ex.state == states.ERROR
and break_on_evaluation)
if not retries_remain or break_triggered or stop_continue_flag:
return
data_flow.invalidate_task_execution_result(task_ex)
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
# NOTE(vgvoleg): join tasks in direct workflows can't be
# retried as-is, because these tasks can't start without
# a correct logical state.
if hasattr(task_spec, "get_join") and task_spec.get_join():
from mistral.engine import task_handler as t_h
_log_task_delay(task_ex, self.delay, states.WAITING)
task_ex.state = states.WAITING
t_h._schedule_refresh_task_state(task_ex.id, self.delay)
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
sched = sched_base.get_system_scheduler()
job = sched_base.SchedulerJob(run_after=self.delay,
func_name=_CONTINUE_TASK_PATH,
func_args={'task_ex_id': task_ex.id})
sched.schedule(job)
def after_task_complete(self, task):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task.task_ex.executions # noqa
ctx_key = 'retry_task_policy'
expr_ctx = task.get_expression_context(
ctx=data_flow.evaluate_task_outbound_context(task.task_ex))
continue_on_evaluation = expressions.evaluate(self._continue_on_clause,
expr_ctx)
break_on_evaluation = expressions.evaluate(self._break_on_clause,
expr_ctx)
state = task.get_state()
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_ctx = task.get_policy_context(ctx_key)
retry_no = 0
if 'retry_no' in policy_ctx:
retry_no = policy_ctx['retry_no']
del policy_ctx['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (task.get_state() == states.SUCCESS
and not self._continue_on_clause)
stop_continue_flag = (stop_continue_flag
or (self._continue_on_clause
and not continue_on_evaluation))
break_triggered = (task.get_state() == states.ERROR
and break_on_evaluation)
if not retries_remain or break_triggered or stop_continue_flag:
return
task.invalidate_result()
policy_ctx['retry_no'] = retry_no + 1
task.touch_runtime_context()
# NOTE(vgvoleg): join tasks in direct workflows can't be
# retried as-is, because these tasks can't start without
# a correct logical state.
if hasattr(task.task_spec, "get_join") and task.task_spec.get_join():
# TODO(rakhmerov): This is an example of broken encapsulation.
# The control over such operations should belong to the class Task.
# If it's done, from the outside of the class there will be just
# one visible operation "continue_task()" or something like that.
from mistral.engine import task_handler as t_h
task.set_state(states.WAITING,
"Delayed by 'retry' policy [delay=%s]" % self.delay)
t_h._schedule_refresh_task_state(task.get_id(), self.delay)
return
task.set_state(states.RUNNING_DELAYED,
"Delayed by 'retry' policy [delay=%s]" % self.delay)
sched = sched_base.get_system_scheduler()
job = sched_base.SchedulerJob(run_after=self.delay,
func_name=_CONTINUE_TASK_PATH,
func_args={'task_ex_id': task.get_id()})
sched.schedule(job)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# TODO(m4dcoder): If the task_ex.executions collection is not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind. task_ex.executions was originally called in
# get_task_execution_result but it was refactored to use
# db_api.get_action_executions to support session-less use cases.
action_ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
data_flow.evaluate_task_outbound_context(task_ex)
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
stop_continue_flag = (task_ex.state == states.SUCCESS and
not self._continue_on_clause)
stop_continue_flag = (stop_continue_flag or
(self._continue_on_clause and
not continue_on_evaluation))
break_triggered = task_ex.state == states.ERROR and self.break_on
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.RUNNING_DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def test_context_view(self):
ctx = data_flow.ContextView(
{
'k1': 'v1',
'k11': 'v11',
'k3': 'v3'
},
{
'k2': 'v2',
'k21': 'v21',
'k3': 'v32'
}
)
self.assertIsInstance(ctx, dict)
self.assertEqual(5, len(ctx))
self.assertIn('k1', ctx)
self.assertIn('k11', ctx)
self.assertIn('k3', ctx)
self.assertIn('k2', ctx)
self.assertIn('k21', ctx)
self.assertEqual('v1', ctx['k1'])
self.assertEqual('v1', ctx.get('k1'))
self.assertEqual('v11', ctx['k11'])
self.assertEqual('v11', ctx.get('k11'))
self.assertEqual('v3', ctx['k3'])
self.assertEqual('v2', ctx['k2'])
self.assertEqual('v2', ctx.get('k2'))
self.assertEqual('v21', ctx['k21'])
self.assertEqual('v21', ctx.get('k21'))
self.assertIsNone(ctx.get('Not existing key'))
self.assertRaises(exc.MistralError, ctx.update)
self.assertRaises(exc.MistralError, ctx.clear)
self.assertRaises(exc.MistralError, ctx.pop, 'k1')
self.assertRaises(exc.MistralError, ctx.popitem)
self.assertRaises(exc.MistralError, ctx.__setitem__, 'k5', 'v5')
self.assertRaises(exc.MistralError, ctx.__delitem__, 'k2')
self.assertEqual('v1', expr.evaluate('<% $.k1 %>', ctx))
self.assertEqual('v2', expr.evaluate('<% $.k2 %>', ctx))
self.assertEqual('v3', expr.evaluate('<% $.k3 %>', ctx))
# Now change the order of dictionaries and make sure to have
# a different for key 'k3'.
ctx = data_flow.ContextView(
{
'k2': 'v2',
'k21': 'v21',
'k3': 'v32'
},
{
'k1': 'v1',
'k11': 'v11',
'k3': 'v3'
}
)
self.assertEqual('v32', expr.evaluate('<% $.k3 %>', ctx))
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# TODO(m4dcoder): If the task_ex.executions collection is not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind. task_ex.executions was originally called in
# get_task_execution_result but it was refactored to use
# db_api.get_action_executions to support session-less use cases.
action_ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
data_flow.evaluate_task_outbound_context(task_ex))
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
stop_continue_flag = (task_ex.state == states.SUCCESS
and not self._continue_on_clause)
stop_continue_flag = (stop_continue_flag
or (self._continue_on_clause
and not continue_on_evaluation))
break_triggered = task_ex.state == states.ERROR and self.break_on
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.RUNNING_DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
data_flow.evaluate_task_outbound_context(task_ex),
wf_ex.context,
wf_ex.input
)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
ctx_view
)
break_on_evaluation = expressions.evaluate(
self._break_on_clause,
ctx_view
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (
task_ex.state == states.SUCCESS and
not self._continue_on_clause
)
stop_continue_flag = (
stop_continue_flag or
(self._continue_on_clause and not continue_on_evaluation)
)
break_triggered = (
task_ex.state == states.ERROR and
break_on_evaluation
)
if not retries_remain or break_triggered or stop_continue_flag:
return
data_flow.invalidate_task_execution_result(task_ex)
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
# NOTE(vgvoleg): join tasks in direct workflows can't be
# retried as is, because this tasks can't start without
# the correct logical state.
if hasattr(task_spec, "get_join") and task_spec.get_join():
from mistral.engine import task_handler as t_h
_log_task_delay(task_ex, self.delay, states.WAITING)
task_ex.state = states.WAITING
t_h._schedule_refresh_task_state(task_ex.id, self.delay)
return
_log_task_delay(task_ex, self.delay)
task_ex.state = states.RUNNING_DELAYED
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
# There is nothing to repeat
if self.count == 0:
return
# TODO(m4dcoder): If the task_ex.action_executions and
# task_ex.workflow_executions collection are not called,
# then the retry_no in the runtime_context of the task_ex will not
# be updated accurately. To be exact, the retry_no will be one
# iteration behind.
ex = task_ex.executions # noqa
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(
task_ex.runtime_context,
context_key
)
wf_ex = task_ex.workflow_execution
ctx_view = data_flow.ContextView(
data_flow.get_current_task_dict(task_ex),
data_flow.evaluate_task_outbound_context(task_ex),
wf_ex.context,
wf_ex.input
)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
ctx_view
)
break_on_evaluation = expressions.evaluate(
self._break_on_clause,
ctx_view
)
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state) or states.is_cancelled(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no < self.count
stop_continue_flag = (
task_ex.state == states.SUCCESS and
not self._continue_on_clause
)
stop_continue_flag = (
stop_continue_flag or
(self._continue_on_clause and not continue_on_evaluation)
)
stop_continue_flag = (
stop_continue_flag or
_has_incomplete_inbound_tasks(task_ex)
)
break_triggered = (
task_ex.state == states.ERROR and
break_on_evaluation
)
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.RUNNING_DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_CONTINUE_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
def after_task_complete(self, task_ex, task_spec):
"""Possible Cases:
1. state = SUCCESS
if continue_on is not specified,
no need to move to next iteration;
if current:count achieve retry:count then policy
breaks the loop (regardless on continue-on condition);
otherwise - check continue_on condition and if
it is True - schedule the next iteration,
otherwise policy breaks the loop.
2. retry:count = 5, current:count = 2, state = ERROR,
state = IDLE/DELAYED, current:count = 3
3. retry:count = 5, current:count = 4, state = ERROR
Iterations complete therefore state = #{state}, current:count = 4.
"""
super(RetryPolicy, self).after_task_complete(task_ex, task_spec)
context_key = 'retry_task_policy'
runtime_context = _ensure_context_has_key(task_ex.runtime_context,
context_key)
continue_on_evaluation = expressions.evaluate(
self._continue_on_clause,
data_flow.evaluate_task_outbound_context(task_ex))
task_ex.runtime_context = runtime_context
state = task_ex.state
if not states.is_completed(state):
return
policy_context = runtime_context[context_key]
retry_no = 0
if 'retry_no' in policy_context:
retry_no = policy_context['retry_no']
del policy_context['retry_no']
retries_remain = retry_no + 1 < self.count
stop_continue_flag = (task_ex.state == states.SUCCESS
and not self._continue_on_clause)
stop_continue_flag = (stop_continue_flag
or (self._continue_on_clause
and not continue_on_evaluation))
break_triggered = task_ex.state == states.ERROR and self.break_on
if not retries_remain or break_triggered or stop_continue_flag:
return
_log_task_delay(task_ex, self.delay)
data_flow.invalidate_task_execution_result(task_ex)
task_ex.state = states.DELAYED
policy_context['retry_no'] = retry_no + 1
runtime_context[context_key] = policy_context
scheduler.schedule_call(
None,
_RUN_EXISTING_TASK_PATH,
self.delay,
task_ex_id=task_ex.id,
)
