def test_with_items_two_tasks_second_starts_on_success(self):
wb_text = """---
version: "2.0"
name: wb1
workflows:
with_items:
type: direct
tasks:
task1:
with-items: i in [1, 2]
action: std.echo output=<% $.i %>
on-success: task2
task2:
with-items: i in [3, 4]
action: std.echo output=<% $.i %>
"""
wb_service.create_workbook_v2(wb_text)
# Start workflow.
wf_ex = self.engine.start_workflow('wb1.with_items', {})
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task1_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.SUCCESS
)
task2_ex = self._assert_single_item(
task_execs,
name='task2',
state=states.SUCCESS
)
with db_api.transaction():
task1_ex = db_api.get_task_execution(task1_ex.id)
task2_ex = db_api.get_task_execution(task2_ex.id)
result_task1 = data_flow.get_task_execution_result(task1_ex)
result_task2 = data_flow.get_task_execution_result(task2_ex)
# Since we know that we can receive results in random order,
# check is not depend on order of items.
self.assertIn(1, result_task1)
self.assertIn(2, result_task1)
self.assertIn(3, result_task2)
self.assertIn(4, result_task2)
def put(self, id, task):
"""Update the specified task execution.
:param id: Task execution ID.
:param task: Task execution object.
"""
acl.enforce('tasks:update', context.ctx())
LOG.info("Update task execution [id=%s, task=%s]" % (id, task))
task_ex = db_api.get_task_execution(id)
task_spec = spec_parser.get_task_spec(task_ex.spec)
task_name = task.name or None
reset = task.reset
env = task.env or None
if task_name and task_name != task_ex.name:
raise exc.WorkflowException('Task name does not match.')
wf_ex = db_api.get_workflow_execution(task_ex.workflow_execution_id)
wf_name = task.workflow_name or None
if wf_name and wf_name != wf_ex.name:
raise exc.WorkflowException('Workflow name does not match.')
if task.state != states.RUNNING:
raise exc.WorkflowException(
'Invalid task state. Only updating task to rerun is supported.'
)
if task_ex.state != states.ERROR:
raise exc.WorkflowException(
'The current task execution must be in ERROR for rerun.'
' Only updating task to rerun is supported.'
)
if not task_spec.get_with_items() and not reset:
raise exc.WorkflowException(
'Only with-items task has the option to not reset.'
)
rpc.get_engine_client().rerun_workflow(
task_ex.id,
reset=reset,
env=env
)
task_ex = db_api.get_task_execution(id)
return _get_task_resource_with_result(task_ex)
def _fail_task_if_incomplete(task_ex_id, timeout):
from mistral.engine import task_handler
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = 'Task timed out [timeout(s)=%s].' % timeout
task_handler.complete_task(
db_api.get_task_execution(task_ex_id),
states.ERROR,
msg
)
def task_(context, task_name=None):
# This section may not exist in a context if it's calculated not in
# task scope.
cur_task = context['__task_execution']
# 1. If task_name is empty it's 'task()' use case, we need to get the
# current task.
# 2. if task_name is not empty but it's equal to the current task name
# we need to take exactly the current instance of this task. Otherwise
# there may be ambiguity if there are many tasks with this name.
# 3. In other case we just find a task in DB by the given name.
if cur_task and (not task_name or cur_task['name'] == task_name):
task_ex = db_api.get_task_execution(cur_task['id'])
else:
task_execs = db_api.get_task_executions(
workflow_execution_id=context['__execution']['id'],
name=task_name
)
# TODO(rakhmerov): Account for multiple executions (i.e. in case of
# cycles).
task_ex = task_execs[-1] if len(task_execs) > 0 else None
if not task_ex:
LOG.warning(
"Task '%s' not found by the task() expression function",
task_name
)
return None
# We don't use to_dict() db model method because not all fields
# make sense for user.
return _convert_to_user_model(task_ex)
def _complete_task(task_ex_id, state, state_info):
from mistral.engine import task_handler
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
task_handler.complete_task(task_ex, state, state_info)
def test_short_action(self):
wf_service.create_workflows(WF_SHORT_ACTION)
self.block_action()
wf_ex = self.engine.start_workflow('wf', None)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(states.RUNNING, wf_ex.state)
task_execs = wf_ex.task_executions
task1_ex = self._assert_single_item(task_execs, name='task1')
task2_ex = self._assert_single_item(
task_execs,
name='task2',
state=states.RUNNING
)
self._await(lambda: self.is_task_success(task1_ex.id))
self.unblock_action()
self._await(lambda: self.is_task_success(task2_ex.id))
self._await(lambda: self.is_execution_success(wf_ex.id))
task1_ex = db_api.get_task_execution(task1_ex.id)
task1_action_ex = db_api.get_action_executions(
task_execution_id=task1_ex.id
)[0]
self.assertEqual(1, task1_action_ex.output['result'])
def test_with_items_action_context(self):
wb_service.create_workbook_v2(WORKBOOK_ACTION_CONTEXT)
# Start workflow.
wf_ex = self.engine.start_workflow(
'wb1.wf1_with_items', WF_INPUT_URLS
)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
act_exs = task_ex.executions
self.engine.on_action_complete(act_exs[0].id, wf_utils.Result("Ivan"))
self.engine.on_action_complete(act_exs[1].id, wf_utils.Result("John"))
self.engine.on_action_complete(
act_exs[2].id, wf_utils.Result("Mistral")
)
self._await(
lambda: self.is_execution_success(wf_ex.id),
)
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = db_api.get_task_execution(task_ex.id)
result = data_flow.get_task_execution_result(task_ex)
self.assertTrue(isinstance(result, list))
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
self.assertEqual(states.SUCCESS, task_ex.state)
def get(self, id):
"""Return the specified task."""
LOG.info("Fetch task [id=%s]" % id)
task_ex = db_api.get_task_execution(id)
return _get_task_resource_with_result(task_ex)
def analyse_task_execution(task_ex_id, stat, filters, cur_depth):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
if filters['errors_only'] and task_ex.state != states.ERROR:
return None
update_statistics_with_task(stat, task_ex)
entry = create_task_execution_entry(task_ex)
child_executions = task_ex.executions
entry.action_executions = []
entry.workflow_executions = []
for c_ex in child_executions:
if isinstance(c_ex, db_models.ActionExecution):
entry.action_executions.append(
create_action_execution_entry(c_ex)
)
else:
entry.workflow_executions.append(
analyse_workflow_execution(c_ex.id, stat, filters, cur_depth)
)
return entry
def _recursive_rerun(self):
"""Rerun all parent workflow executions recursively.
If there is a parent execution that it reruns as well.
"""
from mistral.engine import workflow_handler
self.set_state(states.RUNNING)
# TODO(rakhmerov): We call a internal method of a module here.
# The simplest way is to make it public, however, I believe
# it's another "bad smell" that tells that some refactoring
# of the architecture should be made.
workflow_handler._schedule_check_and_fix_integrity(self.wf_ex)
if self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id
)
parent_wf = Workflow(wf_ex=parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf._recursive_rerun()
from mistral.engine import task_handler
task_handler.rerun_task(parent_task_ex, parent_wf.wf_spec)
def test_with_items_action_context(self):
wb_service.create_workbook_v2(WB_ACTION_CONTEXT)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf', WF_INPUT_URLS)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
act_exs = task_ex.executions
self.engine.on_action_complete(act_exs[0].id, wf_utils.Result("Ivan"))
self.engine.on_action_complete(act_exs[1].id, wf_utils.Result("John"))
self.engine.on_action_complete(
act_exs[2].id,
wf_utils.Result("Mistral")
)
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
result = data_flow.get_task_execution_result(task_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
self.assertEqual(states.SUCCESS, task_ex.state)
def _get_induced_join_state(self, in_task_spec, in_task_ex,
join_task_spec, t_execs_cache):
join_task_name = join_task_spec.get_name()
if not in_task_ex:
possible, depth = self._possible_route(
in_task_spec,
t_execs_cache
)
if possible:
return states.WAITING, depth, None
else:
return states.ERROR, depth, 'impossible route'
if not states.is_completed(in_task_ex.state):
return states.WAITING, 1, None
if self._is_conditional_transition(in_task_ex, in_task_spec) and \
not hasattr(in_task_ex, "in_context"):
in_task_ex = db_api.get_task_execution(in_task_ex.id)
# [(task name, params, event name), ...]
next_tasks_tuples = self._find_next_tasks(in_task_ex)
next_tasks_dict = {tup[0]: tup[2] for tup in next_tasks_tuples}
if join_task_name not in next_tasks_dict:
return states.ERROR, 1, "not triggered"
return states.RUNNING, 1, next_tasks_dict[join_task_name]
def on_task_state_change(self, task_ex_id, state):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked. But by the moment this method is called the
# last execution object may have changed. It's a race condition.
execution = task_ex.executions[-1]
wf_ex_id = task_ex.workflow_execution_id
# Must be before loading the object itself (see method doc).
self._lock_workflow_execution(wf_ex_id)
wf_ex = task_ex.workflow_execution
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]"
% (task_ex.name, task_ex.state, state)
)
task_ex.state = state
self._on_task_state_change(task_ex, wf_ex, action_ex=execution)
def get(self, id):
"""Return the specified task."""
acl.enforce('tasks:get', context.ctx())
LOG.info("Fetch task [id=%s]" % id)
task_ex = db_api.get_task_execution(id)
return _get_task_resource_with_result(task_ex)
def test_with_items_subflow_concurrency_gt_list_length(self):
wb_text = """---
version: "2.0"
name: wb1
workflows:
main:
type: direct
input:
- names
tasks:
task1:
with-items: name in <% $.names %>
workflow: subflow1 name=<% $.name %>
concurrency: 3
subflow1:
type: direct
input:
- name
output:
result: <% task(task1).result %>
tasks:
task1:
action: std.echo output=<% $.name %>
"""
wb_service.create_workbook_v2(wb_text)
# Start workflow.
names = ["Peter", "Susan", "Edmund", "Lucy", "Aslan", "Caspian"]
wf_ex = self.engine.start_workflow('wb1.main', {'names': names})
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.SUCCESS
)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
task_result = data_flow.get_task_execution_result(task_ex)
result = [item['result'] for item in task_result]
self.assertListEqual(sorted(result), sorted(names))
def rerun_workflow(self, task_ex_id, reset=True, env=None):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
wf_ex = task_ex.workflow_execution
wf_handler.rerun_workflow(wf_ex, task_ex, reset=reset, env=env)
return wf_ex.get_clone()
def get(self, id):
"""Return the specified task."""
LOG.info("Fetch task [id=%s]" % id)
task_ex = db_api.get_task_execution(id)
task = Task.from_dict(task_ex.to_dict())
task.result = json.dumps(data_flow.get_task_execution_result(task_ex))
return task
def fail_task_if_incomplete(task_ex_id, timeout):
task_ex = db_api.get_task_execution(task_ex_id)
if not states.is_completed(task_ex.state):
msg = "Task timed out [id=%s, timeout(s)=%s]." % (task_ex_id, timeout)
wf_trace.info(task_ex, msg)
wf_trace.info(task_ex, "Task '%s' [%s -> ERROR]" % (task_ex.name, task_ex.state))
rpc.get_engine_client().on_task_state_change(task_ex_id, states.ERROR)
def test_retry_async_action(self):
retry_wf = """---
version: '2.0'
repeated_retry:
tasks:
async_http:
retry:
delay: 0
count: 100
action: std.mistral_http url='https://google.com'
"""
wf_service.create_workflows(retry_wf)
wf_ex = self.engine.start_workflow('repeated_retry')
self.await_workflow_running(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.await_task_running(task_ex.id)
first_action_ex = task_ex.executions[0]
self.await_action_state(first_action_ex.id, states.RUNNING)
complete_action_params = (
first_action_ex.id,
ml_actions.Result(error="mock")
)
rpc.get_engine_client().on_action_complete(*complete_action_params)
for _ in range(2):
self.assertRaises(
exc.MistralException,
rpc.get_engine_client().on_action_complete,
*complete_action_params
)
self.await_task_running(task_ex.id)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
action_exs = task_ex.executions
self.assertEqual(2, len(action_exs))
for action_ex in action_exs:
if action_ex.id == first_action_ex.id:
expected_state = states.ERROR
else:
expected_state = states.RUNNING
self.assertEqual(expected_state, action_ex.state)
def test_pause_before_with_delay_policy(self):
wb_service.create_workbook_v2(PAUSE_BEFORE_DELAY_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1')
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task_ex = self._assert_single_item(task_execs, name='task1')
self.assertEqual(states.IDLE, task_ex.state)
# Verify wf paused by pause-before
self.await_workflow_paused(wf_ex.id)
# Allow wait-before to expire
self._sleep(2)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# Verify wf still paused (wait-before didn't reactivate)
self.await_workflow_paused(wf_ex.id)
task_ex = db_api.get_task_execution(task_ex.id)
self.assertEqual(states.IDLE, task_ex.state)
self.engine.resume_workflow(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self._assert_single_item(task_execs, name='task1')
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task_ex = self._assert_single_item(task_execs, name='task1')
next_task_ex = self._assert_single_item(task_execs, name='task2')
self.assertEqual(states.SUCCESS, task_ex.state)
self.assertEqual(states.SUCCESS, next_task_ex.state)
def run_existing_task(task_ex_id):
"""This function runs existing task execution.
It is needed mostly by scheduler.
"""
task_ex = db_api.get_task_execution(task_ex_id)
task_spec = spec_parser.get_task_spec(task_ex.spec)
wf_def = db_api.get_workflow_definition(task_ex.workflow_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
# Explicitly change task state to RUNNING.
task_ex.state = states.RUNNING
_run_existing_task(task_ex, task_spec, wf_spec)
def rerun_workflow(self, wf_ex_id, task_ex_id, reset=True, env=None):
# TODO(rakhmerov): Rewrite this functionality with Task abstraction.
with db_api.transaction():
wf_ex = wf_handler.lock_workflow_execution(wf_ex_id)
task_ex = db_api.get_task_execution(task_ex_id)
if task_ex.workflow_execution.id != wf_ex_id:
raise ValueError('Workflow execution ID does not match.')
if wf_ex.state == states.PAUSED:
return wf_ex.get_clone()
# TODO(rakhmerov): This should be a call to workflow handler.
return self._continue_workflow(wf_ex, task_ex, reset, env=env)
def _read_task_params(id, task):
with db_api.transaction():
task_ex = db_api.get_task_execution(id)
task_spec = spec_parser.get_task_spec(task_ex.spec)
task_name = task.name or None
reset = task.reset
env = task.env or None
if task_name and task_name != task_ex.name:
raise exc.WorkflowException('Task name does not match.')
wf_ex = db_api.get_workflow_execution(
task_ex.workflow_execution_id
)
return env, reset, task_ex, task_spec, wf_ex
def run_existing_task(task_ex_id, reset=True):
"""This function runs existing task execution.
It is needed mostly by scheduler.
:param task_ex_id: Task execution id.
:param reset: Reset action executions for the task.
"""
task_ex = db_api.get_task_execution(task_ex_id)
task_spec = spec_parser.get_task_spec(task_ex.spec)
wf_def = db_api.get_workflow_definition(task_ex.workflow_name)
wf_spec = spec_parser.get_workflow_spec(wf_def.spec)
# Throw exception if the existing task already succeeded.
if task_ex.state == states.SUCCESS:
raise exc.EngineException(
'Rerunning existing task that already succeeded is not supported.'
)
# Exit if the existing task failed and reset is not instructed.
# For a with-items task without reset, re-running the existing
# task will re-run the failed and unstarted items.
if (task_ex.state == states.ERROR and not reset and
not task_spec.get_with_items()):
return task_ex
# Reset nested executions only if task is not already RUNNING.
if task_ex.state != states.RUNNING:
# Reset state of processed task and related action executions.
if reset:
action_exs = task_ex.executions
else:
action_exs = db_api.get_action_executions(
task_execution_id=task_ex.id,
state=states.ERROR,
accepted=True
)
for action_ex in action_exs:
action_ex.accepted = False
# Explicitly change task state to RUNNING.
set_task_state(task_ex, states.RUNNING, None, processed=False)
_run_existing_task(task_ex, task_spec, wf_spec)
return task_ex
def set_execution_state(wf_ex, state, state_info=None, set_upstream=False):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex,
"Execution of workflow '%s' [%s -> %s]"
% (wf_ex.workflow_name, cur_state, state)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
wf_ex.accepted = wf_ex.state in (states.SUCCESS, states.ERROR)
# If specified, then recursively set the state of the parent workflow
# executions to the same state. Only changing state to RUNNING is
# supported.
if set_upstream and state == states.RUNNING and wf_ex.task_execution_id:
task_ex = db_api.get_task_execution(wf_ex.task_execution_id)
parent_wf_ex = lock_workflow_execution(task_ex.workflow_execution_id)
set_execution_state(
parent_wf_ex,
state,
state_info=state_info,
set_upstream=set_upstream
)
task_handler.set_task_state(
task_ex,
state,
state_info=None,
processed=False
)
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
self.wf_ex.state = state
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex,
"Execution of workflow '%s' [%s -> %s]"
% (self.wf_ex.workflow_name, cur_state, state)
)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id
)
parent_wf = Workflow(
db_api.get_workflow_definition(parent_task_ex.workflow_id),
parent_task_ex.workflow_execution
)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def test_with_items_concurrency_retry_policy(self):
wf_text = """---
version: "2.0"
wf:
tasks:
task1:
with-items: i in [1, 2, 3, 4]
action: std.fail
retry:
count: 3
delay: 1
concurrency: 2
on-error: task2
task2:
action: std.echo output="With-items failed"
"""
wf_service.create_workflows(wf_text)
# Start workflow.
wf_ex = self.engine.start_workflow('wf', {})
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(2, len(task_execs))
task1_ex = self._assert_single_item(task_execs, name='task1')
with db_api.transaction():
task1_ex = db_api.get_task_execution(task1_ex.id)
task1_execs = task1_ex.executions
self.assertEqual(16, len(task1_execs))
self._assert_multiple_items(task1_execs, 4, accepted=True)
def _possible_route(self, task_spec, t_execs_cache, depth=1):
in_task_specs = self.wf_spec.find_inbound_task_specs(task_spec)
if not in_task_specs:
return True, depth
for t_s in in_task_specs:
t_ex = t_execs_cache.get(t_s.get_name())
if not t_ex:
possible, depth = self._possible_route(
t_s,
t_execs_cache,
depth + 1
)
if possible:
return True, depth
else:
t_name = task_spec.get_name()
if not states.is_completed(t_ex.state):
return True, depth
# By default we don't download task context from the database,
# but just basic fields: 'id', 'name' and 'state'. It's a good
# optimization, because contexts can be too heavy and we don't
# need them most of the time.
# But sometimes we need it for conditional transitions (when
# the decision where to go is based on the current context),
# and if this is the case, we download full task execution
# and then evaluate its context to find the route.
# TODO(mfedosin): Think of a way to avoid this.
if self._is_conditional_transition(t_ex, task_spec) and \
not hasattr(t_ex, "in_context"):
t_ex = db_api.get_task_execution(t_ex.id)
if t_name in self._find_next_task_names(t_ex):
return True, depth
return False, depth
def on_task_state_change(self, task_ex_id, state, state_info=None):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked.
wf_ex_id = task_ex.workflow_execution_id
wf_ex = wf_handler.lock_workflow_execution(wf_ex_id)
wf_spec = spec_parser.get_workflow_spec(wf_ex.spec)
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s] state_info : %s"
% (task_ex.name, task_ex.state, state, state_info)
)
task_ex.state = state
task_ex.state_info = state_info
self._on_task_state_change(task_ex, wf_ex, wf_spec)
def rerun_workflow(self, wf_ex_id, task_ex_id, reset=True, env=None):
try:
with db_api.transaction():
wf_ex = wf_handler.lock_workflow_execution(wf_ex_id)
task_ex = db_api.get_task_execution(task_ex_id)
if task_ex.workflow_execution.id != wf_ex_id:
raise ValueError('Workflow execution ID does not match.')
if wf_ex.state == states.PAUSED:
return wf_ex.get_clone()
return self._continue_workflow(wf_ex, task_ex, reset, env=env)
except Exception as e:
LOG.error(
"Failed to rerun execution id=%s at task=%s: %s\n%s",
wf_ex_id, task_ex_id, e, traceback.format_exc()
)
self._fail_workflow(wf_ex_id, e)
raise e
def test_with_items_concurrency_2_fail(self):
wf_with_concurrency_2_fail = """---
version: "2.0"
concurrency_test_fail:
type: direct
tasks:
task1:
with-items: i in [1, 2, 3, 4]
action: std.fail
concurrency: 2
on-error: task2
task2:
action: std.echo output="With-items failed"
"""
wf_service.create_workflows(wf_with_concurrency_2_fail)
# Start workflow.
wf_ex = self.engine.start_workflow('concurrency_test_fail', {})
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_exs = wf_ex.task_executions
self.assertEqual(2, len(task_exs))
task_2 = self._assert_single_item(task_exs, name='task2')
with db_api.transaction():
task_2 = db_api.get_task_execution(task_2.id)
result = data_flow.get_task_execution_result(task_2)
self.assertEqual('With-items failed', result)
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
self.wf_ex.state = state
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex, "Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name, cur_state, state, state_info))
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id)
parent_wf = Workflow(
db_api.get_workflow_definition(parent_task_ex.workflow_id),
parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def test_pause_before_with_delay_policy(self):
wb_service.create_workbook_v2(PAUSE_BEFORE_DELAY_WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {})
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
self.assertEqual(states.IDLE, task_ex.state)
# Verify wf paused by pause-before
self.await_workflow_paused(wf_ex.id)
# Allow wait-before to expire
self._sleep(2)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
# Verify wf still paused (wait-before didn't reactivate)
self.await_workflow_paused(wf_ex.id)
task_ex = db_api.get_task_execution(task_ex.id)
self.assertEqual(states.IDLE, task_ex.state)
self.engine.resume_workflow(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self._assert_single_item(wf_ex.task_executions, name='task1')
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions, name='task1')
next_task_ex = self._assert_single_item(wf_ex.task_executions,
name='task2')
self.assertEqual(states.SUCCESS, task_ex.state)
self.assertEqual(states.SUCCESS, next_task_ex.state)
def test_with_items_simple(self):
wb_service.create_workbook_v2(WB)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf', wf_input=WF_INPUT)
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task1_ex = self._assert_single_item(task_execs, name='task1')
with_items_ctx = task1_ex.runtime_context['with_items']
self.assertEqual(3, with_items_ctx['count'])
# Since we know that we can receive results in random order,
# check is not depend on order of items.
with db_api.transaction():
task1_ex = db_api.get_task_execution(task1_ex.id)
result = data_flow.get_task_execution_result(task1_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
published = task1_ex.published
self.assertIn(published['result'], ['John', 'Ivan', 'Mistral'])
self.assertEqual(1, len(task_execs))
self.assertEqual(states.SUCCESS, task1_ex.state)
def test_with_items_concurrency_gt_list_length(self):
wf_definition = """---
version: "2.0"
concurrency_test:
type: direct
input:
- names: ["John", "Ivan"]
tasks:
task1:
with-items: name in <% $.names %>
action: std.echo output=<% $.name %>
concurrency: 3
"""
wf_service.create_workflows(wf_definition)
# Start workflow.
wf_ex = self.engine.start_workflow('concurrency_test', {})
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = self._assert_single_item(wf_ex.task_executions,
name='task1',
state=states.SUCCESS)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
result = data_flow.get_task_execution_result(task_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
def _possible_route(self, task_spec, t_execs_cache, depth=1):
in_task_specs = self.wf_spec.find_inbound_task_specs(task_spec)
if not in_task_specs:
return True, depth
for t_s in in_task_specs:
t_ex = t_execs_cache.get(t_s.get_name())
if not t_ex:
possible, depth = self._possible_route(t_s, t_execs_cache,
depth + 1)
if possible:
return True, depth
else:
t_name = task_spec.get_name()
if not states.is_completed(t_ex.state):
return True, depth
# By default we don't download task context from the database,
# but just basic fields: 'id', 'name' and 'state'. It's a good
# optimization, because contexts can be too heavy and we don't
# need them most of the time.
# But sometimes we need it for conditional transitions (when
# the decision where to go is based on the current context),
# and if this is the case, we download full task execution
# and then evaluate its context to find the route.
# TODO(mfedosin): Think of a way to avoid this.
if self._is_conditional_transition(t_ex, task_spec) and \
not hasattr(t_ex, "in_context"):
t_ex = db_api.get_task_execution(t_ex.id)
if t_name in self._find_next_task_names(t_ex):
return True, depth
return False, depth
def test_task_defaults_timeout_policy(self):
wf_text = """---
version: '2.0'
wf:
type: reverse
task-defaults:
timeout: 1
tasks:
task1:
action: std.async_noop
task2:
action: std.echo output=2
requires: [task1]
"""
wf_service.create_workflows(wf_text)
# Start workflow.
wf_ex = self.engine.start_workflow('wf', {}, task_name='task2')
self.await_workflow_error(wf_ex.id)
with db_api.transaction():
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
tasks = wf_ex.task_executions
self.assertEqual(1, len(tasks))
self._assert_single_item(tasks, name='task1', state=states.ERROR)
task_ex = db_api.get_task_execution(tasks[0].id)
self.assertIn("Task timed out", task_ex.state_info)
def test_with_items_action_context(self):
# TODO(rakhmerov): Seems like the name of the test is not valid
# anymore since there's nothing related to action context in it.
# We need to revisit and refactor the entire module.
wb_service.create_workbook_v2(WB_ACTION_CONTEXT)
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf',
wf_input={'items': [1, 2, 3]})
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
act_exs = task_ex.executions
self.engine.on_action_complete(act_exs[0].id,
actions_base.Result("Ivan"))
self.engine.on_action_complete(act_exs[1].id,
actions_base.Result("John"))
self.engine.on_action_complete(act_exs[2].id,
actions_base.Result("Mistral"))
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
result = data_flow.get_task_execution_result(task_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
self.assertEqual(states.SUCCESS, task_ex.state)
def on_task_state_change(self, task_ex_id, state):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
# TODO(rakhmerov): The method is mostly needed for policy and
# we are supposed to get the same action execution as when the
# policy worked. But by the moment this method is called the
# last execution object may have changed. It's a race condition.
execution = task_ex.executions[-1]
wf_ex_id = task_ex.workflow_execution_id
# Must be before loading the object itself (see method doc).
self._lock_workflow_execution(wf_ex_id)
wf_ex = task_ex.workflow_execution
wf_trace.info(
task_ex,
"Task '%s' [%s -> %s]" % (task_ex.name, task_ex.state, state))
task_ex.state = state
self._on_task_state_change(task_ex, wf_ex, action_ex=execution)
def _refresh_task_state(task_ex_id):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
wf_spec = spec_parser.get_workflow_spec_by_execution_id(
task_ex.workflow_execution_id)
wf_ctrl = wf_base.get_controller(task_ex.workflow_execution, wf_spec)
state, state_info = wf_ctrl.get_logical_task_state(task_ex)
if state == states.RUNNING:
continue_task(task_ex)
elif state == states.ERROR:
fail_task(task_ex, state_info)
elif state == states.WAITING:
# TODO(rakhmerov): Algorithm for increasing rescheduling delay.
_schedule_refresh_task_state(task_ex, 1)
else:
# Must never get here.
raise RuntimeError(
'Unexpected logical task state [task_ex=%s, state=%s]' %
(task_ex, state))
def _recursive_rerun(self):
"""Rerun all parent workflow executions recursively.
If there is a parent execution that it reruns as well.
"""
from mistral.engine import workflow_handler
self.set_state(states.RUNNING)
workflow_handler._schedule_check_and_complete(self.wf_ex)
if self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id)
parent_wf = Workflow(wf_ex=parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf._recursive_rerun()
from mistral.engine import task_handler
task_handler.rerun_task(parent_task_ex, parent_wf.wf_spec)
def set_execution_state(wf_ex, state, state_info=None, set_upstream=False):
cur_state = wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex.state = state
wf_ex.state_info = state_info
wf_trace.info(
wf_ex, "Execution of workflow '%s' [%s -> %s]" %
(wf_ex.workflow_name, cur_state, state))
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, wf_ex.name, wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully.
wf_ex.accepted = wf_ex.state == states.SUCCESS
# If specified, then recursively set the state of the parent workflow
# executions to the same state. Only changing state to RUNNING is
# supported.
if set_upstream and state == states.RUNNING and wf_ex.task_execution_id:
task_ex = db_api.get_task_execution(wf_ex.task_execution_id)
parent_wf_ex = lock_workflow_execution(task_ex.workflow_execution_id)
set_execution_state(parent_wf_ex,
state,
state_info=state_info,
set_upstream=set_upstream)
task_handler.set_task_state(task_ex,
state,
state_info=None,
processed=False)
def analyse_task_execution(task_ex_id, stat, filters, cur_depth):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
if filters['errors_only'] and task_ex.state != states.ERROR:
return None
update_statistics_with_task(stat, task_ex)
entry = create_task_execution_entry(task_ex)
child_executions = task_ex.executions
entry.action_executions = []
entry.workflow_executions = []
for c_ex in child_executions:
if isinstance(c_ex, db_models.ActionExecution):
entry.action_executions.append(create_action_execution_entry(c_ex))
else:
entry.workflow_executions.append(
analyse_workflow_execution(c_ex.id, stat, filters, cur_depth))
return entry
def task_(context, task_name):
# This section may not exist in a context if it's calculated not in
# task scope.
cur_task = context['__task_execution']
if cur_task and cur_task['name'] == task_name:
task_ex = db_api.get_task_execution(cur_task['id'])
else:
task_execs = db_api.get_task_executions(
workflow_execution_id=context['__execution']['id'],
name=task_name
)
# TODO(rakhmerov): Account for multiple executions (i.e. in case of
# cycles).
task_ex = task_execs[-1] if len(task_execs) > 0 else None
if not task_ex:
return None
# We don't use to_dict() db model method because not all fields
# make sense for user.
return _convert_to_user_model(task_ex)
def test_with_items_concurrency_3(self):
wf_with_concurrency_3 = """---
version: "2.0"
concurrency_test:
type: direct
input:
- names: ["John", "Ivan", "Mistral"]
tasks:
task1:
action: std.async_noop
with-items: name in <% $.names %>
concurrency: 3
"""
wf_service.create_workflows(wf_with_concurrency_3)
# Start workflow.
wf_ex = self.engine.start_workflow('concurrency_test')
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assertEqual(3, self._get_running_actions_count(task_ex.id))
# 1st iteration complete.
action_ex_id = self._get_incomplete_action(task_ex.id).id
self.engine.on_action_complete(action_ex_id,
actions_base.Result("John"))
# Wait till the delayed on_action_complete is processed.
self._await(lambda: self._action_result_equals(action_ex_id,
{'result': 'John'}))
incomplete_action = self._get_incomplete_action(task_ex.id)
# 2nd iteration complete.
self.engine.on_action_complete(incomplete_action.id,
actions_base.Result("Ivan"))
self._await(lambda: self._action_result_equals(incomplete_action.id,
{'result': 'Ivan'}))
incomplete_action = self._get_incomplete_action(task_ex.id)
# 3rd iteration complete.
self.engine.on_action_complete(incomplete_action.id,
actions_base.Result("Mistral"))
self._await(lambda: self._action_result_equals(incomplete_action.id,
{'result': 'Mistral'}))
task_ex = db_api.get_task_execution(task_ex.id)
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
self.assertEqual(states.SUCCESS, task_ex.state)
# Since we know that we can receive results in random order,
# check is not depend on order of items.
result = data_flow.get_task_execution_result(task_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
def _continue_task(task_ex_id):
from mistral.engine import task_handler
with db_api.transaction():
task_handler.continue_task(db_api.get_task_execution(task_ex_id))
def _get_running_actions_count(task_ex_id):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
return len(
[e for e in task_ex.executions if e.state == states.RUNNING])
def _get_incomplete_action(task_ex_id):
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex_id)
return [e for e in task_ex.executions if not e.accepted][0]
def is_task_in_state(self, ex_id, state):
return db_api.get_task_execution(ex_id).state == state
def on_task_status_update(ex_id, data, event, timestamp, **kwargs):
with db_api.transaction():
task_ex = db_api.load_task_execution(ex_id)
wf_ex = task_ex.workflow_execution
wf_ex_data = wf_ex.to_dict()
parent_wf_tk_id = wf_ex.task_execution_id
root_id = wf_ex_data.get('root_execution_id') or wf_ex_data.get('id')
LOG.info('[%s] The task event %s for %s will be processed for st2.',
root_id, event, ex_id)
if wf_ex_data['state'] == states.CANCELLED:
trigger_workflow_event(root_id, ex_id, event,
events.WORKFLOW_CANCELLED, wf_ex_data,
timestamp, **kwargs)
if wf_ex_data['state'] == states.PAUSED:
trigger_workflow_event(root_id, ex_id, event, events.WORKFLOW_PAUSED,
wf_ex_data, timestamp, **kwargs)
# Cascade event upstream (from workbook subworkflow).
while parent_wf_tk_id:
with db_api.transaction():
parent_wf_tk_ex = db_api.get_task_execution(parent_wf_tk_id)
parent_wf_ex = parent_wf_tk_ex.workflow_execution
parent_wf_ex_data = parent_wf_ex.to_dict()
grant_parent_wf_tk_id = parent_wf_ex.task_execution_id
if parent_wf_ex_data['state'] != states.PAUSED:
break
trigger_workflow_event(root_id, ex_id, event,
events.WORKFLOW_PAUSED, parent_wf_ex_data,
timestamp, **kwargs)
parent_wf_tk_id = grant_parent_wf_tk_id
# Cascade event upstream (from subworkflow action).
st2_ctx = get_st2_ctx(wf_ex_data)
parent_ctx = st2_ctx.get('parent', {}).get('mistral', {})
parent_wf_ex_id = parent_ctx.get('workflow_execution_id')
while parent_wf_ex_id:
with db_api.transaction():
parent_wf_ex = db_api.get_workflow_execution(parent_wf_ex_id)
parent_wf_ex_data = parent_wf_ex.to_dict()
if parent_wf_ex_data['state'] != states.PAUSED:
break
trigger_workflow_event(root_id, ex_id, event,
events.WORKFLOW_PAUSED, parent_wf_ex_data,
timestamp, **kwargs)
st2_ctx = get_st2_ctx(parent_wf_ex_data)
parent_ctx = st2_ctx.get('parent', {}).get('mistral', {})
parent_wf_ex_id = parent_ctx.get('workflow_execution_id')
LOG.info('[%s] The task event %s for %s is processed for st2.', root_id,
event, ex_id)
def test_with_items_concurrency_3(self):
wf_with_concurrency_3 = """---
version: "2.0"
concurrency_test:
type: direct
input:
- names: ["John", "Ivan", "Mistral"]
tasks:
task1:
action: std.async_noop
with-items: name in <% $.names %>
concurrency: 3
"""
wf_service.create_workflows(wf_with_concurrency_3)
# Start workflow.
wf_ex = self.engine.start_workflow('concurrency_test', {})
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
running_cnt = self._get_running_actions_count(task_ex)
self._assert_capacity(0, task_ex)
self.assertEqual(3, running_cnt)
# 1st iteration complete.
self.engine.on_action_complete(
self._get_incomplete_action(task_ex).id, wf_utils.Result("John"))
# Wait till the delayed on_action_complete is processed.
# 1 is always there to periodically check WF completion.
self._await(lambda: len(db_api.get_delayed_calls()) == 1)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
self._assert_capacity(1, task_ex)
incomplete_action = self._get_incomplete_action(task_ex)
# 2nd iteration complete.
self.engine.on_action_complete(incomplete_action.id,
wf_utils.Result("Ivan"))
self._await(lambda: len(db_api.get_delayed_calls()) == 1)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
self._assert_capacity(2, task_ex)
incomplete_action = self._get_incomplete_action(task_ex)
# 3rd iteration complete.
self.engine.on_action_complete(incomplete_action.id,
wf_utils.Result("Mistral"))
self._await(lambda: len(db_api.get_delayed_calls()) in (0, 1))
task_ex = db_api.get_task_execution(task_ex.id)
self._assert_capacity(3, task_ex)
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
self.assertEqual(states.SUCCESS, task_ex.state)
# Since we know that we can receive results in random order,
# check is not depend on order of items.
result = data_flow.get_task_execution_result(task_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
def is_task_processed(self, task_ex_id):
return db_api.get_task_execution(task_ex_id).processed
def _retrieve_task():
with db_api.transaction():
task_ex = db_api.get_task_execution(id)
return _get_task_resource_with_result(task_ex)
def set_state(self, state, state_info=None, recursive=False):
assert self.wf_ex
cur_state = self.wf_ex.state
if states.is_valid_transition(cur_state, state):
wf_ex = db_api.update_workflow_execution_state(id=self.wf_ex.id,
cur_state=cur_state,
state=state)
if wf_ex is None:
# Do nothing because the state was updated previously.
return
self.wf_ex = wf_ex
self.wf_ex.state_info = state_info
wf_trace.info(
self.wf_ex, "Workflow '%s' [%s -> %s, msg=%s]" %
(self.wf_ex.workflow_name, cur_state, state, state_info))
# Add kafka log trace, only record the changed state
if state != cur_state:
kfk_trace.log(kfk_etypes.wf_parse(cur_state, state),
None,
state,
self.wf_ex.workflow_id,
self.wf_ex.id,
None,
None,
self.wf_ex.input,
self.wf_ex.output,
triggered_by=None)
else:
msg = ("Can't change workflow execution state from %s to %s. "
"[workflow=%s, execution_id=%s]" %
(cur_state, state, self.wf_ex.name, self.wf_ex.id))
raise exc.WorkflowException(msg)
# Workflow result should be accepted by parent workflows (if any)
# only if it completed successfully or failed.
self.wf_ex.accepted = states.is_completed(state)
if states.is_completed(state):
# No need to keep task executions of this workflow in the
# lookup cache anymore.
lookup_utils.invalidate_cached_task_executions(self.wf_ex.id)
triggers.on_workflow_complete(self.wf_ex)
if recursive and self.wf_ex.task_execution_id:
parent_task_ex = db_api.get_task_execution(
self.wf_ex.task_execution_id)
parent_wf = Workflow(wf_ex=parent_task_ex.workflow_execution)
parent_wf.lock()
parent_wf.set_state(state, recursive=recursive)
# TODO(rakhmerov): It'd be better to use instance of Task here.
parent_task_ex.state = state
parent_task_ex.state_info = None
parent_task_ex.processed = False
def test_with_items_subflow_concurrency_gt_list_length(self):
wb_text = """---
version: "2.0"
name: wb1
workflows:
main:
type: direct
input:
- names
tasks:
task1:
with-items: name in <% $.names %>
workflow: subflow1 name=<% $.name %>
concurrency: 3
subflow1:
type: direct
input:
- name
output:
result: <% task(task1).result %>
tasks:
task1:
action: std.echo output=<% $.name %>
"""
wb_service.create_workbook_v2(wb_text)
# Start workflow.
names = ["Peter", "Susan", "Edmund", "Lucy", "Aslan", "Caspian"]
wf_ex = self.engine.start_workflow(
'wb1.main',
wf_input={'names': names}
)
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
task_ex = self._assert_single_item(
task_execs,
name='task1',
state=states.SUCCESS
)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
task_result = data_flow.get_task_execution_result(task_ex)
result = [item['result'] for item in task_result]
self.assertListEqual(sorted(result), sorted(names))
def test_resume_diff_env_vars(self):
wb_service.create_workbook_v2(RESUME_WORKBOOK_DIFF_ENV_VAR)
# Initial environment variables for the workflow execution.
env = {'var1': 'fee fi fo fum', 'var2': 'foobar'}
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf1', {}, env=env)
self.await_workflow_paused(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_1_ex = self._assert_single_item(wf_ex.task_executions,
name='task1')
task_2_ex = self._assert_single_item(wf_ex.task_executions,
name='task2')
self.assertEqual(states.PAUSED, wf_ex.state)
self.assertEqual(2, len(wf_ex.task_executions))
self.assertDictEqual(env, wf_ex.params['env'])
self.assertDictEqual(env, wf_ex.context['__env'])
self.assertEqual(states.SUCCESS, task_1_ex.state)
self.assertEqual(states.IDLE, task_2_ex.state)
# Update env in workflow execution with the following.
updated_env = {'var1': 'Task 2', 'var2': 'Task 3'}
# Update the env variables and resume workflow.
self.engine.resume_workflow(wf_ex.id, env=updated_env)
self.await_workflow_success(wf_ex.id)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertDictEqual(updated_env, wf_ex.params['env'])
self.assertDictEqual(updated_env, wf_ex.context['__env'])
self.assertEqual(3, len(wf_ex.task_executions))
# Check result of task2.
task_2_ex = self._assert_single_item(wf_ex.task_executions,
name='task2')
self.assertEqual(states.SUCCESS, task_2_ex.state)
# Re-read task execution, otherwise lazy loading of action executions
# may not work.
with db_api.transaction():
task_2_ex = db_api.get_task_execution(task_2_ex.id)
task_2_result = data_flow.get_task_execution_result(task_2_ex)
self.assertEqual(updated_env['var1'], task_2_result)
# Check result of task3.
task_3_ex = self._assert_single_item(wf_ex.task_executions,
name='task3')
self.assertEqual(states.SUCCESS, task_3_ex.state)
# Re-read task execution, otherwise lazy loading of action executions
# may not work.
with db_api.transaction():
task_3_ex = db_api.get_task_execution(task_3_ex.id)
task_3_result = data_flow.get_task_execution_result(task_3_ex)
self.assertEqual(updated_env['var2'], task_3_result)
def test_with_items_concurrency_3(self):
wf_with_concurrency_3 = """---
version: "2.0"
concurrency_test:
type: direct
input:
- names: ["John", "Ivan", "Mistral"]
tasks:
task1:
action: std.async_noop
with-items: name in <% $.names %>
concurrency: 3
"""
wf_service.create_workflows(wf_with_concurrency_3)
# Start workflow.
wf_ex = self.engine.start_workflow('concurrency_test', {})
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_ex = wf_ex.task_executions[0]
self.assert_capacity(0, task_ex)
self.assertEqual(3, self.get_running_action_exs_number(task_ex))
# 1st iteration complete.
self.engine.on_action_complete(
self.get_incomplete_action_ex(task_ex).id,
wf_utils.Result("John")
)
task_ex = db_api.get_task_execution(task_ex.id)
self.assert_capacity(1, task_ex)
# 2nd iteration complete.
self.engine.on_action_complete(
self.get_incomplete_action_ex(task_ex).id,
wf_utils.Result("Ivan")
)
task_ex = db_api.get_task_execution(task_ex.id)
self.assert_capacity(2, task_ex)
# 3rd iteration complete.
self.engine.on_action_complete(
self.get_incomplete_action_ex(task_ex).id,
wf_utils.Result("Mistral")
)
task_ex = db_api.get_task_execution(task_ex.id)
self.assert_capacity(3, task_ex)
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
task_ex = db_api.get_task_execution(task_ex.id)
self.assertEqual(states.SUCCESS, task_ex.state)
# Since we know that we can receive results in random order,
# check is not depend on order of items.
result = data_flow.get_task_execution_result(task_ex)
self.assertIsInstance(result, list)
self.assertIn('John', result)
self.assertIn('Ivan', result)
self.assertIn('Mistral', result)
def test_complex_cycle(self):
wf_text = """
version: '2.0'
wf:
vars:
cnt: 0
output:
cnt: <% $.cnt %>
tasks:
task1:
on-complete:
- task2
task2:
action: std.echo output=2
publish:
cnt: <% $.cnt + 1 %>
on-success:
- task3
task3:
action: std.echo output=3
on-complete:
- task4
task4:
action: std.echo output=4
on-success:
- task2: <% $.cnt < 2 %>
- task5: <% $.cnt >= 2 %>
task5:
action: std.echo output=<% $.cnt %>
"""
wf_service.create_workflows(wf_text)
wf_ex = self.engine.start_workflow('wf')
self.await_workflow_success(wf_ex.id)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertDictEqual({'cnt': 2}, wf_ex.output)
t_execs = wf_ex.task_executions
# Expecting one execution for task1 and task5 and two executions
# for task2, task3 and task4 because of the cycle
# 'task2 -> task3 -> task4 -> task2'.
self._assert_single_item(t_execs, name='task1')
self._assert_multiple_items(t_execs, 2, name='task2')
self._assert_multiple_items(t_execs, 2, name='task3')
self._assert_multiple_items(t_execs, 2, name='task4')
task5_ex = self._assert_single_item(t_execs, name='task5')
self.assertEqual(8, len(t_execs))
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertTrue(all(states.SUCCESS == t_ex.state for t_ex in t_execs))
with db_api.transaction():
task5_ex = db_api.get_task_execution(task5_ex.id)
self.assertEqual(2, data_flow.get_task_execution_result(task5_ex))
def test_on_action_complete(self):
wf_input = {'param1': 'Hey', 'param2': 'Hi'}
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf',
wf_input=wf_input,
task_name='task2')
self.assertIsNotNone(wf_ex)
self.assertEqual(states.RUNNING, wf_ex.state)
with db_api.transaction():
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
task_execs = wf_ex.task_executions
self.assertEqual(1, len(task_execs))
task1_ex = task_execs[0]
self.assertEqual('task1', task1_ex.name)
self.assertEqual(states.RUNNING, task1_ex.state)
self.assertIsNotNone(task1_ex.spec)
self.assertDictEqual({}, task1_ex.runtime_context)
self.assertNotIn('__execution', task1_ex.in_context)
action_execs = db_api.get_action_executions(
task_execution_id=task1_ex.id)
self.assertEqual(1, len(action_execs))
task1_action_ex = action_execs[0]
self.assertIsNotNone(task1_action_ex)
self.assertDictEqual({'output': 'Hey'}, task1_action_ex.input)
# Finish action of 'task1'.
task1_action_ex = self.engine.on_action_complete(
task1_action_ex.id, ml_actions.Result(data='Hey'))
self.assertIsInstance(task1_action_ex, models.ActionExecution)
self.assertEqual('std.echo', task1_action_ex.name)
self.assertEqual(states.SUCCESS, task1_action_ex.state)
# Data Flow properties.
task1_ex = db_api.get_task_execution(task1_ex.id) # Re-read the state.
self.assertDictEqual({'var': 'Hey'}, task1_ex.published)
self.assertDictEqual({'output': 'Hey'}, task1_action_ex.input)
self.assertDictEqual({'result': 'Hey'}, task1_action_ex.output)
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIsNotNone(wf_ex)
self.assertEqual(states.RUNNING, wf_ex.state)
task_execs = wf_ex.task_executions
self.assertEqual(2, len(task_execs))
task2_ex = self._assert_single_item(task_execs, name='task2')
self.assertEqual(states.RUNNING, task2_ex.state)
action_execs = db_api.get_action_executions(
task_execution_id=task2_ex.id)
self.assertEqual(1, len(action_execs))
task2_action_ex = action_execs[0]
self.assertIsNotNone(task2_action_ex)
self.assertDictEqual({'output': 'Hi'}, task2_action_ex.input)
# Finish 'task2'.
task2_action_ex = self.engine.on_action_complete(
task2_action_ex.id, ml_actions.Result(data='Hi'))
with db_api.transaction():
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIsNotNone(wf_ex)
task_execs = wf_ex.task_executions
# Workflow completion check is done separate with scheduler
# but scheduler doesn't start in this test (in fact, it's just
# a DB test)so the workflow is expected to be in running state.
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertIsInstance(task2_action_ex, models.ActionExecution)
self.assertEqual('std.echo', task2_action_ex.name)
self.assertEqual(states.SUCCESS, task2_action_ex.state)
# Data Flow properties.
self.assertDictEqual({'output': 'Hi'}, task2_action_ex.input)
self.assertDictEqual({}, task2_ex.published)
self.assertDictEqual({'output': 'Hi'}, task2_action_ex.input)
self.assertDictEqual({'result': 'Hi'}, task2_action_ex.output)
self.assertEqual(2, len(task_execs))
self._assert_single_item(task_execs, name='task1')
self._assert_single_item(task_execs, name='task2')
def test_on_action_complete(self):
wf_input = {'param1': 'Hey', 'param2': 'Hi'}
# Start workflow.
wf_ex = self.engine.start_workflow('wb.wf',
wf_input,
task_name='task2')
self.assertIsNotNone(wf_ex)
self.assertEqual(states.RUNNING, wf_ex.state)
# Note: We need to reread execution to access related tasks.
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertEqual(1, len(wf_ex.task_executions))
task1_ex = wf_ex.task_executions[0]
self.assertEqual('task1', task1_ex.name)
self.assertEqual(states.RUNNING, task1_ex.state)
self.assertIsNotNone(task1_ex.spec)
self.assertDictEqual({}, task1_ex.runtime_context)
self._assert_dict_contains_subset(wf_input, task1_ex.in_context)
self.assertIn('__execution', task1_ex.in_context)
action_execs = db_api.get_action_executions(
task_execution_id=task1_ex.id)
self.assertEqual(1, len(action_execs))
task1_action_ex = action_execs[0]
self.assertIsNotNone(task1_action_ex)
self.assertDictEqual({'output': 'Hey'}, task1_action_ex.input)
# Finish action of 'task1'.
task1_action_ex = self.engine.on_action_complete(
task1_action_ex.id, wf_utils.Result(data='Hey'))
self.assertIsInstance(task1_action_ex, models.ActionExecution)
self.assertEqual('std.echo', task1_action_ex.name)
self.assertEqual(states.SUCCESS, task1_action_ex.state)
# Data Flow properties.
task1_ex = db_api.get_task_execution(task1_ex.id) # Re-read the state.
self._assert_dict_contains_subset(wf_input, task1_ex.in_context)
self.assertIn('__execution', task1_ex.in_context)
self.assertDictEqual({'var': 'Hey'}, task1_ex.published)
self.assertDictEqual({'output': 'Hey'}, task1_action_ex.input)
self.assertDictEqual({'result': 'Hey'}, task1_action_ex.output)
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIsNotNone(wf_ex)
self.assertEqual(states.RUNNING, wf_ex.state)
self.assertEqual(2, len(wf_ex.task_executions))
task2_ex = self._assert_single_item(wf_ex.task_executions,
name='task2')
self.assertEqual(states.RUNNING, task2_ex.state)
action_execs = db_api.get_action_executions(
task_execution_id=task2_ex.id)
self.assertEqual(1, len(action_execs))
task2_action_ex = action_execs[0]
self.assertIsNotNone(task2_action_ex)
self.assertDictEqual({'output': 'Hi'}, task2_action_ex.input)
# Finish 'task2'.
task2_action_ex = self.engine.on_action_complete(
task2_action_ex.id, wf_utils.Result(data='Hi'))
wf_ex = db_api.get_workflow_execution(wf_ex.id)
self.assertIsNotNone(wf_ex)
self.assertEqual(states.SUCCESS, wf_ex.state)
self.assertIsInstance(task2_action_ex, models.ActionExecution)
self.assertEqual('std.echo', task2_action_ex.name)
self.assertEqual(states.SUCCESS, task2_action_ex.state)
# Data Flow properties.
self.assertIn('__tasks', task2_ex.in_context)
self.assertIn('__execution', task1_ex.in_context)
self.assertDictEqual({'output': 'Hi'}, task2_action_ex.input)
self.assertDictEqual({}, task2_ex.published)
self.assertDictEqual({'output': 'Hi'}, task2_action_ex.input)
self.assertDictEqual({'result': 'Hi'}, task2_action_ex.output)
self.assertEqual(2, len(wf_ex.task_executions))
self._assert_single_item(wf_ex.task_executions, name='task1')
self._assert_single_item(wf_ex.task_executions, name='task2')
