def test_set_current_task_empty_context(self):
task = {"id": "t1", "route": 0}
context = ctx_util.set_current_task(dict(), task)
expected_context = {"__current_task": json_util.deepcopy(task)}
self.assertDictEqual(context, expected_context)
def test_set_current_task_empty_context(self):
task = {'id': 't1', 'route': 0}
context = ctx_util.set_current_task(dict(), task)
expected_context = {'__current_task': copy.deepcopy(task)}
self.assertDictEqual(context, expected_context)
def test_set_current_task_nonetype_context(self):
task = {'id': 't1', 'route': 0}
context = ctx_util.set_current_task(None, task)
expected_context = {'__current_task': json_util.deepcopy(task)}
self.assertDictEqual(context, expected_context)
def test_set_current_task_nonetype_context(self):
task = {'id': 't1_1', 'name': 't1'}
context = ctx.set_current_task(None, task)
expected_context = {'__current_task': copy.deepcopy(task)}
self.assertDictEqual(context, expected_context)
def test_set_current_task_with_result(self):
context = {'var1': 'foobar'}
task = {'id': 't1_1', 'name': 't1', 'result': 'foobar'}
context = ctx.set_current_task(context, task)
expected_context = dict([('__current_task', copy.deepcopy(task))] + list(context.items()))
self.assertDictEqual(context, expected_context)
def test_set_current_task(self):
context = {'var1': 'foobar'}
task = {'id': 't1', 'route': 0}
context = ctx_util.set_current_task(context, task)
expected_context = dict([('__current_task', copy.deepcopy(task))] +
list(context.items()))
self.assertDictEqual(context, expected_context)
def test_set_current_task_with_result(self):
context = {"var1": "foobar"}
task = {"id": "t1", "route": 0, "result": "foobar"}
context = ctx_util.set_current_task(context, task)
expected_context = dict(
[("__current_task", json_util.deepcopy(task))] + list(context.items())
)
self.assertDictEqual(context, expected_context)
def make_task_context(self, task_state_entry, task_result=None):
in_ctx_idxs = task_state_entry['ctxs']['in']
in_ctx_val = self.get_task_context(in_ctx_idxs)
current_task = {
'id': task_state_entry['id'],
'route': task_state_entry['route'],
'result': task_result
}
current_ctx = ctx_util.set_current_task(in_ctx_val, current_task)
state_ctx = {'__state': self.workflow_state.serialize()}
current_ctx = dict_util.merge_dicts(current_ctx, state_ctx, True)
return current_ctx
def get_task(self, task_id, route):
try:
task_ctx = self.get_task_initial_context(task_id, route)
except ValueError:
task_ctx = self.get_workflow_initial_context()
state_ctx = {'__state': self.workflow_state.serialize()}
current_task = {'id': task_id, 'route': route}
task_ctx = ctx_util.set_current_task(task_ctx, current_task)
task_ctx = dict_util.merge_dicts(task_ctx, state_ctx, True)
task_spec = self.spec.tasks.get_task(task_id).copy()
task_spec, action_specs = task_spec.render(task_ctx)
task = {
'id': task_id,
'route': route,
'ctx': task_ctx,
'spec': task_spec,
'actions': action_specs
}
# If there is a task delay specified, evaluate the delay value.
if getattr(task_spec, 'delay', None):
task_delay = task_spec.delay
if isinstance(task_delay, six.string_types):
task_delay = expr_base.evaluate(task_delay, task_ctx)
if not isinstance(task_delay, int):
raise TypeError(
'The value of task delay is not type of integer.')
task['delay'] = task_delay
# Add items and related meta data to the task details.
if task_spec.has_items():
items_spec = getattr(task_spec, 'with')
concurrency = getattr(items_spec, 'concurrency', None)
task['items_count'] = len(action_specs)
task['concurrency'] = expr_base.evaluate(concurrency, task_ctx)
return task
def get_task(self, task_id):
task_node = self.graph.get_task(task_id)
task_name = task_node['name']
try:
task_ctx = self.get_task_initial_context(task_id)['value']
except ValueError:
task_ctx = self.get_workflow_initial_context()
current_task = {'id': task_id, 'name': task_name}
task_ctx = ctx.set_current_task(task_ctx, current_task)
task_spec = self.spec.tasks.get_task(task_name).copy()
task_spec, action_specs = task_spec.render(task_ctx)
task = {
'id': task_id,
'name': task_name,
'ctx': task_ctx,
'spec': task_spec,
'actions': action_specs
}
# If there is a task delay specified, evaluate the delay value.
if getattr(task_spec, 'delay', None):
task_delay = task_spec.delay
if isinstance(task_delay, six.string_types):
task_delay = expr.evaluate(task_delay, task_ctx)
if not isinstance(task_delay, int):
raise TypeError('The value of task delay is not type of integer.')
task['delay'] = task_delay
# Add items and related meta data to the task details.
if task_spec.has_items():
items_spec = getattr(task_spec, 'with')
task['items_count'] = len(action_specs)
task['concurrency'] = expr.evaluate(getattr(items_spec, 'concurrency', None), task_ctx)
return task
def update_task_flow(self, task_id, event):
in_ctx_idx = 0
engine_event_queue = queue.Queue()
# Throw exception if not expected event type.
if not issubclass(type(event), events.ExecutionEvent):
raise TypeError('Event is not type of ExecutionEvent.')
# Throw exception if task does not exist in the workflow graph.
if not self.graph.has_task(task_id):
raise exc.InvalidTask(task_id)
# Try to get the task flow entry.
task_flow_entry = self.get_task_flow_entry(task_id)
# Throw exception if task is not staged and there is no task flow entry.
if task_id not in self.flow.staged and not task_flow_entry:
raise exc.InvalidTaskFlowEntry(task_id)
# Get the incoming context from the staged task.
if task_id in self.flow.staged:
in_ctx_idxs = self.flow.staged[task_id]['ctxs']
if len(in_ctx_idxs) <= 0 or all(x == in_ctx_idxs[0] for x in in_ctx_idxs):
in_ctx_idx = in_ctx_idxs[0]
else:
new_ctx_entry = self._converge_task_contexts(in_ctx_idxs)
self.flow.contexts.append(new_ctx_entry)
in_ctx_idx = len(self.flow.contexts) - 1
# Create new task flow entry if it does not exist.
if not task_flow_entry:
task_flow_entry = self.add_task_flow(task_id, in_ctx_idx=in_ctx_idx)
# If task is already completed and in cycle, then create new task flow entry.
if self.graph.in_cycle(task_id) and task_flow_entry.get('state') in states.COMPLETED_STATES:
task_flow_entry = self.add_task_flow(task_id, in_ctx_idx=in_ctx_idx)
# Remove task from staging if task is not with items.
if event.state and task_id in self.flow.staged and 'items' not in self.flow.staged[task_id]:
del self.flow.staged[task_id]
# If action execution is for a task item, then store the execution state for the item.
if (event.state and event.context and
'item_id' in event.context and event.context['item_id'] is not None):
item_result = {'state': event.state, 'result': event.result}
self.flow.staged[task_id]['items'][event.context['item_id']] = item_result
# Log the error if it is a failed execution event.
if event.state == states.FAILED:
message = 'Execution failed. See result for details.'
self.log_entry('error', message, task_id=task_id, result=event.result)
# Process the action execution event using the task state machine and update the task state.
old_task_state = task_flow_entry.get('state', states.UNSET)
machines.TaskStateMachine.process_event(self, task_flow_entry, event)
new_task_state = task_flow_entry.get('state', states.UNSET)
# Get task result and set current context if task is completed.
if new_task_state in states.COMPLETED_STATES:
# Get task details required for updating outgoing context.
task_node = self.graph.get_task(task_id)
task_name = task_node['name']
task_spec = self.spec.tasks.get_task(task_name)
task_flow_idx = self._get_task_flow_idx(task_id)
# Get task result.
task_result = (
[item.get('result') for item in self.flow.staged[task_id]['items']]
if task_spec.has_items() else event.result
)
# Remove remaining task from staging.
self.flow.remove_staged_task(task_id)
# Set current task in the context.
in_ctx_idx = task_flow_entry['ctx']
in_ctx_val = self.flow.contexts[in_ctx_idx]['value']
current_task = {'id': task_id, 'name': task_name, 'result': task_result}
current_ctx = ctx.set_current_task(in_ctx_val, current_task)
# Setup context for evaluating expressions in task transition criteria.
flow_ctx = {'__flow': self.flow.serialize()}
current_ctx = dx.merge_dicts(current_ctx, flow_ctx, True)
# Evaluate task transitions if task is completed and state change is not processed.
if new_task_state in states.COMPLETED_STATES and new_task_state != old_task_state:
# Identify task transitions for the current completed task.
task_transitions = self.graph.get_next_transitions(task_id)
# Update workflow context when there is no transitions.
if not task_transitions:
self._update_workflow_terminal_context(in_ctx_val, task_flow_idx)
# Iterate thru each outbound task transitions.
for task_transition in task_transitions:
task_transition_id = task_transition[1] + '__' + str(task_transition[2])
# Evaluate the criteria for task transition. If there is a failure while
# evaluating expression(s), fail the workflow.
try:
criteria = task_transition[3].get('criteria') or []
evaluated_criteria = [expr.evaluate(c, current_ctx) for c in criteria]
task_flow_entry[task_transition_id] = all(evaluated_criteria)
except Exception as e:
self.log_error(e, task_id, task_transition_id)
self.request_workflow_state(states.FAILED)
continue
# If criteria met, then mark the next task staged and calculate outgoing context.
if task_flow_entry[task_transition_id]:
next_task_node = self.graph.get_task(task_transition[1])
next_task_name = next_task_node['name']
next_task_id = next_task_node['id']
out_ctx_val, errors = task_spec.finalize_context(
next_task_name,
task_transition,
copy.deepcopy(current_ctx)
)
if errors:
self.log_errors(errors, task_id, task_transition_id)
self.request_workflow_state(states.FAILED)
continue
if out_ctx_val != in_ctx_val:
task_flow_idx = self._get_task_flow_idx(task_id)
self.flow.contexts.append({'srcs': [task_flow_idx], 'value': out_ctx_val})
out_ctx_idx = len(self.flow.contexts) - 1
else:
out_ctx_idx = in_ctx_idx
# Check if inbound criteria are met.
ready = self._inbound_criteria_satisfied(task_transition[1])
if (task_transition[1] in self.flow.staged and
'ctxs' in self.flow.staged[task_transition[1]]):
self.flow.staged[task_transition[1]]['ctxs'].append(out_ctx_idx)
self.flow.staged[task_transition[1]]['ready'] = ready
else:
staging_data = {'ctxs': [out_ctx_idx], 'ready': ready}
self.flow.staged[task_transition[1]] = staging_data
# If the next task is noop, then mark the task as completed.
if next_task_name in events.ENGINE_EVENT_MAP.keys():
engine_event_queue.put((next_task_id, next_task_name))
# Process the task event using the workflow state machine and update the workflow state.
task_ex_event = events.TaskExecutionEvent(task_id, task_flow_entry['state'])
machines.WorkflowStateMachine.process_event(self, task_ex_event)
# Process any engine commands in the queue.
while not engine_event_queue.empty():
next_task_id, next_task_name = engine_event_queue.get()
engine_event = events.ENGINE_EVENT_MAP[next_task_name]
self.update_task_flow(next_task_id, engine_event())
# Render workflow output if workflow is completed.
if self.get_workflow_state() in states.COMPLETED_STATES:
in_ctx_idx = task_flow_entry['ctx']
in_ctx_val = self.flow.contexts[in_ctx_idx]['value']
task_flow_idx = self._get_task_flow_idx(task_id)
self._update_workflow_terminal_context(in_ctx_val, task_flow_idx)
self._render_workflow_outputs()
return task_flow_entry
def update_task_state(self, task_id, route, event):
engine_event_queue = queue.Queue()
# Throw exception if not expected event type.
if not issubclass(type(event), events.ExecutionEvent):
raise TypeError('Event is not type of ExecutionEvent.')
# Throw exception if task does not exist in the workflow graph.
if not self.graph.has_task(task_id):
raise exc.InvalidTask(task_id)
# Try to get the task metadata from staging or task state.
staged_task = self.workflow_state.get_staged_task(task_id, route)
task_state_entry = self.get_task_state_entry(task_id, route)
# Throw exception if task is not staged and there is no task state entry.
if not staged_task and not task_state_entry:
raise exc.InvalidTaskStateEntry(task_id)
# Create new task state entry if it does not exist.
if not task_state_entry:
task_state_entry = self.add_task_state(
task_id,
staged_task['route'],
in_ctx_idxs=staged_task['ctxs']['in'],
prev=staged_task['prev'])
# Identify the index for the task state object for later use.
task_state_idx = self._get_task_state_idx(task_id, route)
# If task is already completed and in cycle, then create new task state entry.
if (self.graph.in_cycle(task_id) and task_state_entry.get('status')
in statuses.COMPLETED_STATUSES):
task_state_entry = self.add_task_state(
task_id,
staged_task['route'],
in_ctx_idxs=staged_task['ctxs']['in'],
prev=staged_task['prev'])
# Update the index value since a new entry is created.
task_state_idx = self._get_task_state_idx(task_id, route)
# Remove task from staging if task is not with items.
if event.status and staged_task and 'items' not in staged_task:
self.workflow_state.remove_staged_task(task_id, route)
# If action execution is for a task item, then store the execution status for the item.
if (staged_task and event.status and event.context
and 'item_id' in event.context
and event.context['item_id'] is not None):
item_result = {'status': event.status, 'result': event.result}
staged_task['items'][event.context['item_id']] = item_result
# Log the error if it is a failed execution event.
if event.status == statuses.FAILED:
message = 'Execution failed. See result for details.'
self.log_entry('error',
message,
task_id=task_id,
result=event.result)
# Process the action execution event using the
# task state machine and update the task status.
old_task_status = task_state_entry.get('status', statuses.UNSET)
machines.TaskStateMachine.process_event(self.workflow_state,
task_state_entry, event)
new_task_status = task_state_entry.get('status', statuses.UNSET)
# Get task result and set current context if task is completed.
if new_task_status in statuses.COMPLETED_STATUSES:
# Get task details required for updating outgoing context.
task_spec = self.spec.tasks.get_task(task_id)
# Get task result.
task_result = ([
item.get('result') for item in staged_task.get('items', [])
] if staged_task and task_spec.has_items() else event.result)
# Remove remaining task from staging.
self.workflow_state.remove_staged_task(task_id, route)
# Set current task in the context.
in_ctx_idxs = task_state_entry['ctxs']['in']
in_ctx_val = self.get_task_context(in_ctx_idxs)
current_task = {
'id': task_id,
'route': route,
'result': task_result
}
current_ctx = ctx_util.set_current_task(in_ctx_val, current_task)
# Setup context for evaluating expressions in task transition criteria.
state_ctx = {'__state': self.workflow_state.serialize()}
current_ctx = dict_util.merge_dicts(current_ctx, state_ctx, True)
# Evaluate task transitions if task is completed and status change is not processed.
if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status:
has_manual_fail = False
staged_next_tasks = []
# Identify task transitions for the current completed task.
task_transitions = self.graph.get_next_transitions(task_id)
# Mark task as terminal when there is no transitions.
if not task_transitions:
task_state_entry['term'] = True
# Iterate thru each outbound task transitions.
for task_transition in task_transitions:
task_transition_id = (
constants.TASK_STATE_TRANSITION_FORMAT %
(task_transition[1], str(task_transition[2])))
# Evaluate the criteria for task transition. If there is a failure while
# evaluating expression(s), fail the workflow.
try:
criteria = task_transition[3].get('criteria') or []
evaluated_criteria = [
expr_base.evaluate(c, current_ctx) for c in criteria
]
task_state_entry['next'][task_transition_id] = all(
evaluated_criteria)
except Exception as e:
self.log_error(e, task_id, route, task_transition_id)
self.request_workflow_status(statuses.FAILED)
continue
# If criteria met, then mark the next task staged and calculate outgoing context.
if task_state_entry['next'][task_transition_id]:
next_task_node = self.graph.get_task(task_transition[1])
next_task_id = next_task_node['id']
new_ctx_idx = None
# Get and process new context for the task transition.
out_ctx, new_ctx, errors = task_spec.finalize_context(
next_task_id, task_transition,
copy.deepcopy(current_ctx))
if errors:
self.log_errors(errors, task_id, route,
task_transition_id)
self.request_workflow_status(statuses.FAILED)
continue
out_ctx_idxs = copy.deepcopy(
task_state_entry['ctxs']['in'])
if new_ctx:
self.workflow_state.contexts.append(new_ctx)
new_ctx_idx = len(self.workflow_state.contexts) - 1
# Add to the list of contexts for the next task in this transition.
out_ctx_idxs.append(new_ctx_idx)
# Record the outgoing context for this task transition.
if 'out' not in task_state_entry['ctxs']:
task_state_entry['ctxs']['out'] = {}
task_state_entry['ctxs']['out'] = {
task_transition_id: new_ctx_idx
}
# Stage the next task if it is not in staging.
next_task_route = self._evaluate_route(
task_transition, route)
staged_next_task = self.workflow_state.get_staged_task(
next_task_id, next_task_route)
backref = (constants.TASK_STATE_TRANSITION_FORMAT %
(task_id, str(task_transition[2])))
# If the next task is already staged.
if staged_next_task:
# Remove the root context to avoid overwriting vars.
out_ctx_idxs.remove(0)
# Extend the outgoing context from this task.
staged_next_task['ctxs']['in'].extend(out_ctx_idxs)
# Add a backref for the current task in the next task.
staged_next_task['prev'][backref] = task_state_idx
else:
# Otherwise create a new entry in staging for the next task.
staged_next_task = self.workflow_state.add_staged_task(
next_task_id,
next_task_route,
ctxs=out_ctx_idxs,
prev={backref: task_state_idx},
ready=False)
# Check if inbound criteria are met. Must use the original route
# to identify the inbound task transitions.
staged_next_task[
'ready'] = self._inbound_criteria_satisfied(
next_task_id, route)
# Put the next task in the engine event queue if it is an engine command.
if next_task_id in events.ENGINE_EVENT_MAP.keys():
queue_entry = (staged_next_task['id'],
staged_next_task['route'])
engine_event_queue.put(queue_entry)
# Flag if there is at least one fail command in the task transition.
if not has_manual_fail:
has_manual_fail = (next_task_id == 'fail')
else:
# If not an engine command and the next task is ready, then
# make a record of it for processing manual fail below.
if staged_next_task['ready']:
staged_next_tasks.append(staged_next_task)
# Task failure is remediable. For example, there may be workflow that wants
# to run a cleanup task on failure. In certain cases, we still want to fail
# the workflow after the remediation. The fail command can be in the
# task transition under the cleanup task. However, the cleanup task may be
# reusable when either workflow succeed or fail. It does not make sense to
# put the fail command in the task transition of the cleanup task. So we
# want to allow user to be able to define both the cleanup and fail in the
# task transition under the current task but still return the cleanup task
# even when the workflow is set to failed status.
if has_manual_fail:
for staged_next_task in staged_next_tasks:
staged_next_task['run_on_fail'] = True
# Process the task event using the workflow state machine and update the workflow status.
task_ex_event = events.TaskExecutionEvent(task_id, route,
task_state_entry['status'])
machines.WorkflowStateMachine.process_event(self.workflow_state,
task_ex_event)
# Process any engine commands in the queue.
while not engine_event_queue.empty():
next_task_id, next_task_route = engine_event_queue.get()
engine_event = events.ENGINE_EVENT_MAP[next_task_id]
self.update_task_state(next_task_id, next_task_route,
engine_event())
# Render workflow output if workflow is completed.
if self.get_workflow_status() in statuses.COMPLETED_STATUSES:
task_state_entry['term'] = True
self._render_workflow_outputs()
return task_state_entry
def assert_conducting_sequences(self,
wf_name,
expected_task_seq,
inputs=None,
mock_states=None,
mock_results=None,
expected_workflow_state=None,
expected_output=None):
if inputs is None:
inputs = {}
wf_def = self.get_wf_def(wf_name)
wf_spec = self.spec_module.instantiate(wf_def)
conductor = conducting.WorkflowConductor(wf_spec, inputs=inputs)
conductor.request_workflow_state(states.RUNNING)
context = {}
q = queue.Queue()
state_q = queue.Queue()
result_q = queue.Queue()
if mock_states:
for item in mock_states:
state_q.put(item)
if mock_results:
for item in mock_results:
result_q.put(item)
# Get start tasks and being conducting workflow.
for task in conductor.get_next_tasks():
q.put(task)
# Serialize workflow conductor to mock async execution.
wf_conducting_state = conductor.serialize()
while not q.empty():
current_task = q.get()
current_task_id = current_task['id']
# Deserialize workflow conductor to mock async execution.
conductor = conducting.WorkflowConductor.deserialize(
wf_conducting_state)
# Set task state to running.
ac_ex_event = events.ActionExecutionEvent(states.RUNNING)
conductor.update_task_flow(current_task_id, ac_ex_event)
# Set current task in context.
context = ctx.set_current_task(context, current_task)
# Mock completion of the task.
state = state_q.get() if not state_q.empty() else states.SUCCEEDED
result = result_q.get() if not result_q.empty() else None
ac_ex_event = events.ActionExecutionEvent(state, result=result)
conductor.update_task_flow(current_task_id, ac_ex_event)
# Identify the next set of tasks.
next_tasks = conductor.get_next_tasks(current_task_id)
for next_task in next_tasks:
q.put(next_task)
# Serialize workflow execution graph to mock async execution.
wf_conducting_state = conductor.serialize()
self.assertListEqual(
expected_task_seq,
[entry['id'] for entry in conductor.flow.sequence])
if expected_workflow_state is not None:
self.assertEqual(conductor.get_workflow_state(),
expected_workflow_state)
if expected_output is not None:
self.assertDictEqual(conductor.get_workflow_output(),
expected_output)
