def get_task_transition_contexts(self, task_id, route):
contexts = {}
task_state_entry = self.get_task_state_entry(task_id, route)
if not task_state_entry:
raise exc.InvalidTaskStateEntry(task_id)
for t in self.graph.get_next_transitions(task_id):
task_transition_id = constants.TASK_STATE_TRANSITION_FORMAT % (t[1], str(t[2]))
if (task_transition_id in task_state_entry['next'] and
task_state_entry['next'][task_transition_id]):
contexts[task_transition_id] = self.get_task_initial_context(t[1], route)
return contexts
def update_task_state(self, task_id, route, event):
engine_event_queue = queue.Queue()
# Throw exception if not expected event type.
if not issubclass(type(event), events.ExecutionEvent):
raise TypeError('Event is not type of ExecutionEvent.')
# Throw exception if task does not exist in the workflow graph.
if not self.graph.has_task(task_id):
raise exc.InvalidTask(task_id)
# Try to get the task metadata from staging or task state.
staged_task = self.workflow_state.get_staged_task(task_id, route)
task_state_entry = self.get_task_state_entry(task_id, route)
# Throw exception if task is not staged and there is no task state entry.
if not staged_task and not task_state_entry:
raise exc.InvalidTaskStateEntry(task_id)
# Create new task state entry if it does not exist.
if not task_state_entry:
task_state_entry = self.add_task_state(
task_id,
staged_task['route'],
in_ctx_idxs=staged_task['ctxs']['in'],
prev=staged_task['prev'])
# Identify the index for the task state object for later use.
task_state_idx = self._get_task_state_idx(task_id, route)
# If task is already completed and in cycle, then create new task state entry.
if (self.graph.in_cycle(task_id) and task_state_entry.get('status')
in statuses.COMPLETED_STATUSES):
task_state_entry = self.add_task_state(
task_id,
staged_task['route'],
in_ctx_idxs=staged_task['ctxs']['in'],
prev=staged_task['prev'])
# Update the index value since a new entry is created.
task_state_idx = self._get_task_state_idx(task_id, route)
# Remove task from staging if task is not with items.
if event.status and staged_task and 'items' not in staged_task:
self.workflow_state.remove_staged_task(task_id, route)
# If action execution is for a task item, then store the execution status for the item.
if (staged_task and event.status and event.context
and 'item_id' in event.context
and event.context['item_id'] is not None):
item_result = {'status': event.status, 'result': event.result}
staged_task['items'][event.context['item_id']] = item_result
# Log the error if it is a failed execution event.
if event.status == statuses.FAILED:
message = 'Execution failed. See result for details.'
self.log_entry('error',
message,
task_id=task_id,
result=event.result)
# Process the action execution event using the
# task state machine and update the task status.
old_task_status = task_state_entry.get('status', statuses.UNSET)
machines.TaskStateMachine.process_event(self.workflow_state,
task_state_entry, event)
new_task_status = task_state_entry.get('status', statuses.UNSET)
# Get task result and set current context if task is completed.
if new_task_status in statuses.COMPLETED_STATUSES:
# Get task details required for updating outgoing context.
task_spec = self.spec.tasks.get_task(task_id)
# Get task result.
task_result = ([
item.get('result') for item in staged_task.get('items', [])
] if staged_task and task_spec.has_items() else event.result)
# Remove remaining task from staging.
self.workflow_state.remove_staged_task(task_id, route)
# Set current task in the context.
in_ctx_idxs = task_state_entry['ctxs']['in']
in_ctx_val = self.get_task_context(in_ctx_idxs)
current_task = {
'id': task_id,
'route': route,
'result': task_result
}
current_ctx = ctx_util.set_current_task(in_ctx_val, current_task)
# Setup context for evaluating expressions in task transition criteria.
state_ctx = {'__state': self.workflow_state.serialize()}
current_ctx = dict_util.merge_dicts(current_ctx, state_ctx, True)
# Evaluate task transitions if task is completed and status change is not processed.
if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status:
has_manual_fail = False
staged_next_tasks = []
# Identify task transitions for the current completed task.
task_transitions = self.graph.get_next_transitions(task_id)
# Mark task as terminal when there is no transitions.
if not task_transitions:
task_state_entry['term'] = True
# Iterate thru each outbound task transitions.
for task_transition in task_transitions:
task_transition_id = (
constants.TASK_STATE_TRANSITION_FORMAT %
(task_transition[1], str(task_transition[2])))
# Evaluate the criteria for task transition. If there is a failure while
# evaluating expression(s), fail the workflow.
try:
criteria = task_transition[3].get('criteria') or []
evaluated_criteria = [
expr_base.evaluate(c, current_ctx) for c in criteria
]
task_state_entry['next'][task_transition_id] = all(
evaluated_criteria)
except Exception as e:
self.log_error(e, task_id, route, task_transition_id)
self.request_workflow_status(statuses.FAILED)
continue
# If criteria met, then mark the next task staged and calculate outgoing context.
if task_state_entry['next'][task_transition_id]:
next_task_node = self.graph.get_task(task_transition[1])
next_task_id = next_task_node['id']
new_ctx_idx = None
# Get and process new context for the task transition.
out_ctx, new_ctx, errors = task_spec.finalize_context(
next_task_id, task_transition,
copy.deepcopy(current_ctx))
if errors:
self.log_errors(errors, task_id, route,
task_transition_id)
self.request_workflow_status(statuses.FAILED)
continue
out_ctx_idxs = copy.deepcopy(
task_state_entry['ctxs']['in'])
if new_ctx:
self.workflow_state.contexts.append(new_ctx)
new_ctx_idx = len(self.workflow_state.contexts) - 1
# Add to the list of contexts for the next task in this transition.
out_ctx_idxs.append(new_ctx_idx)
# Record the outgoing context for this task transition.
if 'out' not in task_state_entry['ctxs']:
task_state_entry['ctxs']['out'] = {}
task_state_entry['ctxs']['out'] = {
task_transition_id: new_ctx_idx
}
# Stage the next task if it is not in staging.
next_task_route = self._evaluate_route(
task_transition, route)
staged_next_task = self.workflow_state.get_staged_task(
next_task_id, next_task_route)
backref = (constants.TASK_STATE_TRANSITION_FORMAT %
(task_id, str(task_transition[2])))
# If the next task is already staged.
if staged_next_task:
# Remove the root context to avoid overwriting vars.
out_ctx_idxs.remove(0)
# Extend the outgoing context from this task.
staged_next_task['ctxs']['in'].extend(out_ctx_idxs)
# Add a backref for the current task in the next task.
staged_next_task['prev'][backref] = task_state_idx
else:
# Otherwise create a new entry in staging for the next task.
staged_next_task = self.workflow_state.add_staged_task(
next_task_id,
next_task_route,
ctxs=out_ctx_idxs,
prev={backref: task_state_idx},
ready=False)
# Check if inbound criteria are met. Must use the original route
# to identify the inbound task transitions.
staged_next_task[
'ready'] = self._inbound_criteria_satisfied(
next_task_id, route)
# Put the next task in the engine event queue if it is an engine command.
if next_task_id in events.ENGINE_EVENT_MAP.keys():
queue_entry = (staged_next_task['id'],
staged_next_task['route'])
engine_event_queue.put(queue_entry)
# Flag if there is at least one fail command in the task transition.
if not has_manual_fail:
has_manual_fail = (next_task_id == 'fail')
else:
# If not an engine command and the next task is ready, then
# make a record of it for processing manual fail below.
if staged_next_task['ready']:
staged_next_tasks.append(staged_next_task)
# Task failure is remediable. For example, there may be workflow that wants
# to run a cleanup task on failure. In certain cases, we still want to fail
# the workflow after the remediation. The fail command can be in the
# task transition under the cleanup task. However, the cleanup task may be
# reusable when either workflow succeed or fail. It does not make sense to
# put the fail command in the task transition of the cleanup task. So we
# want to allow user to be able to define both the cleanup and fail in the
# task transition under the current task but still return the cleanup task
# even when the workflow is set to failed status.
if has_manual_fail:
for staged_next_task in staged_next_tasks:
staged_next_task['run_on_fail'] = True
# Process the task event using the workflow state machine and update the workflow status.
task_ex_event = events.TaskExecutionEvent(task_id, route,
task_state_entry['status'])
machines.WorkflowStateMachine.process_event(self.workflow_state,
task_ex_event)
# Process any engine commands in the queue.
while not engine_event_queue.empty():
next_task_id, next_task_route = engine_event_queue.get()
engine_event = events.ENGINE_EVENT_MAP[next_task_id]
self.update_task_state(next_task_id, next_task_route,
engine_event())
# Render workflow output if workflow is completed.
if self.get_workflow_status() in statuses.COMPLETED_STATUSES:
task_state_entry['term'] = True
self._render_workflow_outputs()
return task_state_entry
def update_task_state(self, task_id, route, event):
engine_event_queue = queue.Queue()
# Throw exception if not expected event type.
if not issubclass(type(event), events.ExecutionEvent):
raise TypeError('Event is not type of ExecutionEvent.')
# Throw exception if task does not exist in the workflow graph.
if not self.graph.has_task(task_id):
raise exc.InvalidTask(task_id)
# Try to get the task metadata from staging or task state.
staged_task = self.workflow_state.get_staged_task(task_id, route)
task_state_entry = self.get_task_state_entry(task_id, route)
# Get the task spec for the task which contains additional meta data.
task_spec = self.spec.tasks.get_task(task_id)
# Throw exception if task is not staged and there is no task state entry.
if not staged_task and not task_state_entry:
raise exc.InvalidTaskStateEntry(task_id)
# Create new task state entry if it does not exist or if it is an engine command.
if not task_state_entry or task_id in events.ENGINE_EVENT_MAP.keys():
task_state_entry = self.add_task_state(
task_id,
staged_task['route'],
in_ctx_idxs=staged_task['ctxs']['in'],
prev=staged_task['prev']
)
# Identify the index for the task state object for later use.
task_state_idx = self._get_task_state_idx(task_id, route)
# If task is already completed and in cycle, then create new task state entry.
# Unfortunately, the method in the graph to check for cycle is too simple and
# misses forks that extends from the cycle. The check here assumes that the
# last task entry is already completed and the new task status is one of the
# starting statuses, then there is high likelihood that this is a cycle.
if (task_state_entry.get('status') in statuses.COMPLETED_STATUSES and
event.status and event.status in statuses.STARTING_STATUSES):
task_state_entry = self.add_task_state(
task_id,
staged_task['route'],
in_ctx_idxs=staged_task['ctxs']['in'],
prev=staged_task['prev']
)
# Update the index value since a new entry is created.
task_state_idx = self._get_task_state_idx(task_id, route)
# Remove task from staging if task is not with items.
if event.status and staged_task and 'items' not in staged_task:
self.workflow_state.remove_staged_task(task_id, route)
# If action execution is for a task item, then record the execution status for the item.
# Result for each item is not recorded in the staged_task because it impacts database
# write performance if there are a lot of items and/or item result size is huge.
if staged_task and isinstance(event, events.TaskItemActionExecutionEvent):
staged_task['items'][event.item_id] = {'status': event.status}
# Log the error if it is a failed execution event.
if event.status == statuses.FAILED:
message = 'Execution failed. See result for details.'
self.log_entry('error', message, task_id=task_id, result=event.result)
# Process the action execution event using the
# task state machine and update the task status.
old_task_status = task_state_entry.get('status', statuses.UNSET)
machines.TaskStateMachine.process_event(self.workflow_state, task_state_entry, event)
new_task_status = task_state_entry.get('status', statuses.UNSET)
# If retrying, staged the task to be returned in get_next_tasks.
if new_task_status == statuses.RETRYING:
# Increment the number of times that the task has retried.
task_state_entry['retry']['tally'] += 1
# Reset the staged task to be returned in get_next_tasks
self.workflow_state.remove_staged_task(task_id, route)
self.workflow_state.add_staged_task(
task_id,
route,
ctxs=task_state_entry['ctxs']['in'],
prev=task_state_entry['prev'],
retry=task_state_entry['retry'],
ready=True
)
# Get task result and set current context if task is completed.
if new_task_status in statuses.COMPLETED_STATUSES:
# Remove task from staging if exists but keep entry
# if task has items and failed for manual rerun.
if not (task_spec.has_items() and new_task_status in statuses.ABENDED_STATUSES):
self.workflow_state.remove_staged_task(task_id, route)
# Format task result depending on the type of task.
task_result = self.make_task_result(task_spec, event)
# Set current task in the context.
current_ctx = self.make_task_context(task_state_entry, task_result=task_result)
# Evaluate if there is a task retry. The task retry can only be evaluated after
# the state machine has determined the status for the task execution. If the task
# is completed, get the task result and context which is required to evaluate the
# the condition if a retry for the task is required.
if (self.get_workflow_status() in statuses.ACTIVE_STATUSES and
self._evaluate_task_retry(task_state_entry, current_ctx)):
self.update_task_state(task_id, route, events.TaskRetryEvent())
# Evaluate task transitions if task is completed and status change is not processed.
if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status:
has_manual_fail = False
staged_next_tasks = []
# Identify task transitions for the current completed task.
task_transitions = self.graph.get_next_transitions(task_id)
# Mark task as terminal when there is no transitions.
if not task_transitions:
task_state_entry['term'] = True
# Iterate thru each outbound task transitions.
for task_transition in task_transitions:
task_transition_id = (
constants.TASK_STATE_TRANSITION_FORMAT %
(task_transition[1], str(task_transition[2]))
)
# Evaluate the criteria for task transition. If there is a failure while
# evaluating expression(s), fail the workflow.
try:
criteria = task_transition[3].get('criteria') or []
evaluated_criteria = [expr_base.evaluate(c, current_ctx) for c in criteria]
task_state_entry['next'][task_transition_id] = all(evaluated_criteria)
except Exception as e:
self.log_error(e, task_id, route, task_transition_id)
self.request_workflow_status(statuses.FAILED)
continue
# If criteria met, then mark the next task staged and calculate outgoing context.
if task_state_entry['next'][task_transition_id]:
next_task_node = self.graph.get_task(task_transition[1])
next_task_id = next_task_node['id']
new_ctx_idx = None
# Get and process new context for the task transition.
out_ctx, new_ctx, errors = task_spec.finalize_context(
next_task_id,
task_transition,
json_util.deepcopy(current_ctx)
)
if errors:
self.log_errors(errors, task_id, route, task_transition_id)
self.request_workflow_status(statuses.FAILED)
continue
out_ctx_idxs = json_util.deepcopy(task_state_entry['ctxs']['in'])
if new_ctx:
self.workflow_state.contexts.append(new_ctx)
new_ctx_idx = len(self.workflow_state.contexts) - 1
# Add to the list of contexts for the next task in this transition.
out_ctx_idxs.append(new_ctx_idx)
# Record the outgoing context for this task transition.
if 'out' not in task_state_entry['ctxs']:
task_state_entry['ctxs']['out'] = {}
task_state_entry['ctxs']['out'] = {task_transition_id: new_ctx_idx}
# Stage the next task if it is not in staging.
next_task_route = self._evaluate_route(task_transition, route)
staged_next_task = self.workflow_state.get_staged_task(
next_task_id,
next_task_route
)
backref = (
constants.TASK_STATE_TRANSITION_FORMAT %
(task_id, str(task_transition[2]))
)
# If the next task is already staged.
if staged_next_task:
# Remove the root context to avoid overwriting vars.
out_ctx_idxs.remove(0)
# Extend the outgoing context from this task.
staged_next_task['ctxs']['in'].extend(out_ctx_idxs)
# Add a backref for the current task in the next task.
staged_next_task['prev'][backref] = task_state_idx
# Clear list of items for with items task.
staged_next_task.pop('items', None)
else:
# Otherwise create a new entry in staging for the next task.
staged_next_task = self.workflow_state.add_staged_task(
next_task_id,
next_task_route,
ctxs=out_ctx_idxs,
prev={backref: task_state_idx},
ready=False
)
# Check if inbound criteria are met. Must use the original route
# to identify the inbound task transitions.
staged_next_task['ready'] = self._inbound_criteria_satisfied(
next_task_id,
route
)
# Put the next task in the engine event queue if it is an engine command.
if next_task_id in events.ENGINE_EVENT_MAP.keys():
queue_entry = (staged_next_task['id'], staged_next_task['route'])
engine_event_queue.put(queue_entry)
# Flag if there is at least one fail command in the task transition.
if not has_manual_fail:
has_manual_fail = (next_task_id == 'fail')
else:
# If not an engine command and the next task is ready, then
# make a record of it for processing manual fail below.
if staged_next_task['ready']:
staged_next_tasks.append(staged_next_task)
# Task failure is remediable. For example, there may be workflow that wants
# to run a cleanup task on failure. In certain cases, we still want to fail
# the workflow after the remediation. The fail command can be in the
# task transition under the cleanup task. However, the cleanup task may be
# reusable when either workflow succeed or fail. It does not make sense to
# put the fail command in the task transition of the cleanup task. So we
# want to allow user to be able to define both the cleanup and fail in the
# task transition under the current task but still return the cleanup task
# even when the workflow is set to failed status.
if has_manual_fail:
for staged_next_task in staged_next_tasks:
staged_next_task['run_on_fail'] = True
# Process the task event using the workflow state machine and update the workflow status.
task_ex_event = events.TaskExecutionEvent(task_id, route, task_state_entry['status'])
machines.WorkflowStateMachine.process_event(self.workflow_state, task_ex_event)
# Process any engine commands in the queue.
while not engine_event_queue.empty():
next_task_id, next_task_route = engine_event_queue.get()
engine_event = events.ENGINE_EVENT_MAP[next_task_id]
self.update_task_state(next_task_id, next_task_route, engine_event())
# Mark the task as a terminal task if workflow execution is completed.
if self.get_workflow_status() in statuses.COMPLETED_STATUSES:
task_state_entry['term'] = True
return task_state_entry