def test_skip_add_tasks(self):
wf_graph = graphing.WorkflowGraph()
self._add_transitions(wf_graph)
self._add_barriers(wf_graph)
self.assert_graph_equal(wf_graph, EXPECTED_WF_GRAPH)
def _prep_graph(self):
wf_graph = graphing.WorkflowGraph()
self._add_tasks(wf_graph)
self._add_transitions(wf_graph)
return wf_graph
def _compose_wf_graph(cls, wf_spec):
if not isinstance(wf_spec, cls.wf_spec_type):
raise TypeError('Workflow spec is not typeof %s.' %
cls.wf_spec_type.__name__)
q = queue.Queue()
wf_graph = graphing.WorkflowGraph()
for task_name, condition, task_transition_item_idx in wf_spec.tasks.get_start_tasks(
):
q.put((task_name, []))
while not q.empty():
task_name, splits = q.get()
wf_graph.add_task(task_name)
if wf_spec.tasks.is_join_task(task_name):
task_spec = wf_spec.tasks[task_name]
barrier = '*' if task_spec.join == 'all' else task_spec.join
wf_graph.set_barrier(task_name, value=barrier)
# Determine if the task is a split task and if it is in a cycle. If the task is a
# split task, keep track of where the split(s) occurs.
if wf_spec.tasks.is_split_task(
task_name) and not wf_spec.tasks.in_cycle(task_name):
splits.append(task_name)
if splits:
wf_graph.update_task(task_name, splits=splits)
next_tasks = wf_spec.tasks.get_next_tasks(task_name)
for next_task_name, condition, task_transition_item_idx in next_tasks:
if (not wf_graph.has_task(next_task_name)
or not wf_spec.tasks.in_cycle(next_task_name)):
q.put((next_task_name, list(splits)))
crta = [condition] if condition else []
seqs = wf_graph.has_transition(task_name,
next_task_name,
criteria=crta,
ref=task_transition_item_idx)
# Use existing transition if present otherwise create new transition.
if seqs:
wf_graph.update_transition(task_name,
next_task_name,
key=seqs[0][2],
criteria=crta,
ref=task_transition_item_idx)
else:
wf_graph.add_transition(task_name,
next_task_name,
criteria=crta,
ref=task_transition_item_idx)
return wf_graph
def test_get_ambiguous_transition(self):
wf_graph = graphing.WorkflowGraph()
self._add_tasks(wf_graph)
wf_graph._graph.add_edge("task1", "task2")
wf_graph._graph.add_edge("task1", "task2")
self.assertRaises(exc.AmbiguousTaskTransition, wf_graph.get_transition,
"task1", "task2")
def compose(cls, spec):
if not cls.wf_spec_type:
raise TypeError('Undefined spec type for composer.')
if not isinstance(spec, cls.wf_spec_type):
raise TypeError('Unsupported spec type "%s".' % str(type(spec)))
wf_graph = graphing.WorkflowGraph()
return wf_graph
def _compose_wf_ex_graph(cls, wf_graph):
q = queue.Queue()
split_counter = {}
wf_ex_graph = graphing.WorkflowGraph()
def _create_task_ex_name(task_name, split_id):
return task_name + '__' + str(
split_id) if split_id > 0 else task_name
for task in wf_graph.roots:
q.put((task['id'], None, None, []))
while not q.empty():
task_name, prev_task_ex_name, criteria, splits = q.get()
task_ex_attrs = wf_graph.get_task(task_name)
task_ex_attrs['name'] = task_name
# For complex multi-level splits and joins, if a task from higher in the hierarchy is
# processed first, then ignore the task for now. This task will be processed again
# later in the hierarchy. Otherwise, if this task is processed now, it will be placed
# in a separate workflow branch.
expected_splits = task_ex_attrs.pop('splits', [])
prev_task_ex = wf_ex_graph.get_task(
prev_task_ex_name) if prev_task_ex_name else {}
if (expected_splits and task_name not in expected_splits
and not prev_task_ex.get('splits', [])):
continue
# Determine if the task is a split task and if it is in a cycle. If the task is a split
# task, keep track of how many instances and which branch the instance belongs to.
is_split_task = (len(wf_graph.get_prev_transitions(task_name)) > 1
and not wf_graph.has_barrier(task_name))
is_task_in_cycle = wf_graph.in_cycle(task_name)
if is_split_task and not is_task_in_cycle:
split_counter[task_name] = split_counter.get(task_name, 0) + 1
splits.append((task_name, split_counter[task_name]))
if splits:
task_ex_attrs['splits'] = splits
task_ex_name = _create_task_ex_name(task_name,
splits[-1][1] if splits else 0)
# If the task already exists in the execution graph, the task is already processed
# and this is a cycle in the graph.
if wf_ex_graph.has_task(task_ex_name):
wf_ex_graph.update_task(task_ex_name, **task_ex_attrs)
else:
wf_ex_graph.add_task(task_ex_name, **task_ex_attrs)
for next_seq in wf_graph.get_next_transitions(task_name):
next_seq_criteria = [
c.replace('task_state(%s)' % task_name,
'task_state(%s)' % task_ex_name)
for c in next_seq[3]['criteria']
]
q.put((next_seq[1], task_ex_name, next_seq_criteria,
list(splits)))
# A split task should only have one previous transition even if there are multiple
# different tasks transitioning to it. Since it has no join requirement, the split
# task will create a new instance and execute.
if is_split_task and prev_task_ex_name:
wf_ex_graph.add_transition(prev_task_ex_name,
task_ex_name,
criteria=criteria)
continue
# Finally, process all inbound task transitions.
for prev_seq in wf_graph.get_prev_transitions(task_name):
prev_task = wf_graph.get_task(prev_seq[0])
split_id = 0
for prev_task_split in prev_task.get('splits', []):
matches = [s for s in splits if s[0] == prev_task_split]
split_id = matches[0][1] if matches else split_id
p_task_name = prev_seq[0]
p_task_ex_name = _create_task_ex_name(p_task_name, split_id)
p_seq_criteria = [
c.replace('task_state(%s)' % p_task_name,
'task_state(%s)' % p_task_ex_name)
for c in prev_seq[3]['criteria']
]
seqs = wf_ex_graph.has_transition(p_task_ex_name,
task_ex_name,
criteria=p_seq_criteria)
# Use existing transition if present otherwise create new transition.
if seqs:
wf_ex_graph.update_transition(p_task_ex_name,
task_ex_name,
key=seqs[0][2],
criteria=p_seq_criteria)
else:
wf_ex_graph.add_transition(p_task_ex_name,
task_ex_name,
criteria=p_seq_criteria)
return wf_ex_graph
def _compose_wf_graph(cls, wf_spec):
if not isinstance(wf_spec, cls.wf_spec_type):
raise TypeError("Workflow spec is not typeof %s." % cls.wf_spec_type.__name__)
q = queue.Queue()
wf_graph = graphing.WorkflowGraph()
for task_name, condition, task_transition_item_idx in wf_spec.tasks.get_start_tasks():
q.put((task_name, []))
while not q.empty():
task_name, splits = q.get()
wf_graph.add_task(task_name)
if wf_spec.tasks.is_join_task(task_name):
task_spec = wf_spec.tasks[task_name]
barrier = "*" if task_spec.join == "all" else task_spec.join
wf_graph.set_barrier(task_name, value=barrier)
# Determine if the task is a split task and if it is in a cycle. If the task is a
# split task, keep track of where the split(s) occurs.
if wf_spec.tasks.is_split_task(task_name) and not wf_spec.tasks.in_cycle(task_name):
splits.append(task_name)
if splits:
wf_graph.update_task(task_name, splits=splits)
# Update task attributes if task spec has retry criteria.
task_spec = wf_spec.tasks.get_task(task_name)
if task_spec.has_retry():
retry_spec = {
"when": getattr(task_spec.retry, "when", None),
"count": getattr(task_spec.retry, "count", None),
"delay": getattr(task_spec.retry, "delay", None),
}
wf_graph.update_task(task_name, retry=retry_spec)
# Add task transition to the workflow graph.
next_tasks = wf_spec.tasks.get_next_tasks(task_name)
for next_task_name, condition, task_transition_item_idx in next_tasks:
if next_task_name == "retry":
retry_spec = {"when": condition or "<% completed() %>", "count": 3}
wf_graph.update_task(task_name, retry=retry_spec)
continue
if not wf_graph.has_task(next_task_name) or not wf_spec.tasks.in_cycle(
next_task_name
):
q.put((next_task_name, list(splits)))
crta = [condition] if condition else []
seqs = wf_graph.has_transition(
task_name, next_task_name, criteria=crta, ref=task_transition_item_idx
)
# Use existing transition if present otherwise create new transition.
if seqs:
wf_graph.update_transition(
task_name,
next_task_name,
key=seqs[0][2],
criteria=crta,
ref=task_transition_item_idx,
)
else:
wf_graph.add_transition(
task_name, next_task_name, criteria=crta, ref=task_transition_item_idx
)
return wf_graph
