def test_skip_add_tasks(self): wf_graph = graphing.WorkflowGraph() self._add_transitions(wf_graph) self._add_barriers(wf_graph) self.assert_graph_equal(wf_graph, EXPECTED_WF_GRAPH)
def _prep_graph(self): wf_graph = graphing.WorkflowGraph() self._add_tasks(wf_graph) self._add_transitions(wf_graph) return wf_graph
def _compose_wf_graph(cls, wf_spec): if not isinstance(wf_spec, cls.wf_spec_type): raise TypeError('Workflow spec is not typeof %s.' % cls.wf_spec_type.__name__) q = queue.Queue() wf_graph = graphing.WorkflowGraph() for task_name, condition, task_transition_item_idx in wf_spec.tasks.get_start_tasks( ): q.put((task_name, [])) while not q.empty(): task_name, splits = q.get() wf_graph.add_task(task_name) if wf_spec.tasks.is_join_task(task_name): task_spec = wf_spec.tasks[task_name] barrier = '*' if task_spec.join == 'all' else task_spec.join wf_graph.set_barrier(task_name, value=barrier) # Determine if the task is a split task and if it is in a cycle. If the task is a # split task, keep track of where the split(s) occurs. if wf_spec.tasks.is_split_task( task_name) and not wf_spec.tasks.in_cycle(task_name): splits.append(task_name) if splits: wf_graph.update_task(task_name, splits=splits) next_tasks = wf_spec.tasks.get_next_tasks(task_name) for next_task_name, condition, task_transition_item_idx in next_tasks: if (not wf_graph.has_task(next_task_name) or not wf_spec.tasks.in_cycle(next_task_name)): q.put((next_task_name, list(splits))) crta = [condition] if condition else [] seqs = wf_graph.has_transition(task_name, next_task_name, criteria=crta, ref=task_transition_item_idx) # Use existing transition if present otherwise create new transition. if seqs: wf_graph.update_transition(task_name, next_task_name, key=seqs[0][2], criteria=crta, ref=task_transition_item_idx) else: wf_graph.add_transition(task_name, next_task_name, criteria=crta, ref=task_transition_item_idx) return wf_graph
def test_get_ambiguous_transition(self): wf_graph = graphing.WorkflowGraph() self._add_tasks(wf_graph) wf_graph._graph.add_edge("task1", "task2") wf_graph._graph.add_edge("task1", "task2") self.assertRaises(exc.AmbiguousTaskTransition, wf_graph.get_transition, "task1", "task2")
def compose(cls, spec): if not cls.wf_spec_type: raise TypeError('Undefined spec type for composer.') if not isinstance(spec, cls.wf_spec_type): raise TypeError('Unsupported spec type "%s".' % str(type(spec))) wf_graph = graphing.WorkflowGraph() return wf_graph
def _compose_wf_ex_graph(cls, wf_graph): q = queue.Queue() split_counter = {} wf_ex_graph = graphing.WorkflowGraph() def _create_task_ex_name(task_name, split_id): return task_name + '__' + str( split_id) if split_id > 0 else task_name for task in wf_graph.roots: q.put((task['id'], None, None, [])) while not q.empty(): task_name, prev_task_ex_name, criteria, splits = q.get() task_ex_attrs = wf_graph.get_task(task_name) task_ex_attrs['name'] = task_name # For complex multi-level splits and joins, if a task from higher in the hierarchy is # processed first, then ignore the task for now. This task will be processed again # later in the hierarchy. Otherwise, if this task is processed now, it will be placed # in a separate workflow branch. expected_splits = task_ex_attrs.pop('splits', []) prev_task_ex = wf_ex_graph.get_task( prev_task_ex_name) if prev_task_ex_name else {} if (expected_splits and task_name not in expected_splits and not prev_task_ex.get('splits', [])): continue # Determine if the task is a split task and if it is in a cycle. If the task is a split # task, keep track of how many instances and which branch the instance belongs to. is_split_task = (len(wf_graph.get_prev_transitions(task_name)) > 1 and not wf_graph.has_barrier(task_name)) is_task_in_cycle = wf_graph.in_cycle(task_name) if is_split_task and not is_task_in_cycle: split_counter[task_name] = split_counter.get(task_name, 0) + 1 splits.append((task_name, split_counter[task_name])) if splits: task_ex_attrs['splits'] = splits task_ex_name = _create_task_ex_name(task_name, splits[-1][1] if splits else 0) # If the task already exists in the execution graph, the task is already processed # and this is a cycle in the graph. if wf_ex_graph.has_task(task_ex_name): wf_ex_graph.update_task(task_ex_name, **task_ex_attrs) else: wf_ex_graph.add_task(task_ex_name, **task_ex_attrs) for next_seq in wf_graph.get_next_transitions(task_name): next_seq_criteria = [ c.replace('task_state(%s)' % task_name, 'task_state(%s)' % task_ex_name) for c in next_seq[3]['criteria'] ] q.put((next_seq[1], task_ex_name, next_seq_criteria, list(splits))) # A split task should only have one previous transition even if there are multiple # different tasks transitioning to it. Since it has no join requirement, the split # task will create a new instance and execute. if is_split_task and prev_task_ex_name: wf_ex_graph.add_transition(prev_task_ex_name, task_ex_name, criteria=criteria) continue # Finally, process all inbound task transitions. for prev_seq in wf_graph.get_prev_transitions(task_name): prev_task = wf_graph.get_task(prev_seq[0]) split_id = 0 for prev_task_split in prev_task.get('splits', []): matches = [s for s in splits if s[0] == prev_task_split] split_id = matches[0][1] if matches else split_id p_task_name = prev_seq[0] p_task_ex_name = _create_task_ex_name(p_task_name, split_id) p_seq_criteria = [ c.replace('task_state(%s)' % p_task_name, 'task_state(%s)' % p_task_ex_name) for c in prev_seq[3]['criteria'] ] seqs = wf_ex_graph.has_transition(p_task_ex_name, task_ex_name, criteria=p_seq_criteria) # Use existing transition if present otherwise create new transition. if seqs: wf_ex_graph.update_transition(p_task_ex_name, task_ex_name, key=seqs[0][2], criteria=p_seq_criteria) else: wf_ex_graph.add_transition(p_task_ex_name, task_ex_name, criteria=p_seq_criteria) return wf_ex_graph
def _compose_wf_graph(cls, wf_spec): if not isinstance(wf_spec, cls.wf_spec_type): raise TypeError("Workflow spec is not typeof %s." % cls.wf_spec_type.__name__) q = queue.Queue() wf_graph = graphing.WorkflowGraph() for task_name, condition, task_transition_item_idx in wf_spec.tasks.get_start_tasks(): q.put((task_name, [])) while not q.empty(): task_name, splits = q.get() wf_graph.add_task(task_name) if wf_spec.tasks.is_join_task(task_name): task_spec = wf_spec.tasks[task_name] barrier = "*" if task_spec.join == "all" else task_spec.join wf_graph.set_barrier(task_name, value=barrier) # Determine if the task is a split task and if it is in a cycle. If the task is a # split task, keep track of where the split(s) occurs. if wf_spec.tasks.is_split_task(task_name) and not wf_spec.tasks.in_cycle(task_name): splits.append(task_name) if splits: wf_graph.update_task(task_name, splits=splits) # Update task attributes if task spec has retry criteria. task_spec = wf_spec.tasks.get_task(task_name) if task_spec.has_retry(): retry_spec = { "when": getattr(task_spec.retry, "when", None), "count": getattr(task_spec.retry, "count", None), "delay": getattr(task_spec.retry, "delay", None), } wf_graph.update_task(task_name, retry=retry_spec) # Add task transition to the workflow graph. next_tasks = wf_spec.tasks.get_next_tasks(task_name) for next_task_name, condition, task_transition_item_idx in next_tasks: if next_task_name == "retry": retry_spec = {"when": condition or "<% completed() %>", "count": 3} wf_graph.update_task(task_name, retry=retry_spec) continue if not wf_graph.has_task(next_task_name) or not wf_spec.tasks.in_cycle( next_task_name ): q.put((next_task_name, list(splits))) crta = [condition] if condition else [] seqs = wf_graph.has_transition( task_name, next_task_name, criteria=crta, ref=task_transition_item_idx ) # Use existing transition if present otherwise create new transition. if seqs: wf_graph.update_transition( task_name, next_task_name, key=seqs[0][2], criteria=crta, ref=task_transition_item_idx, ) else: wf_graph.add_transition( task_name, next_task_name, criteria=crta, ref=task_transition_item_idx ) return wf_graph