def _flatten_graph(flow, flattened): graph = nx.DiGraph(name=_graph_name(flow)) subgraph_map = {} # Flatten all nodes for n in flow.graph.nodes_iter(): subgraph = _flatten(n, flattened) subgraph_map[n] = subgraph graph = gu.merge_graphs([graph, subgraph]) # Reconnect all nodes to there corresponding subgraphs for (u, v) in flow.graph.edges_iter(): # Retain and update the original edge attributes. u_v_attrs = gu.get_edge_attrs(flow.graph, u, v) if not u_v_attrs: u_v_attrs = FLATTEN_EDGE_DATA.copy() else: u_v_attrs.update(FLATTEN_EDGE_DATA) u_no_succ = list(gu.get_no_successors(subgraph_map[u])) # Connect the ones with no predecessors in v to the ones with no # successors in u (thus maintaining the edge dependency). for n in gu.get_no_predecessors(subgraph_map[v]): # NOTE(harlowja): give each edge its own copy so that if its later # modified that the same copy isn't modified. graph.add_edges_from(((n2, n, copy.deepcopy(u_v_attrs)) for n2 in u_no_succ if not graph.has_edge(n2, n))) return graph
def test_unordered_flatten(self): a, b, c, d = _make_many(4) flo = uf.Flow("test") flo.add(a, b, c, d) g = f_utils.flatten(flo) self.assertEquals(4, len(g)) self.assertEquals(0, g.number_of_edges()) self.assertEquals(set([a, b, c, d]), set(g_utils.get_no_successors(g))) self.assertEquals(set([a, b, c, d]), set(g_utils.get_no_predecessors(g)))
def test_graph_flatten_links(self): a, b, c, d = _make_many(4) flo = gf.Flow("test") flo.add(a, b, c, d) flo.link(a, b) flo.link(b, c) flo.link(c, d) g = f_utils.flatten(flo) self.assertEquals(4, len(g)) self.assertEquals(3, g.number_of_edges()) self.assertEquals(set([a]), set(g_utils.get_no_predecessors(g))) self.assertEquals(set([d]), set(g_utils.get_no_successors(g)))
def test_linear_flatten(self): a, b, c, d = _make_many(4) flo = lf.Flow("test") flo.add(a, b, c) sflo = lf.Flow("sub-test") sflo.add(d) flo.add(sflo) g = f_utils.flatten(flo) self.assertEquals(4, len(g)) order = nx.topological_sort(g) self.assertEquals([a, b, c, d], order) self.assertTrue(g.has_edge(c, d)) self.assertEquals([d], list(g_utils.get_no_successors(g))) self.assertEquals([a], list(g_utils.get_no_predecessors(g)))
def test_linear_flatten(self): a, b, c, d = _make_many(4) flo = lf.Flow("test") flo.add(a, b, c) sflo = lf.Flow("sub-test") sflo.add(d) flo.add(sflo) g = f_utils.flatten(flo) self.assertEqual(4, len(g)) order = nx.topological_sort(g) self.assertEqual([a, b, c, d], order) self.assertTrue(g.has_edge(c, d)) self.assertEqual([d], list(g_utils.get_no_successors(g))) self.assertEqual([a], list(g_utils.get_no_predecessors(g)))
def _flatten_linear(self, flow): """Flattens a linear flow.""" graph = nx.DiGraph(name=flow.name) previous_nodes = [] for item in flow: subgraph = self._flatten(item) graph = gu.merge_graphs([graph, subgraph]) # Find nodes that have no predecessor, make them have a predecessor # of the previous nodes so that the linearity ordering is # maintained. Find the ones with no successors and use this list # to connect the next subgraph (if any). self._add_new_edges(graph, previous_nodes, list(gu.get_no_predecessors(subgraph))) # There should always be someone without successors, otherwise we # have a cycle A -> B -> A situation, which should not be possible. previous_nodes = list(gu.get_no_successors(subgraph)) return graph
def test_ordering(self): wf = gw.Flow("the-test-action") test_1 = utils.ProvidesRequiresTask('test-1', requires=[], provides=set(['a', 'b'])) test_2 = utils.ProvidesRequiresTask('test-2', provides=['c'], requires=['a', 'b']) test_3 = utils.ProvidesRequiresTask('test-3', provides=[], requires=['c']) wf.add(test_1, test_2, test_3) self.assertTrue(wf.graph.has_edge(test_1, test_2)) self.assertTrue(wf.graph.has_edge(test_2, test_3)) self.assertEqual(3, len(wf.graph)) self.assertEqual([test_1], list(gu.get_no_predecessors(wf.graph))) self.assertEqual([test_3], list(gu.get_no_successors(wf.graph)))
def _flatten_linear(flow, flattened): graph = nx.DiGraph(name=_graph_name(flow)) previous_nodes = [] for f in flow: subgraph = _flatten(f, flattened) graph = gu.merge_graphs([graph, subgraph]) # Find nodes that have no predecessor, make them have a predecessor of # the previous nodes so that the linearity ordering is maintained. Find # the ones with no successors and use this list to connect the next # subgraph (if any). for n in gu.get_no_predecessors(subgraph): graph.add_edges_from(((n2, n, FLATTEN_EDGE_DATA) for n2 in previous_nodes if not graph.has_edge(n2, n))) # There should always be someone without successors, otherwise we have # a cycle A -> B -> A situation, which should not be possible. previous_nodes = list(gu.get_no_successors(subgraph)) return graph
def _flatten_graph(flow, flattened): graph = nx.DiGraph(name=_graph_name(flow)) subgraph_map = {} # Flatten all nodes for n in flow.graph.nodes_iter(): subgraph = _flatten(n, flattened) subgraph_map[n] = subgraph graph = gu.merge_graphs([graph, subgraph]) # Reconnect all nodes to there corresponding subgraphs for (u, v) in flow.graph.edges_iter(): u_no_succ = list(gu.get_no_successors(subgraph_map[u])) # Connect the ones with no predecessors in v to the ones with no # successors in u (thus maintaining the edge dependency). for n in gu.get_no_predecessors(subgraph_map[v]): graph.add_edges_from(((n2, n, FLATTEN_EDGE_DATA) for n2 in u_no_succ if not graph.has_edge(n2, n))) return graph
def _flatten_graph(self, flow): """Flattens a graph flow.""" graph = nx.DiGraph(name=flow.name) # Flatten all nodes into a single subgraph per node. subgraph_map = {} for item in flow: subgraph = self._flatten(item) subgraph_map[item] = subgraph graph = gu.merge_graphs([graph, subgraph]) # Reconnect all node edges to there corresponding subgraphs. for (u, v) in flow.graph.edges_iter(): # Retain and update the original edge attributes. u_v_attrs = gu.get_edge_attrs(flow.graph, u, v) # Connect the ones with no predecessors in v to the ones with no # successors in u (thus maintaining the edge dependency). self._add_new_edges(graph, list(gu.get_no_successors(subgraph_map[u])), list(gu.get_no_predecessors(subgraph_map[v])), edge_attrs=u_v_attrs) return graph
def _flatten_linear(flow, flattened): graph = nx.DiGraph(name=_graph_name(flow)) previous_nodes = [] for f in flow: subgraph = _flatten(f, flattened) graph = gu.merge_graphs([graph, subgraph]) # Find nodes that have no predecessor, make them have a predecessor of # the previous nodes so that the linearity ordering is maintained. Find # the ones with no successors and use this list to connect the next # subgraph (if any). for n in gu.get_no_predecessors(subgraph): # NOTE(harlowja): give each edge its own copy so that if its later # modified that the same copy isn't modified. graph.add_edges_from(((n2, n, FLATTEN_EDGE_DATA.copy()) for n2 in previous_nodes if not graph.has_edge(n2, n))) # There should always be someone without successors, otherwise we have # a cycle A -> B -> A situation, which should not be possible. previous_nodes = list(gu.get_no_successors(subgraph)) return graph
def translate(self, flow): graph = flow_utils.flatten(flow) #TODO(jlucci): Logic to be re-written once task_id logic decided on for node in graph.nodes(): if getattr(node, '_id', None) is None: node._id = "%s.%s" % (node.name, uuidutils.generate_uuid()) self.engine.tasks[node._id] = (self.HybridTask(node, self.engine.celery_app)) self.engine.storage.add_task(node._id, node.name) for (u, v) in graph.edges_iter(): self._add(self.engine.tasks[u._id]) self._add(self.engine.tasks[v._id]) if not (u.provides).intersection(v.requires): self._link(self.engine.tasks[u._id], self.engine.tasks[v._id]) roots = graph_utils.get_no_predecessors(graph) for root in roots: self.engine.roots.append(self.engine.tasks[root._id]) return graph