def _flatten_flow(self, flow): """Flattens a graph flow.""" graph = gr.DiGraph(name=flow.name) # Flatten all nodes into a single subgraph per node. subgraph_map = {} for item in flow: subgraph = self._flatten(item) subgraph_map[item] = subgraph graph = gr.merge_graphs([graph, subgraph]) # Reconnect all node edges to their corresponding subgraphs. for (u, v, attrs) in flow.iter_links(): u_g = subgraph_map[u] v_g = subgraph_map[v] if any(attrs.get(k) for k in ("invariant", "manual", "retry")): # Connect nodes with no predecessors in v to nodes with # no successors in u (thus maintaining the edge dependency). self._add_new_edges(graph, u_g.no_successors_iter(), v_g.no_predecessors_iter(), edge_attrs=attrs) else: # This is dependency-only edge, connect corresponding # providers and consumers. for provider in u_g: for consumer in v_g: reasons = provider.provides & consumer.requires if reasons: graph.add_edge(provider, consumer, reasons=reasons) if flow.retry is not None: self._connect_retry(flow.retry, graph) return graph
def test_overlap_detector(self): g = graph.DiGraph() g.add_node("a") g.add_node("b") g.add_edge('a', 'b') g2 = graph.DiGraph() g2.add_node('a') g2.add_node('d') g2.add_edge('a', 'd') self.assertRaises(ValueError, graph.merge_graphs, g, g2) def occurence_detector(to_graph, from_graph): return sum(1 for node in from_graph.nodes_iter() if node in to_graph) self.assertRaises(ValueError, graph.merge_graphs, g, g2, overlap_detector=occurence_detector) g3 = graph.merge_graphs(g, g2, allow_overlaps=True) self.assertEqual(3, len(g3)) self.assertTrue(g3.has_edge('a', 'b')) self.assertTrue(g3.has_edge('a', 'd'))
def test_merge(self): g = graph.DiGraph() g.add_node("a") g.add_node("b") g2 = graph.DiGraph() g2.add_node('c') g3 = graph.merge_graphs(g, g2) self.assertEqual(3, len(g3))
def _decompose_flow(self, flow, parent): """Decomposes a flow into a graph, tree node + decomposed subgraphs.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) decomposed_members = {} for item in flow: subgraph, _subnode = self._flatten(item, node) decomposed_members[item] = subgraph if subgraph.number_of_nodes(): graph = gr.merge_graphs([graph, subgraph]) return graph, node, decomposed_members
def test_merge_edges(self): g = graph.DiGraph() g.add_node("a") g.add_node("b") g.add_edge('a', 'b') g2 = graph.DiGraph() g2.add_node('c') g2.add_node('d') g2.add_edge('c', 'd') g3 = graph.merge_graphs(g, g2) self.assertEqual(4, len(g3)) self.assertTrue(g3.has_edge('c', 'd')) self.assertTrue(g3.has_edge('a', 'b'))
def compile(self, flow, parent=None): """Decomposes a flow into a graph and scope tree hierarchy.""" graph = gr.DiGraph(name=flow.name) graph.add_node(flow, kind=FLOW, noop=True) tree_node = tr.Node(flow, kind=FLOW, noop=True) if parent is not None: parent.add(tree_node) if flow.retry is not None: tree_node.add(tr.Node(flow.retry, kind=RETRY)) decomposed = dict( (child, self._deep_compiler_func(child, parent=tree_node)[0]) for child in flow) decomposed_graphs = list(six.itervalues(decomposed)) graph = gr.merge_graphs(graph, *decomposed_graphs, overlap_detector=_overlap_occurence_detector) for u, v, attr_dict in flow.iter_links(): u_graph = decomposed[u] v_graph = decomposed[v] _add_update_edges(graph, u_graph.no_successors_iter(), list(v_graph.no_predecessors_iter()), attr_dict=attr_dict) if flow.retry is not None: graph.add_node(flow.retry, kind=RETRY) _add_update_edges(graph, [flow], [flow.retry], attr_dict={LINK_INVARIANT: True}) for node in graph.nodes_iter(): if node is not flow.retry and node is not flow: graph.node[node].setdefault(RETRY, flow.retry) from_nodes = [flow.retry] connected_attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True} else: from_nodes = [flow] connected_attr_dict = {LINK_INVARIANT: True} connected_to = [ node for node in graph.no_predecessors_iter() if node is not flow ] if connected_to: # Ensure all nodes in this graph(s) that have no # predecessors depend on this flow (or this flow's retry) so that # we can depend on the flow being traversed before its # children (even though at the current time it will be skipped). _add_update_edges(graph, from_nodes, connected_to, attr_dict=connected_attr_dict) return graph, tree_node
def _decompose_flow(self, flow, parent=None): """Decomposes a flow into a graph, tree node + decomposed subgraphs.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) decomposed_members = {} for item in flow: subgraph, _subnode = self._deep_compiler_func(item, parent=node) decomposed_members[item] = subgraph if subgraph.number_of_nodes(): graph = gr.merge_graphs( graph, subgraph, # We can specialize this to be simpler than the default # algorithm which creates overhead that we don't # need for our purposes... overlap_detector=self._occurence_detector) return graph, node, decomposed_members
def _flatten_flow(self, flow, parent): """Flattens a flow.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) # Flatten all nodes into a single subgraph per item (and track origin # item to its newly expanded graph). subgraphs = {} for item in flow: subgraph = self._flatten(item, node)[0] subgraphs[item] = subgraph graph = gr.merge_graphs([graph, subgraph]) # Reconnect all items edges to their corresponding subgraphs. for (u, v, attrs) in flow.iter_links(): u_g = subgraphs[u] v_g = subgraphs[v] if any(attrs.get(k) for k in _EDGE_INVARIANTS): # Connect nodes with no predecessors in v to nodes with # no successors in u (thus maintaining the edge dependency). self._add_new_edges(graph, u_g.no_successors_iter(), v_g.no_predecessors_iter(), edge_attrs=attrs) else: # This is symbol dependency edge, connect corresponding # providers and consumers. for provider in u_g: for consumer in v_g: reasons = provider.provides & consumer.requires if reasons: graph.add_edge(provider, consumer, reasons=reasons) if flow.retry is not None: self._connect_retry(flow.retry, graph) return graph, node
def _flatten_flow(self, flow, parent): """Flattens a flow.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) # Flatten all nodes into a single subgraph per item (and track origin # item to its newly expanded graph). subgraphs = {} for item in flow: subgraph = self._flatten(item, node)[0] subgraphs[item] = subgraph graph = gr.merge_graphs([graph, subgraph]) # Reconnect all items edges to their corresponding subgraphs. for (u, v, attrs) in flow.iter_links(): u_g = subgraphs[u] v_g = subgraphs[v] if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')): # Connect nodes with no predecessors in v to nodes with # no successors in u (thus maintaining the edge dependency). self._add_new_edges(graph, u_g.no_successors_iter(), v_g.no_predecessors_iter(), edge_attrs=attrs) else: # This is dependency-only edge, connect corresponding # providers and consumers. for provider in u_g: for consumer in v_g: reasons = provider.provides & consumer.requires if reasons: graph.add_edge(provider, consumer, reasons=reasons) if flow.retry is not None: self._connect_retry(flow.retry, graph) return graph, node
def compile(self, flow, parent=None): """Decomposes a flow into a graph and scope tree hierarchy.""" graph = gr.DiGraph(name=flow.name) graph.add_node(flow, kind=FLOW, noop=True) tree_node = tr.Node(flow, kind=FLOW, noop=True) if parent is not None: parent.add(tree_node) if flow.retry is not None: tree_node.add(tr.Node(flow.retry, kind=RETRY)) decomposed = dict( (child, self._deep_compiler_func(child, parent=tree_node)[0]) for child in flow) decomposed_graphs = list(six.itervalues(decomposed)) graph = gr.merge_graphs(graph, *decomposed_graphs, overlap_detector=_overlap_occurence_detector) for u, v, attr_dict in flow.iter_links(): u_graph = decomposed[u] v_graph = decomposed[v] _add_update_edges(graph, u_graph.no_successors_iter(), list(v_graph.no_predecessors_iter()), attr_dict=attr_dict) # Insert the flow(s) retry if needed, and always make sure it # is the **immediate** successor of the flow node itself. if flow.retry is not None: graph.add_node(flow.retry, kind=RETRY) _add_update_edges(graph, [flow], [flow.retry], attr_dict={LINK_INVARIANT: True}) for node in graph.nodes_iter(): if node is not flow.retry and node is not flow: graph.node[node].setdefault(RETRY, flow.retry) from_nodes = [flow.retry] attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True} else: from_nodes = [flow] attr_dict = {LINK_INVARIANT: True} # Ensure all nodes with no predecessors are connected to this flow # or its retry node (so that the invariant that the flow node is # traversed through before its contents is maintained); this allows # us to easily know when we have entered a flow (when running) and # do special and/or smart things such as only traverse up to the # start of a flow when looking for node deciders. _add_update_edges(graph, from_nodes, [ node for node in graph.no_predecessors_iter() if node is not flow ], attr_dict=attr_dict) # Connect all nodes with no successors into a special terminator # that is used to identify the end of the flow and ensure that all # execution traversals will traverse over this node before executing # further work (this is especially useful for nesting and knowing # when we have exited a nesting level); it allows us to do special # and/or smart things such as applying deciders up to (but not # beyond) a flow termination point. # # Do note that in a empty flow this will just connect itself to # the flow node itself... and also note we can not use the flow # object itself (primarily because the underlying graph library # uses hashing to identify node uniqueness and we can easily create # a loop if we don't do this correctly, so avoid that by just # creating this special node and tagging it with a special kind); we # may be able to make this better in the future with a multidigraph # that networkx provides?? flow_term = Terminator(flow) graph.add_node(flow_term, kind=FLOW_END, noop=True) _add_update_edges(graph, [ node for node in graph.no_successors_iter() if node is not flow_term ], [flow_term], attr_dict={LINK_INVARIANT: True}) return graph, tree_node