def _flatten_graph(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
subgraph_map = {}
# Flatten all nodes
for n in flow.graph.nodes_iter():
subgraph = _flatten(n, flattened)
subgraph_map[n] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all nodes to there corresponding subgraphs
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
if not u_v_attrs:
u_v_attrs = FLATTEN_EDGE_DATA.copy()
else:
u_v_attrs.update(FLATTEN_EDGE_DATA)
u_no_succ = list(gu.get_no_successors(subgraph_map[u]))
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
for n in gu.get_no_predecessors(subgraph_map[v]):
# NOTE(harlowja): give each edge its own copy so that if its later
# modified that the same copy isn't modified.
graph.add_edges_from(((n2, n, copy.deepcopy(u_v_attrs))
for n2 in u_no_succ
if not graph.has_edge(n2, n)))
return graph
def _flatten_graph(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
subgraph_map = {}
# Flatten all nodes
for n in flow.graph.nodes_iter():
subgraph = _flatten(n, flattened)
subgraph_map[n] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all nodes to there corresponding subgraphs
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
if not u_v_attrs:
u_v_attrs = FLATTEN_EDGE_DATA.copy()
else:
u_v_attrs.update(FLATTEN_EDGE_DATA)
u_no_succ = list(gu.get_no_successors(subgraph_map[u]))
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
for n in gu.get_no_predecessors(subgraph_map[v]):
# NOTE(harlowja): give each edge its own copy so that if its later
# modified that the same copy isn't modified.
graph.add_edges_from(((n2, n, copy.deepcopy(u_v_attrs))
for n2 in u_no_succ
if not graph.has_edge(n2, n)))
return graph
def test_basic_edge_reasons(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=set(['a', 'b']))
test_2 = utils.ProvidesRequiresTask('test-2',
provides=['c'],
requires=['a', 'b'])
wf.add(test_1, test_2)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
edge_attrs = gu.get_edge_attrs(wf.graph, test_1, test_2)
self.assertTrue(len(edge_attrs) > 0)
self.assertIn('reasons', edge_attrs)
self.assertEqual(set(['a', 'b']), edge_attrs['reasons'])
# 2 -> 1 should not be linked, and therefore have no attrs
no_edge_attrs = gu.get_edge_attrs(wf.graph, test_2, test_1)
self.assertFalse(no_edge_attrs)
def test_basic_edge_reasons(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=set(['a', 'b']))
test_2 = utils.ProvidesRequiresTask('test-2',
provides=['c'],
requires=['a', 'b'])
wf.add(test_1, test_2)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
edge_attrs = gu.get_edge_attrs(wf.graph, test_1, test_2)
self.assertTrue(len(edge_attrs) > 0)
self.assertIn('reasons', edge_attrs)
self.assertEqual(set(['a', 'b']), edge_attrs['reasons'])
# 2 -> 1 should not be linked, and therefore have no attrs
no_edge_attrs = gu.get_edge_attrs(wf.graph, test_2, test_1)
self.assertFalse(no_edge_attrs)
def test_flatten_attribute(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1', requires=[], provides=[])
test_2 = utils.ProvidesRequiresTask('test-2', provides=[], requires=[])
wf.add(test_1, test_2)
wf.link(test_1, test_2)
g = fu.flatten(wf)
self.assertEqual(2, len(g))
edge_attrs = gu.get_edge_attrs(g, test_1, test_2)
self.assertTrue(edge_attrs.get('manual'))
self.assertTrue(edge_attrs.get('flatten'))
def test_linked_edge_reasons(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1', requires=[], provides=[])
test_2 = utils.ProvidesRequiresTask('test-2', provides=[], requires=[])
wf.add(test_1, test_2)
self.assertFalse(wf.graph.has_edge(test_1, test_2))
wf.link(test_1, test_2)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
edge_attrs = gu.get_edge_attrs(wf.graph, test_1, test_2)
self.assertTrue(len(edge_attrs) > 0)
self.assertTrue(edge_attrs.get('manual'))
def test_flatten_attribute(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=[])
test_2 = utils.ProvidesRequiresTask('test-2',
provides=[],
requires=[])
wf.add(test_1, test_2)
wf.link(test_1, test_2)
g = fu.flatten(wf)
self.assertEqual(2, len(g))
edge_attrs = gu.get_edge_attrs(g, test_1, test_2)
self.assertTrue(edge_attrs.get('manual'))
self.assertTrue(edge_attrs.get('flatten'))
def test_linked_edge_reasons(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=[])
test_2 = utils.ProvidesRequiresTask('test-2',
provides=[],
requires=[])
wf.add(test_1, test_2)
self.assertFalse(wf.graph.has_edge(test_1, test_2))
wf.link(test_1, test_2)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
edge_attrs = gu.get_edge_attrs(wf.graph, test_1, test_2)
self.assertTrue(len(edge_attrs) > 0)
self.assertTrue(edge_attrs.get('manual'))
def _link(self, u, v, graph=None, reason=None, manual=False):
mutable_graph = True
if graph is None:
graph = self._graph
mutable_graph = False
# NOTE(harlowja): Add an edge to a temporary copy and only if that
# copy is valid then do we swap with the underlying graph.
attrs = graph_utils.get_edge_attrs(graph, u, v)
if not attrs:
attrs = {}
if manual:
attrs['manual'] = True
if reason is not None:
if 'reasons' not in attrs:
attrs['reasons'] = set()
attrs['reasons'].add(reason)
if not mutable_graph:
graph = nx.DiGraph(graph)
graph.add_edge(u, v, **attrs)
return graph
def _flatten_graph(self, flow):
"""Flattens a graph flow."""
graph = nx.DiGraph(name=flow.name)
# Flatten all nodes into a single subgraph per node.
subgraph_map = {}
for item in flow:
subgraph = self._flatten(item)
subgraph_map[item] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all node edges to there corresponding subgraphs.
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
self._add_new_edges(graph,
list(gu.get_no_successors(subgraph_map[u])),
list(gu.get_no_predecessors(subgraph_map[v])),
edge_attrs=u_v_attrs)
return graph
def _link(self, u, v, graph=None, reason=None, manual=False):
mutable_graph = True
if graph is None:
graph = self._graph
mutable_graph = False
# NOTE(harlowja): Add an edge to a temporary copy and only if that
# copy is valid then do we swap with the underlying graph.
attrs = graph_utils.get_edge_attrs(graph, u, v)
if not attrs:
attrs = {}
if manual:
attrs['manual'] = True
if reason is not None:
if 'reasons' not in attrs:
attrs['reasons'] = set()
attrs['reasons'].add(reason)
if not mutable_graph:
graph = nx.DiGraph(graph)
graph.add_edge(u, v, **attrs)
return graph
def _flatten_graph(self, flow):
"""Flattens a graph flow."""
graph = nx.DiGraph(name=flow.name)
# Flatten all nodes into a single subgraph per node.
subgraph_map = {}
for item in flow:
subgraph = self._flatten(item)
subgraph_map[item] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all node edges to there corresponding subgraphs.
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
self._add_new_edges(graph,
list(gu.get_no_successors(subgraph_map[u])),
list(gu.get_no_predecessors(subgraph_map[v])),
edge_attrs=u_v_attrs)
return graph
