def _flatten_graph(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
subgraph_map = {}
# Flatten all nodes
for n in flow.graph.nodes_iter():
subgraph = _flatten(n, flattened)
subgraph_map[n] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all nodes to there corresponding subgraphs
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
if not u_v_attrs:
u_v_attrs = FLATTEN_EDGE_DATA.copy()
else:
u_v_attrs.update(FLATTEN_EDGE_DATA)
u_no_succ = list(gu.get_no_successors(subgraph_map[u]))
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
for n in gu.get_no_predecessors(subgraph_map[v]):
# NOTE(harlowja): give each edge its own copy so that if its later
# modified that the same copy isn't modified.
graph.add_edges_from(((n2, n, copy.deepcopy(u_v_attrs))
for n2 in u_no_succ
if not graph.has_edge(n2, n)))
return graph
def _flatten_graph(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
subgraph_map = {}
# Flatten all nodes
for n in flow.graph.nodes_iter():
subgraph = _flatten(n, flattened)
subgraph_map[n] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all nodes to there corresponding subgraphs
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
if not u_v_attrs:
u_v_attrs = FLATTEN_EDGE_DATA.copy()
else:
u_v_attrs.update(FLATTEN_EDGE_DATA)
u_no_succ = list(gu.get_no_successors(subgraph_map[u]))
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
for n in gu.get_no_predecessors(subgraph_map[v]):
# NOTE(harlowja): give each edge its own copy so that if its later
# modified that the same copy isn't modified.
graph.add_edges_from(((n2, n, copy.deepcopy(u_v_attrs))
for n2 in u_no_succ
if not graph.has_edge(n2, n)))
return graph
def test_unordered_flatten(self):
a, b, c, d = _make_many(4)
flo = uf.Flow("test")
flo.add(a, b, c, d)
g = f_utils.flatten(flo)
self.assertEquals(4, len(g))
self.assertEquals(0, g.number_of_edges())
self.assertEquals(set([a, b, c, d]), set(g_utils.get_no_successors(g)))
self.assertEquals(set([a, b, c, d]),
set(g_utils.get_no_predecessors(g)))
def test_unordered_flatten(self):
a, b, c, d = _make_many(4)
flo = uf.Flow("test")
flo.add(a, b, c, d)
g = f_utils.flatten(flo)
self.assertEquals(4, len(g))
self.assertEquals(0, g.number_of_edges())
self.assertEquals(set([a, b, c, d]),
set(g_utils.get_no_successors(g)))
self.assertEquals(set([a, b, c, d]),
set(g_utils.get_no_predecessors(g)))
def test_graph_flatten_links(self):
a, b, c, d = _make_many(4)
flo = gf.Flow("test")
flo.add(a, b, c, d)
flo.link(a, b)
flo.link(b, c)
flo.link(c, d)
g = f_utils.flatten(flo)
self.assertEquals(4, len(g))
self.assertEquals(3, g.number_of_edges())
self.assertEquals(set([a]), set(g_utils.get_no_predecessors(g)))
self.assertEquals(set([d]), set(g_utils.get_no_successors(g)))
def test_graph_flatten_links(self):
a, b, c, d = _make_many(4)
flo = gf.Flow("test")
flo.add(a, b, c, d)
flo.link(a, b)
flo.link(b, c)
flo.link(c, d)
g = f_utils.flatten(flo)
self.assertEquals(4, len(g))
self.assertEquals(3, g.number_of_edges())
self.assertEquals(set([a]),
set(g_utils.get_no_predecessors(g)))
self.assertEquals(set([d]),
set(g_utils.get_no_successors(g)))
def test_linear_flatten(self):
a, b, c, d = _make_many(4)
flo = lf.Flow("test")
flo.add(a, b, c)
sflo = lf.Flow("sub-test")
sflo.add(d)
flo.add(sflo)
g = f_utils.flatten(flo)
self.assertEquals(4, len(g))
order = nx.topological_sort(g)
self.assertEquals([a, b, c, d], order)
self.assertTrue(g.has_edge(c, d))
self.assertEquals([d], list(g_utils.get_no_successors(g)))
self.assertEquals([a], list(g_utils.get_no_predecessors(g)))
def test_linear_flatten(self):
a, b, c, d = _make_many(4)
flo = lf.Flow("test")
flo.add(a, b, c)
sflo = lf.Flow("sub-test")
sflo.add(d)
flo.add(sflo)
g = f_utils.flatten(flo)
self.assertEqual(4, len(g))
order = nx.topological_sort(g)
self.assertEqual([a, b, c, d], order)
self.assertTrue(g.has_edge(c, d))
self.assertEqual([d], list(g_utils.get_no_successors(g)))
self.assertEqual([a], list(g_utils.get_no_predecessors(g)))
def _flatten_linear(self, flow):
"""Flattens a linear flow."""
graph = nx.DiGraph(name=flow.name)
previous_nodes = []
for item in flow:
subgraph = self._flatten(item)
graph = gu.merge_graphs([graph, subgraph])
# Find nodes that have no predecessor, make them have a predecessor
# of the previous nodes so that the linearity ordering is
# maintained. Find the ones with no successors and use this list
# to connect the next subgraph (if any).
self._add_new_edges(graph, previous_nodes,
list(gu.get_no_predecessors(subgraph)))
# There should always be someone without successors, otherwise we
# have a cycle A -> B -> A situation, which should not be possible.
previous_nodes = list(gu.get_no_successors(subgraph))
return graph
def test_ordering(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=set(['a', 'b']))
test_2 = utils.ProvidesRequiresTask('test-2',
provides=['c'],
requires=['a', 'b'])
test_3 = utils.ProvidesRequiresTask('test-3',
provides=[],
requires=['c'])
wf.add(test_1, test_2, test_3)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
self.assertTrue(wf.graph.has_edge(test_2, test_3))
self.assertEqual(3, len(wf.graph))
self.assertEqual([test_1], list(gu.get_no_predecessors(wf.graph)))
self.assertEqual([test_3], list(gu.get_no_successors(wf.graph)))
def test_ordering(self):
wf = gw.Flow("the-test-action")
test_1 = utils.ProvidesRequiresTask('test-1',
requires=[],
provides=set(['a', 'b']))
test_2 = utils.ProvidesRequiresTask('test-2',
provides=['c'],
requires=['a', 'b'])
test_3 = utils.ProvidesRequiresTask('test-3',
provides=[],
requires=['c'])
wf.add(test_1, test_2, test_3)
self.assertTrue(wf.graph.has_edge(test_1, test_2))
self.assertTrue(wf.graph.has_edge(test_2, test_3))
self.assertEqual(3, len(wf.graph))
self.assertEqual([test_1], list(gu.get_no_predecessors(wf.graph)))
self.assertEqual([test_3], list(gu.get_no_successors(wf.graph)))
def _flatten_linear(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
previous_nodes = []
for f in flow:
subgraph = _flatten(f, flattened)
graph = gu.merge_graphs([graph, subgraph])
# Find nodes that have no predecessor, make them have a predecessor of
# the previous nodes so that the linearity ordering is maintained. Find
# the ones with no successors and use this list to connect the next
# subgraph (if any).
for n in gu.get_no_predecessors(subgraph):
graph.add_edges_from(((n2, n, FLATTEN_EDGE_DATA)
for n2 in previous_nodes
if not graph.has_edge(n2, n)))
# There should always be someone without successors, otherwise we have
# a cycle A -> B -> A situation, which should not be possible.
previous_nodes = list(gu.get_no_successors(subgraph))
return graph
def _flatten_graph(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
subgraph_map = {}
# Flatten all nodes
for n in flow.graph.nodes_iter():
subgraph = _flatten(n, flattened)
subgraph_map[n] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all nodes to there corresponding subgraphs
for (u, v) in flow.graph.edges_iter():
u_no_succ = list(gu.get_no_successors(subgraph_map[u]))
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
for n in gu.get_no_predecessors(subgraph_map[v]):
graph.add_edges_from(((n2, n, FLATTEN_EDGE_DATA)
for n2 in u_no_succ
if not graph.has_edge(n2, n)))
return graph
def _flatten_linear(self, flow):
"""Flattens a linear flow."""
graph = nx.DiGraph(name=flow.name)
previous_nodes = []
for item in flow:
subgraph = self._flatten(item)
graph = gu.merge_graphs([graph, subgraph])
# Find nodes that have no predecessor, make them have a predecessor
# of the previous nodes so that the linearity ordering is
# maintained. Find the ones with no successors and use this list
# to connect the next subgraph (if any).
self._add_new_edges(graph,
previous_nodes,
list(gu.get_no_predecessors(subgraph)))
# There should always be someone without successors, otherwise we
# have a cycle A -> B -> A situation, which should not be possible.
previous_nodes = list(gu.get_no_successors(subgraph))
return graph
def _flatten_graph(self, flow):
"""Flattens a graph flow."""
graph = nx.DiGraph(name=flow.name)
# Flatten all nodes into a single subgraph per node.
subgraph_map = {}
for item in flow:
subgraph = self._flatten(item)
subgraph_map[item] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all node edges to there corresponding subgraphs.
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
self._add_new_edges(graph,
list(gu.get_no_successors(subgraph_map[u])),
list(gu.get_no_predecessors(subgraph_map[v])),
edge_attrs=u_v_attrs)
return graph
def _flatten_linear(flow, flattened):
graph = nx.DiGraph(name=_graph_name(flow))
previous_nodes = []
for f in flow:
subgraph = _flatten(f, flattened)
graph = gu.merge_graphs([graph, subgraph])
# Find nodes that have no predecessor, make them have a predecessor of
# the previous nodes so that the linearity ordering is maintained. Find
# the ones with no successors and use this list to connect the next
# subgraph (if any).
for n in gu.get_no_predecessors(subgraph):
# NOTE(harlowja): give each edge its own copy so that if its later
# modified that the same copy isn't modified.
graph.add_edges_from(((n2, n, FLATTEN_EDGE_DATA.copy())
for n2 in previous_nodes
if not graph.has_edge(n2, n)))
# There should always be someone without successors, otherwise we have
# a cycle A -> B -> A situation, which should not be possible.
previous_nodes = list(gu.get_no_successors(subgraph))
return graph
def _flatten_graph(self, flow):
"""Flattens a graph flow."""
graph = nx.DiGraph(name=flow.name)
# Flatten all nodes into a single subgraph per node.
subgraph_map = {}
for item in flow:
subgraph = self._flatten(item)
subgraph_map[item] = subgraph
graph = gu.merge_graphs([graph, subgraph])
# Reconnect all node edges to there corresponding subgraphs.
for (u, v) in flow.graph.edges_iter():
# Retain and update the original edge attributes.
u_v_attrs = gu.get_edge_attrs(flow.graph, u, v)
# Connect the ones with no predecessors in v to the ones with no
# successors in u (thus maintaining the edge dependency).
self._add_new_edges(graph,
list(gu.get_no_successors(subgraph_map[u])),
list(gu.get_no_predecessors(subgraph_map[v])),
edge_attrs=u_v_attrs)
return graph
def translate(self, flow):
graph = flow_utils.flatten(flow)
#TODO(jlucci): Logic to be re-written once task_id logic decided on
for node in graph.nodes():
if getattr(node, '_id', None) is None:
node._id = "%s.%s" % (node.name, uuidutils.generate_uuid())
self.engine.tasks[node._id] = (self.HybridTask(node,
self.engine.celery_app))
self.engine.storage.add_task(node._id, node.name)
for (u, v) in graph.edges_iter():
self._add(self.engine.tasks[u._id])
self._add(self.engine.tasks[v._id])
if not (u.provides).intersection(v.requires):
self._link(self.engine.tasks[u._id],
self.engine.tasks[v._id])
roots = graph_utils.get_no_predecessors(graph)
for root in roots:
self.engine.roots.append(self.engine.tasks[root._id])
return graph