def test_overlap_detector(self):
g = graph.DiGraph()
g.add_node("a")
g.add_node("b")
g.add_edge('a', 'b')
g2 = graph.DiGraph()
g2.add_node('a')
g2.add_node('d')
g2.add_edge('a', 'd')
self.assertRaises(ValueError,
graph.merge_graphs, g, g2)
def occurence_detector(to_graph, from_graph):
return sum(1 for node in from_graph.nodes_iter()
if node in to_graph)
self.assertRaises(ValueError,
graph.merge_graphs, g, g2,
overlap_detector=occurence_detector)
g3 = graph.merge_graphs(g, g2, allow_overlaps=True)
self.assertEqual(3, len(g3))
self.assertTrue(g3.has_edge('a', 'b'))
self.assertTrue(g3.has_edge('a', 'd'))
def test_merge(self):
g = graph.DiGraph()
g.add_node("a")
g.add_node("b")
g2 = graph.DiGraph()
g2.add_node('c')
g3 = graph.merge_graphs(g, g2)
self.assertEqual(3, len(g3))
def test_invalid_detector(self):
g = graph.DiGraph()
g.add_node("a")
g2 = graph.DiGraph()
g2.add_node('c')
self.assertRaises(ValueError,
graph.merge_graphs, g, g2,
overlap_detector='b')
def test_merge_edges(self):
g = graph.DiGraph()
g.add_node("a")
g.add_node("b")
g.add_edge('a', 'b')
g2 = graph.DiGraph()
g2.add_node('c')
g2.add_node('d')
g2.add_edge('c', 'd')
g3 = graph.merge_graphs(g, g2)
self.assertEqual(4, len(g3))
self.assertTrue(g3.has_edge('c', 'd'))
self.assertTrue(g3.has_edge('a', 'b'))
def compile(self, task, parent=None):
graph = gr.DiGraph(name=task.name)
graph.add_node(task, kind=TASK)
node = tr.Node(task, kind=TASK)
if parent is not None:
parent.add(node)
return graph, node
def test_directed(self):
g = graph.DiGraph()
g.add_node("a")
g.add_node("b")
g.add_edge("a", "b")
self.assertTrue(g.is_directed_acyclic())
g.add_edge("b", "a")
self.assertFalse(g.is_directed_acyclic())
def test_no_successors_no_predecessors(self):
g = graph.DiGraph()
g.add_node("a")
g.add_node("b")
g.add_node("c")
g.add_edge("b", "c")
self.assertEqual(set(['a', 'b']), set(g.no_predecessors_iter()))
self.assertEqual(set(['a', 'c']), set(g.no_successors_iter()))
def _flatten_task(self, task, parent):
"""Flattens a individual task."""
graph = gr.DiGraph(name=task.name)
graph.add_node(task)
node = tr.Node(task)
if parent is not None:
parent.add(node)
return graph, node
def _get_subgraph(self):
if self._subgraph is not None:
return self._subgraph
if self._target is None:
return self._graph
nodes = [self._target]
nodes.extend(self._graph.bfs_predecessors_iter(self._target))
self._subgraph = gr.DiGraph(data=self._graph.subgraph(nodes))
self._subgraph.freeze()
return self._subgraph
def compile(self, flow, parent=None):
"""Decomposes a flow into a graph and scope tree hierarchy."""
graph = gr.DiGraph(name=flow.name)
graph.add_node(flow, kind=FLOW, noop=True)
tree_node = tr.Node(flow, kind=FLOW, noop=True)
if parent is not None:
parent.add(tree_node)
if flow.retry is not None:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = dict(
(child, self._deep_compiler_func(child, parent=tree_node)[0])
for child in flow)
decomposed_graphs = list(six.itervalues(decomposed))
graph = gr.merge_graphs(graph,
*decomposed_graphs,
overlap_detector=_overlap_occurence_detector)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph,
u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
for node in graph.nodes_iter():
if node is not flow.retry and node is not flow:
graph.node[node].setdefault(RETRY, flow.retry)
from_nodes = [flow.retry]
connected_attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
else:
from_nodes = [flow]
connected_attr_dict = {LINK_INVARIANT: True}
connected_to = [
node for node in graph.no_predecessors_iter() if node is not flow
]
if connected_to:
# Ensure all nodes in this graph(s) that have no
# predecessors depend on this flow (or this flow's retry) so that
# we can depend on the flow being traversed before its
# children (even though at the current time it will be skipped).
_add_update_edges(graph,
from_nodes,
connected_to,
attr_dict=connected_attr_dict)
return graph, tree_node
def _link(self, u, v, graph=None, reason=None, manual=False):
mutable_graph = True
if graph is None:
graph = self._graph
mutable_graph = False
# NOTE(harlowja): Add an edge to a temporary copy and only if that
# copy is valid then do we swap with the underlying graph.
attrs = graph.get_edge_data(u, v)
if not attrs:
attrs = {}
if manual:
attrs[flow.LINK_MANUAL] = True
if reason is not None:
if flow.LINK_REASONS not in attrs:
attrs[flow.LINK_REASONS] = set()
attrs[flow.LINK_REASONS].add(reason)
if not mutable_graph:
graph = gr.DiGraph(graph)
graph.add_edge(u, v, **attrs)
return graph
def _decompose_flow(self, flow, parent=None):
"""Decomposes a flow into a graph, tree node + decomposed subgraphs."""
graph = gr.DiGraph(name=flow.name)
node = tr.Node(flow)
if parent is not None:
parent.add(node)
if flow.retry is not None:
node.add(tr.Node(flow.retry))
decomposed_members = {}
for item in flow:
subgraph, _subnode = self._deep_compiler_func(item, parent=node)
decomposed_members[item] = subgraph
if subgraph.number_of_nodes():
graph = gr.merge_graphs(
graph, subgraph,
# We can specialize this to be simpler than the default
# algorithm which creates overhead that we don't
# need for our purposes...
overlap_detector=self._occurence_detector)
return graph, node, decomposed_members
def _link(self,
u,
v,
graph=None,
reason=None,
manual=False,
decider=None,
decider_depth=None):
mutable_graph = True
if graph is None:
graph = self._graph
mutable_graph = False
# NOTE(harlowja): Add an edge to a temporary copy and only if that
# copy is valid then do we swap with the underlying graph.
attrs = graph.get_edge_data(u, v)
if not attrs:
attrs = {}
if decider is not None:
attrs[flow.LINK_DECIDER] = decider
try:
# Remove existing decider depth, if one existed.
del attrs[flow.LINK_DECIDER_DEPTH]
except KeyError:
pass
if decider_depth is not None:
if decider is None:
raise ValueError("Decider depth requires a decider to be"
" provided along with it")
else:
decider_depth = de.Depth.translate(decider_depth)
attrs[flow.LINK_DECIDER_DEPTH] = decider_depth
if manual:
attrs[flow.LINK_MANUAL] = True
if reason is not None:
if flow.LINK_REASONS not in attrs:
attrs[flow.LINK_REASONS] = set()
attrs[flow.LINK_REASONS].add(reason)
if not mutable_graph:
graph = gr.DiGraph(graph)
graph.add_edge(u, v, **attrs)
return graph
def _flatten_flow(self, flow, parent):
"""Flattens a flow."""
graph = gr.DiGraph(name=flow.name)
node = tr.Node(flow)
if parent is not None:
parent.add(node)
if flow.retry is not None:
node.add(tr.Node(flow.retry))
# Flatten all nodes into a single subgraph per item (and track origin
# item to its newly expanded graph).
subgraphs = {}
for item in flow:
subgraph = self._flatten(item, node)[0]
subgraphs[item] = subgraph
graph = gr.merge_graphs([graph, subgraph])
# Reconnect all items edges to their corresponding subgraphs.
for (u, v, attrs) in flow.iter_links():
u_g = subgraphs[u]
v_g = subgraphs[v]
if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')):
# Connect nodes with no predecessors in v to nodes with
# no successors in u (thus maintaining the edge dependency).
self._add_new_edges(graph,
u_g.no_successors_iter(),
v_g.no_predecessors_iter(),
edge_attrs=attrs)
else:
# This is dependency-only edge, connect corresponding
# providers and consumers.
for provider in u_g:
for consumer in v_g:
reasons = provider.provides & consumer.requires
if reasons:
graph.add_edge(provider, consumer, reasons=reasons)
if flow.retry is not None:
self._connect_retry(flow.retry, graph)
return graph, node
def compile(self, flow, parent=None):
"""Decomposes a flow into a graph and scope tree hierarchy."""
graph = gr.DiGraph(name=flow.name)
graph.add_node(flow, kind=FLOW, noop=True)
tree_node = tr.Node(flow, kind=FLOW, noop=True)
if parent is not None:
parent.add(tree_node)
if flow.retry is not None:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = dict(
(child, self._deep_compiler_func(child, parent=tree_node)[0])
for child in flow)
decomposed_graphs = list(six.itervalues(decomposed))
graph = gr.merge_graphs(graph, *decomposed_graphs,
overlap_detector=_overlap_occurence_detector)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph, u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
# Insert the flow(s) retry if needed, and always make sure it
# is the **immediate** successor of the flow node itself.
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
for node in graph.nodes_iter():
if node is not flow.retry and node is not flow:
graph.node[node].setdefault(RETRY, flow.retry)
from_nodes = [flow.retry]
attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
else:
from_nodes = [flow]
attr_dict = {LINK_INVARIANT: True}
# Ensure all nodes with no predecessors are connected to this flow
# or its retry node (so that the invariant that the flow node is
# traversed through before its contents is maintained); this allows
# us to easily know when we have entered a flow (when running) and
# do special and/or smart things such as only traverse up to the
# start of a flow when looking for node deciders.
_add_update_edges(graph, from_nodes, [
node for node in graph.no_predecessors_iter()
if node is not flow
], attr_dict=attr_dict)
# Connect all nodes with no successors into a special terminator
# that is used to identify the end of the flow and ensure that all
# execution traversals will traverse over this node before executing
# further work (this is especially useful for nesting and knowing
# when we have exited a nesting level); it allows us to do special
# and/or smart things such as applying deciders up to (but not
# beyond) a flow termination point.
#
# Do note that in a empty flow this will just connect itself to
# the flow node itself... and also note we can not use the flow
# object itself (primarily because the underlying graph library
# uses hashing to identify node uniqueness and we can easily create
# a loop if we don't do this correctly, so avoid that by just
# creating this special node and tagging it with a special kind); we
# may be able to make this better in the future with a multidigraph
# that networkx provides??
flow_term = Terminator(flow)
graph.add_node(flow_term, kind=FLOW_END, noop=True)
_add_update_edges(graph, [
node for node in graph.no_successors_iter()
if node is not flow_term
], [flow_term], attr_dict={LINK_INVARIANT: True})
return graph, tree_node
def test_frozen(self):
g = graph.DiGraph()
self.assertFalse(g.frozen)
g.add_node("b")
g.freeze()
self.assertRaises(nx.NetworkXError, g.add_node, "c")
def __init__(self, name, retry=None):
super(Flow, self).__init__(name, retry)
self._graph = gr.DiGraph(name=name)
self._graph.freeze()
def add(self, *items, **kwargs):
"""Adds a given task/tasks/flow/flows to this flow.
:param items: items to add to the flow
:param kwargs: keyword arguments, the two keyword arguments
currently processed are:
* ``resolve_requires`` a boolean that when true (the
default) implies that when items are added their
symbol requirements will be matched to existing items
and links will be automatically made to those
providers. If multiple possible providers exist
then a AmbiguousDependency exception will be raised.
* ``resolve_existing``, a boolean that when true (the
default) implies that on addition of a new item that
existing items will have their requirements scanned
for symbols that this newly added item can provide.
If a match is found a link is automatically created
from the newly added item to the requiree.
"""
items = [i for i in items if not self._graph.has_node(i)]
if not items:
return self
# This syntax will *hopefully* be better in future versions of python.
#
# See: http://legacy.python.org/dev/peps/pep-3102/ (python 3.0+)
resolve_requires = bool(kwargs.get('resolve_requires', True))
resolve_existing = bool(kwargs.get('resolve_existing', True))
# Figure out what the existing nodes *still* require and what they
# provide so we can do this lookup later when inferring.
required = collections.defaultdict(list)
provided = collections.defaultdict(list)
retry_provides = set()
if self._retry is not None:
for value in self._retry.requires:
required[value].append(self._retry)
for value in self._retry.provides:
retry_provides.add(value)
provided[value].append(self._retry)
for item in self._graph.nodes_iter():
for value in _unsatisfied_requires(item, self._graph,
retry_provides):
required[value].append(item)
for value in item.provides:
provided[value].append(item)
# NOTE(harlowja): Add items and edges to a temporary copy of the
# underlying graph and only if that is successful added to do we then
# swap with the underlying graph.
tmp_graph = gr.DiGraph(self._graph)
for item in items:
tmp_graph.add_node(item)
# Try to find a valid provider.
if resolve_requires:
for value in _unsatisfied_requires(item, tmp_graph,
retry_provides):
if value in provided:
providers = provided[value]
if len(providers) > 1:
provider_names = [n.name for n in providers]
raise exc.AmbiguousDependency(
"Resolution error detected when"
" adding %(item)s, multiple"
" providers %(providers)s found for"
" required symbol '%(value)s'" %
dict(item=item.name,
providers=sorted(provider_names),
value=value))
else:
self._link(providers[0],
item,
graph=tmp_graph,
reason=value)
else:
required[value].append(item)
for value in item.provides:
provided[value].append(item)
# See if what we provide fulfills any existing requiree.
if resolve_existing:
for value in item.provides:
if value in required:
for requiree in list(required[value]):
if requiree is not item:
self._link(item,
requiree,
graph=tmp_graph,
reason=value)
required[value].remove(requiree)
self._swap(tmp_graph)
return self
def add(self, *nodes, **kwargs):
"""Adds a given task/tasks/flow/flows to this flow.
Note that if the addition of these nodes (and any edges) creates
a `cyclic`_ graph then
a :class:`~taskflow.exceptions.DependencyFailure` will be
raised and the applied changes will be discarded.
:param nodes: node(s) to add to the flow
:param kwargs: keyword arguments, the two keyword arguments
currently processed are:
* ``resolve_requires`` a boolean that when true (the
default) implies that when node(s) are added their
symbol requirements will be matched to existing
node(s) and links will be automatically made to those
providers. If multiple possible providers exist
then a
:class:`~taskflow.exceptions.AmbiguousDependency`
exception will be raised and the provided additions
will be discarded.
* ``resolve_existing``, a boolean that when true (the
default) implies that on addition of a new node that
existing node(s) will have their requirements scanned
for symbols that this newly added node can provide.
If a match is found a link is automatically created
from the newly added node to the requiree.
.. _cyclic: https://en.wikipedia.org/wiki/Cycle_graph
"""
# Let's try to avoid doing any work if we can; since the below code
# after this filter can create more temporary graphs that aren't needed
# if the nodes already exist...
nodes = [i for i in nodes if not self._graph.has_node(i)]
if not nodes:
return self
# This syntax will *hopefully* be better in future versions of python.
#
# See: http://legacy.python.org/dev/peps/pep-3102/ (python 3.0+)
resolve_requires = bool(kwargs.get('resolve_requires', True))
resolve_existing = bool(kwargs.get('resolve_existing', True))
# Figure out what the existing nodes *still* require and what they
# provide so we can do this lookup later when inferring.
required = collections.defaultdict(list)
provided = collections.defaultdict(list)
retry_provides = set()
if self._retry is not None:
for value in self._retry.requires:
required[value].append(self._retry)
for value in self._retry.provides:
retry_provides.add(value)
provided[value].append(self._retry)
for node in self._graph.nodes_iter():
for value in self._unsatisfied_requires(node, self._graph,
retry_provides):
required[value].append(node)
for value in node.provides:
provided[value].append(node)
# NOTE(harlowja): Add node(s) and edge(s) to a temporary copy of the
# underlying graph and only if that is successful added to do we then
# swap with the underlying graph.
tmp_graph = gr.DiGraph(self._graph)
for node in nodes:
tmp_graph.add_node(node)
# Try to find a valid provider.
if resolve_requires:
for value in self._unsatisfied_requires(
node, tmp_graph, retry_provides):
if value in provided:
providers = provided[value]
if len(providers) > 1:
provider_names = [n.name for n in providers]
raise exc.AmbiguousDependency(
"Resolution error detected when"
" adding '%(node)s', multiple"
" providers %(providers)s found for"
" required symbol '%(value)s'" %
dict(node=node.name,
providers=sorted(provider_names),
value=value))
else:
self._link(providers[0],
node,
graph=tmp_graph,
reason=value)
else:
required[value].append(node)
for value in node.provides:
provided[value].append(node)
# See if what we provide fulfills any existing requiree.
if resolve_existing:
for value in node.provides:
if value in required:
for requiree in list(required[value]):
if requiree is not node:
self._link(node,
requiree,
graph=tmp_graph,
reason=value)
required[value].remove(requiree)
self._swap(tmp_graph)
return self
