def compile(self, task, parent=None):
graph = gr.DiGraph(name=task.name)
graph.add_node(task, kind=TASK)
node = tr.Node(task, kind=TASK)
if parent is not None:
parent.add(node)
return graph, node
def _link(self,
u,
v,
graph=None,
reason=None,
manual=False,
decider=None,
decider_depth=None):
mutable_graph = True
if graph is None:
graph = self._graph
mutable_graph = False
# NOTE(harlowja): Add an edge to a temporary copy and only if that
# copy is valid then do we swap with the underlying graph.
attrs = graph.get_edge_data(u, v)
if not attrs:
attrs = {}
if decider is not None:
attrs[flow.LINK_DECIDER] = decider
try:
# Remove existing decider depth, if one existed.
del attrs[flow.LINK_DECIDER_DEPTH]
except KeyError:
pass
if decider_depth is not None:
if decider is None:
raise ValueError("Decider depth requires a decider to be"
" provided along with it")
else:
decider_depth = de.Depth.translate(decider_depth)
attrs[flow.LINK_DECIDER_DEPTH] = decider_depth
if manual:
attrs[flow.LINK_MANUAL] = True
if reason is not None:
if flow.LINK_REASONS not in attrs:
attrs[flow.LINK_REASONS] = set()
attrs[flow.LINK_REASONS].add(reason)
if not mutable_graph:
graph = gr.DiGraph(graph)
graph.add_edge(u, v, **attrs)
return graph
def __init__(self, name, retry=None):
super(Flow, self).__init__(name, retry)
self._graph = gr.DiGraph(name=name)
self._graph.freeze()
def add(self, *nodes, **kwargs):
"""Adds a given task/tasks/flow/flows to this flow.
Note that if the addition of these nodes (and any edges) creates
a `cyclic`_ graph then
a :class:`~taskflow.exceptions.DependencyFailure` will be
raised and the applied changes will be discarded.
:param nodes: node(s) to add to the flow
:param kwargs: keyword arguments, the two keyword arguments
currently processed are:
* ``resolve_requires`` a boolean that when true (the
default) implies that when node(s) are added their
symbol requirements will be matched to existing
node(s) and links will be automatically made to those
providers. If multiple possible providers exist
then a
:class:`~taskflow.exceptions.AmbiguousDependency`
exception will be raised and the provided additions
will be discarded.
* ``resolve_existing``, a boolean that when true (the
default) implies that on addition of a new node that
existing node(s) will have their requirements scanned
for symbols that this newly added node can provide.
If a match is found a link is automatically created
from the newly added node to the requiree.
.. _cyclic: https://en.wikipedia.org/wiki/Cycle_graph
"""
# Let's try to avoid doing any work if we can; since the below code
# after this filter can create more temporary graphs that aren't needed
# if the nodes already exist...
nodes = [i for i in nodes if not self._graph.has_node(i)]
if not nodes:
return self
# This syntax will *hopefully* be better in future versions of python.
#
# See: http://legacy.python.org/dev/peps/pep-3102/ (python 3.0+)
resolve_requires = bool(kwargs.get('resolve_requires', True))
resolve_existing = bool(kwargs.get('resolve_existing', True))
# Figure out what the existing nodes *still* require and what they
# provide so we can do this lookup later when inferring.
required = collections.defaultdict(list)
provided = collections.defaultdict(list)
retry_provides = set()
if self._retry is not None:
for value in self._retry.requires:
required[value].append(self._retry)
for value in self._retry.provides:
retry_provides.add(value)
provided[value].append(self._retry)
for node in self._graph.nodes_iter():
for value in self._unsatisfied_requires(node, self._graph,
retry_provides):
required[value].append(node)
for value in node.provides:
provided[value].append(node)
# NOTE(harlowja): Add node(s) and edge(s) to a temporary copy of the
# underlying graph and only if that is successful added to do we then
# swap with the underlying graph.
tmp_graph = gr.DiGraph(self._graph)
for node in nodes:
tmp_graph.add_node(node)
# Try to find a valid provider.
if resolve_requires:
for value in self._unsatisfied_requires(
node, tmp_graph, retry_provides):
if value in provided:
providers = provided[value]
if len(providers) > 1:
provider_names = [n.name for n in providers]
raise exc.AmbiguousDependency(
"Resolution error detected when"
" adding '%(node)s', multiple"
" providers %(providers)s found for"
" required symbol '%(value)s'" %
dict(node=node.name,
providers=sorted(provider_names),
value=value))
else:
self._link(providers[0],
node,
graph=tmp_graph,
reason=value)
else:
required[value].append(node)
for value in node.provides:
provided[value].append(node)
# See if what we provide fulfills any existing requiree.
if resolve_existing:
for value in node.provides:
if value in required:
for requiree in list(required[value]):
if requiree is not node:
self._link(node,
requiree,
graph=tmp_graph,
reason=value)
required[value].remove(requiree)
self._swap(tmp_graph)
return self
def compile(self, flow, parent=None):
"""Decomposes a flow into a graph and scope tree hierarchy."""
graph = gr.DiGraph(name=flow.name)
graph.add_node(flow, kind=FLOW, noop=True)
tree_node = tr.Node(flow, kind=FLOW, noop=True)
if parent is not None:
parent.add(tree_node)
if flow.retry is not None:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = dict(
(child, self._deep_compiler_func(child, parent=tree_node)[0])
for child in flow)
decomposed_graphs = list(six.itervalues(decomposed))
graph = gr.merge_graphs(graph,
*decomposed_graphs,
overlap_detector=_overlap_occurrence_detector)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph,
u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
# Insert the flow(s) retry if needed, and always make sure it
# is the **immediate** successor of the flow node itself.
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
for node in graph.nodes_iter():
if node is not flow.retry and node is not flow:
graph.node[node].setdefault(RETRY, flow.retry)
from_nodes = [flow.retry]
attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
else:
from_nodes = [flow]
attr_dict = {LINK_INVARIANT: True}
# Ensure all nodes with no predecessors are connected to this flow
# or its retry node (so that the invariant that the flow node is
# traversed through before its contents is maintained); this allows
# us to easily know when we have entered a flow (when running) and
# do special and/or smart things such as only traverse up to the
# start of a flow when looking for node deciders.
_add_update_edges(
graph,
from_nodes, [
node
for node in graph.no_predecessors_iter() if node is not flow
],
attr_dict=attr_dict)
# Connect all nodes with no successors into a special terminator
# that is used to identify the end of the flow and ensure that all
# execution traversals will traverse over this node before executing
# further work (this is especially useful for nesting and knowing
# when we have exited a nesting level); it allows us to do special
# and/or smart things such as applying deciders up to (but not
# beyond) a flow termination point.
#
# Do note that in a empty flow this will just connect itself to
# the flow node itself... and also note we can not use the flow
# object itself (primarily because the underlying graph library
# uses hashing to identify node uniqueness and we can easily create
# a loop if we don't do this correctly, so avoid that by just
# creating this special node and tagging it with a special kind); we
# may be able to make this better in the future with a multidigraph
# that networkx provides??
flow_term = Terminator(flow)
graph.add_node(flow_term, kind=FLOW_END, noop=True)
_add_update_edges(graph, [
node
for node in graph.no_successors_iter() if node is not flow_term
], [flow_term],
attr_dict={LINK_INVARIANT: True})
return graph, tree_node