def test_pformat_flat(self):
root = tree.Node("josh")
root.add(tree.Node("josh.1"))
expected = """
josh
|__josh.1
"""
self.assertEqual(expected.strip(), root.pformat())
root[0].add(tree.Node("josh.1.1"))
expected = """
josh
|__josh.1
|__josh.1.1
"""
self.assertEqual(expected.strip(), root.pformat())
root[0][0].add(tree.Node("josh.1.1.1"))
expected = """
josh
|__josh.1
|__josh.1.1
|__josh.1.1.1
"""
self.assertEqual(expected.strip(), root.pformat())
root[0][0][0].add(tree.Node("josh.1.1.1.1"))
expected = """
josh
|__josh.1
|__josh.1.1
|__josh.1.1.1
|__josh.1.1.1.1
"""
self.assertEqual(expected.strip(), root.pformat())
def test_to_digraph_retains_metadata(self):
root = tree.Node("chickens", alive=True)
dead_chicken = tree.Node("chicken.1", alive=False)
root.add(dead_chicken)
g = root.to_digraph()
self.assertEqual(g.nodes['chickens'], {'alive': True})
self.assertEqual(g.nodes['chicken.1'], {'alive': False})
def test_after_frozen(self):
root = tree.Node("josh")
root.add(tree.Node("josh.1"))
root.freeze()
self.assertTrue(all(n.frozen
for n in root.dfs_iter(include_self=True)))
self.assertRaises(tree.FrozenNode, root.remove, "josh.1")
self.assertRaises(tree.FrozenNode, root.disassociate)
self.assertRaises(tree.FrozenNode, root.add, tree.Node("josh.2"))
def _fetch_predecessor_tree(graph, atom):
"""Creates a tree of predecessors, rooted at given atom."""
root = tree.Node(atom)
stack = [(root, atom)]
seen = set()
while stack:
parent, node = stack.pop()
for pred_node in graph.predecessors_iter(node):
child = tree.Node(pred_node, **graph.node[pred_node])
parent.add(child)
stack.append((child, pred_node))
seen.add(pred_node)
return len(seen), root
def _decompose_flow(self, flow, parent):
"""Decomposes a flow into a graph, tree node + decomposed subgraphs."""
graph = gr.DiGraph(name=flow.name)
node = tr.Node(flow)
if parent is not None:
parent.add(node)
if flow.retry is not None:
node.add(tr.Node(flow.retry))
decomposed_members = {}
for item in flow:
subgraph, _subnode = self._flatten(item, node)
decomposed_members[item] = subgraph
if subgraph.number_of_nodes():
graph = gr.merge_graphs([graph, subgraph])
return graph, node, decomposed_members
def compile(self, flow, parent=None):
"""Decomposes a flow into a graph and scope tree hierarchy."""
graph = gr.DiGraph(name=flow.name)
graph.add_node(flow, kind=FLOW, noop=True)
tree_node = tr.Node(flow, kind=FLOW, noop=True)
if parent is not None:
parent.add(tree_node)
if flow.retry is not None:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = dict(
(child, self._deep_compiler_func(child, parent=tree_node)[0])
for child in flow)
decomposed_graphs = list(six.itervalues(decomposed))
graph = gr.merge_graphs(graph,
*decomposed_graphs,
overlap_detector=_overlap_occurence_detector)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph,
u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
for node in graph.nodes_iter():
if node is not flow.retry and node is not flow:
graph.node[node].setdefault(RETRY, flow.retry)
from_nodes = [flow.retry]
connected_attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
else:
from_nodes = [flow]
connected_attr_dict = {LINK_INVARIANT: True}
connected_to = [
node for node in graph.no_predecessors_iter() if node is not flow
]
if connected_to:
# Ensure all nodes in this graph(s) that have no
# predecessors depend on this flow (or this flow's retry) so that
# we can depend on the flow being traversed before its
# children (even though at the current time it will be skipped).
_add_update_edges(graph,
from_nodes,
connected_to,
attr_dict=connected_attr_dict)
return graph, tree_node
def compile(self, task, parent=None):
graph = gr.DiGraph(name=task.name)
graph.add_node(task, kind=TASK)
node = tr.Node(task, kind=TASK)
if parent is not None:
parent.add(node)
return graph, node
def _flatten_task(self, task, parent):
"""Flattens a individual task."""
graph = gr.DiGraph(name=task.name)
graph.add_node(task)
node = tr.Node(task)
if parent is not None:
parent.add(node)
return graph, node
def _fetch_predecessor_tree(graph, atom):
"""Creates a tree of predecessors, rooted at given atom."""
root = tree.Node(atom)
stack = [(root, atom)]
while stack:
parent, node = stack.pop()
for pred_node in graph.predecessors(node):
pred_node_data = graph.nodes[pred_node]
if pred_node_data['kind'] == compiler.FLOW_END:
# Jump over and/or don't show flow end nodes...
for pred_pred_node in graph.predecessors(pred_node):
stack.append((parent, pred_pred_node))
else:
child = tree.Node(pred_node, **pred_node_data)
parent.add(child)
# And go further backwards...
stack.append((child, pred_node))
return root
def __setitem__(self, path, value):
path = self._normpath(path)
value = self._copier(value)
try:
item_node = self._fetch_node(path)
item_node.metadata.update(value=value)
except exc.NotFound:
dirname, basename = os.path.split(path)
parent_node = self._fetch_node(dirname)
parent_node.add(tree.Node(basename, value=value))
def _decompose_flow(self, flow, parent=None):
"""Decomposes a flow into a graph, tree node + decomposed subgraphs."""
graph = gr.DiGraph(name=flow.name)
node = tr.Node(flow)
if parent is not None:
parent.add(node)
if flow.retry is not None:
node.add(tr.Node(flow.retry))
decomposed_members = {}
for item in flow:
subgraph, _subnode = self._deep_compiler_func(item, parent=node)
decomposed_members[item] = subgraph
if subgraph.number_of_nodes():
graph = gr.merge_graphs(
graph, subgraph,
# We can specialize this to be simpler than the default
# algorithm which creates overhead that we don't
# need for our purposes...
overlap_detector=self._occurence_detector)
return graph, node, decomposed_members
def symlink(self, src_path, dest_path):
dest_path = self._normpath(dest_path)
src_path = self._normpath(src_path)
dirname, basename = os.path.split(dest_path)
parent_node = self._fetch_node(dirname)
child_node = parent_node.find(basename,
only_direct=True,
include_self=False)
if child_node is None:
child_node = tree.Node(basename, value=None)
parent_node.add(child_node)
child_node.metadata['target'] = src_path
def _flatten_flow(self, flow, parent):
"""Flattens a flow."""
graph = gr.DiGraph(name=flow.name)
node = tr.Node(flow)
if parent is not None:
parent.add(node)
if flow.retry is not None:
node.add(tr.Node(flow.retry))
# Flatten all nodes into a single subgraph per item (and track origin
# item to its newly expanded graph).
subgraphs = {}
for item in flow:
subgraph = self._flatten(item, node)[0]
subgraphs[item] = subgraph
graph = gr.merge_graphs([graph, subgraph])
# Reconnect all items edges to their corresponding subgraphs.
for (u, v, attrs) in flow.iter_links():
u_g = subgraphs[u]
v_g = subgraphs[v]
if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')):
# Connect nodes with no predecessors in v to nodes with
# no successors in u (thus maintaining the edge dependency).
self._add_new_edges(graph,
u_g.no_successors_iter(),
v_g.no_predecessors_iter(),
edge_attrs=attrs)
else:
# This is dependency-only edge, connect corresponding
# providers and consumers.
for provider in u_g:
for consumer in v_g:
reasons = provider.provides & consumer.requires
if reasons:
graph.add_edge(provider, consumer, reasons=reasons)
if flow.retry is not None:
self._connect_retry(flow.retry, graph)
return graph, node
def ensure_path(self, path):
path = self._normpath(path)
# Ignore the root path as we already checked for that; and it
# will always exist/can't be removed anyway...
if path == self._root.item:
return
node = self._root
for piece in self._iter_pieces(path):
child_node = node.find(piece, only_direct=True,
include_self=False)
if child_node is None:
child_node = tree.Node(piece, value=None)
node.add(child_node)
node = child_node
def _make_species(self):
# This is the following tree:
#
# animal
# |__mammal
# | |__horse
# | |__primate
# | |__monkey
# | |__human
# |__reptile
a = tree.Node("animal")
m = tree.Node("mammal")
r = tree.Node("reptile")
a.add(m)
a.add(r)
m.add(tree.Node("horse"))
p = tree.Node("primate")
m.add(p)
p.add(tree.Node("monkey"))
p.add(tree.Node("human"))
return a
def test_pformat(self):
root = tree.Node("CEO")
expected = """
CEO
"""
self.assertEqual(expected.strip(), root.pformat())
root.add(tree.Node("Infra"))
expected = """
CEO
|__Infra
"""
self.assertEqual(expected.strip(), root.pformat())
root[0].add(tree.Node("Infra.1"))
expected = """
CEO
|__Infra
|__Infra.1
"""
self.assertEqual(expected.strip(), root.pformat())
root.add(tree.Node("Mail"))
expected = """
CEO
|__Infra
| |__Infra.1
|__Mail
"""
self.assertEqual(expected.strip(), root.pformat())
root.add(tree.Node("Search"))
expected = """
CEO
|__Infra
| |__Infra.1
|__Mail
|__Search
"""
self.assertEqual(expected.strip(), root.pformat())
root[-1].add(tree.Node("Search.1"))
expected = """
CEO
|__Infra
| |__Infra.1
|__Mail
|__Search
|__Search.1
"""
self.assertEqual(expected.strip(), root.pformat())
root[-1].add(tree.Node("Search.2"))
expected = """
CEO
|__Infra
| |__Infra.1
|__Mail
|__Search
|__Search.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
root[0].add(tree.Node("Infra.2"))
expected = """
CEO
|__Infra
| |__Infra.1
| |__Infra.2
|__Mail
|__Search
|__Search.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
root[0].add(tree.Node("Infra.3"))
expected = """
CEO
|__Infra
| |__Infra.1
| |__Infra.2
| |__Infra.3
|__Mail
|__Search
|__Search.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
root[0][-1].add(tree.Node("Infra.3.1"))
expected = """
CEO
|__Infra
| |__Infra.1
| |__Infra.2
| |__Infra.3
| |__Infra.3.1
|__Mail
|__Search
|__Search.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
root[-1][0].add(tree.Node("Search.1.1"))
expected = """
CEO
|__Infra
| |__Infra.1
| |__Infra.2
| |__Infra.3
| |__Infra.3.1
|__Mail
|__Search
|__Search.1
| |__Search.1.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
root[1].add(tree.Node("Mail.1"))
expected = """
CEO
|__Infra
| |__Infra.1
| |__Infra.2
| |__Infra.3
| |__Infra.3.1
|__Mail
| |__Mail.1
|__Search
|__Search.1
| |__Search.1.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
root[1][0].add(tree.Node("Mail.1.1"))
expected = """
CEO
|__Infra
| |__Infra.1
| |__Infra.2
| |__Infra.3
| |__Infra.3.1
|__Mail
| |__Mail.1
| |__Mail.1.1
|__Search
|__Search.1
| |__Search.1.1
|__Search.2
"""
self.assertEqual(expected.strip(), root.pformat())
def compile(self, flow, parent=None):
"""Decomposes a flow into a graph and scope tree hierarchy."""
graph = gr.DiGraph(name=flow.name)
graph.add_node(flow, kind=FLOW, noop=True)
tree_node = tr.Node(flow, kind=FLOW, noop=True)
if parent is not None:
parent.add(tree_node)
if flow.retry is not None:
tree_node.add(tr.Node(flow.retry, kind=RETRY))
decomposed = dict(
(child, self._deep_compiler_func(child, parent=tree_node)[0])
for child in flow)
decomposed_graphs = list(six.itervalues(decomposed))
graph = gr.merge_graphs(graph, *decomposed_graphs,
overlap_detector=_overlap_occurence_detector)
for u, v, attr_dict in flow.iter_links():
u_graph = decomposed[u]
v_graph = decomposed[v]
_add_update_edges(graph, u_graph.no_successors_iter(),
list(v_graph.no_predecessors_iter()),
attr_dict=attr_dict)
# Insert the flow(s) retry if needed, and always make sure it
# is the **immediate** successor of the flow node itself.
if flow.retry is not None:
graph.add_node(flow.retry, kind=RETRY)
_add_update_edges(graph, [flow], [flow.retry],
attr_dict={LINK_INVARIANT: True})
for node in graph.nodes_iter():
if node is not flow.retry and node is not flow:
graph.node[node].setdefault(RETRY, flow.retry)
from_nodes = [flow.retry]
attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
else:
from_nodes = [flow]
attr_dict = {LINK_INVARIANT: True}
# Ensure all nodes with no predecessors are connected to this flow
# or its retry node (so that the invariant that the flow node is
# traversed through before its contents is maintained); this allows
# us to easily know when we have entered a flow (when running) and
# do special and/or smart things such as only traverse up to the
# start of a flow when looking for node deciders.
_add_update_edges(graph, from_nodes, [
node for node in graph.no_predecessors_iter()
if node is not flow
], attr_dict=attr_dict)
# Connect all nodes with no successors into a special terminator
# that is used to identify the end of the flow and ensure that all
# execution traversals will traverse over this node before executing
# further work (this is especially useful for nesting and knowing
# when we have exited a nesting level); it allows us to do special
# and/or smart things such as applying deciders up to (but not
# beyond) a flow termination point.
#
# Do note that in a empty flow this will just connect itself to
# the flow node itself... and also note we can not use the flow
# object itself (primarily because the underlying graph library
# uses hashing to identify node uniqueness and we can easily create
# a loop if we don't do this correctly, so avoid that by just
# creating this special node and tagging it with a special kind); we
# may be able to make this better in the future with a multidigraph
# that networkx provides??
flow_term = Terminator(flow)
graph.add_node(flow_term, kind=FLOW_END, noop=True)
_add_update_edges(graph, [
node for node in graph.no_successors_iter()
if node is not flow_term
], [flow_term], attr_dict={LINK_INVARIANT: True})
return graph, tree_node
def test_empty(self):
root = tree.Node("josh")
self.assertTrue(root.empty())
def __init__(self, deep_copy=True):
self._root = tree.Node(self.root_path, value=None)
if deep_copy:
self._copier = copy.deepcopy
else:
self._copier = copy.copy
