def test_pformat_flat(self): root = tree.Node("josh") root.add(tree.Node("josh.1")) expected = """ josh |__josh.1 """ self.assertEqual(expected.strip(), root.pformat()) root[0].add(tree.Node("josh.1.1")) expected = """ josh |__josh.1 |__josh.1.1 """ self.assertEqual(expected.strip(), root.pformat()) root[0][0].add(tree.Node("josh.1.1.1")) expected = """ josh |__josh.1 |__josh.1.1 |__josh.1.1.1 """ self.assertEqual(expected.strip(), root.pformat()) root[0][0][0].add(tree.Node("josh.1.1.1.1")) expected = """ josh |__josh.1 |__josh.1.1 |__josh.1.1.1 |__josh.1.1.1.1 """ self.assertEqual(expected.strip(), root.pformat())
def test_to_digraph_retains_metadata(self): root = tree.Node("chickens", alive=True) dead_chicken = tree.Node("chicken.1", alive=False) root.add(dead_chicken) g = root.to_digraph() self.assertEqual(g.nodes['chickens'], {'alive': True}) self.assertEqual(g.nodes['chicken.1'], {'alive': False})
def test_after_frozen(self): root = tree.Node("josh") root.add(tree.Node("josh.1")) root.freeze() self.assertTrue(all(n.frozen for n in root.dfs_iter(include_self=True))) self.assertRaises(tree.FrozenNode, root.remove, "josh.1") self.assertRaises(tree.FrozenNode, root.disassociate) self.assertRaises(tree.FrozenNode, root.add, tree.Node("josh.2"))
def _fetch_predecessor_tree(graph, atom): """Creates a tree of predecessors, rooted at given atom.""" root = tree.Node(atom) stack = [(root, atom)] seen = set() while stack: parent, node = stack.pop() for pred_node in graph.predecessors_iter(node): child = tree.Node(pred_node, **graph.node[pred_node]) parent.add(child) stack.append((child, pred_node)) seen.add(pred_node) return len(seen), root
def _decompose_flow(self, flow, parent): """Decomposes a flow into a graph, tree node + decomposed subgraphs.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) decomposed_members = {} for item in flow: subgraph, _subnode = self._flatten(item, node) decomposed_members[item] = subgraph if subgraph.number_of_nodes(): graph = gr.merge_graphs([graph, subgraph]) return graph, node, decomposed_members
def compile(self, flow, parent=None): """Decomposes a flow into a graph and scope tree hierarchy.""" graph = gr.DiGraph(name=flow.name) graph.add_node(flow, kind=FLOW, noop=True) tree_node = tr.Node(flow, kind=FLOW, noop=True) if parent is not None: parent.add(tree_node) if flow.retry is not None: tree_node.add(tr.Node(flow.retry, kind=RETRY)) decomposed = dict( (child, self._deep_compiler_func(child, parent=tree_node)[0]) for child in flow) decomposed_graphs = list(six.itervalues(decomposed)) graph = gr.merge_graphs(graph, *decomposed_graphs, overlap_detector=_overlap_occurence_detector) for u, v, attr_dict in flow.iter_links(): u_graph = decomposed[u] v_graph = decomposed[v] _add_update_edges(graph, u_graph.no_successors_iter(), list(v_graph.no_predecessors_iter()), attr_dict=attr_dict) if flow.retry is not None: graph.add_node(flow.retry, kind=RETRY) _add_update_edges(graph, [flow], [flow.retry], attr_dict={LINK_INVARIANT: True}) for node in graph.nodes_iter(): if node is not flow.retry and node is not flow: graph.node[node].setdefault(RETRY, flow.retry) from_nodes = [flow.retry] connected_attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True} else: from_nodes = [flow] connected_attr_dict = {LINK_INVARIANT: True} connected_to = [ node for node in graph.no_predecessors_iter() if node is not flow ] if connected_to: # Ensure all nodes in this graph(s) that have no # predecessors depend on this flow (or this flow's retry) so that # we can depend on the flow being traversed before its # children (even though at the current time it will be skipped). _add_update_edges(graph, from_nodes, connected_to, attr_dict=connected_attr_dict) return graph, tree_node
def compile(self, task, parent=None): graph = gr.DiGraph(name=task.name) graph.add_node(task, kind=TASK) node = tr.Node(task, kind=TASK) if parent is not None: parent.add(node) return graph, node
def _flatten_task(self, task, parent): """Flattens a individual task.""" graph = gr.DiGraph(name=task.name) graph.add_node(task) node = tr.Node(task) if parent is not None: parent.add(node) return graph, node
def _fetch_predecessor_tree(graph, atom): """Creates a tree of predecessors, rooted at given atom.""" root = tree.Node(atom) stack = [(root, atom)] while stack: parent, node = stack.pop() for pred_node in graph.predecessors(node): pred_node_data = graph.nodes[pred_node] if pred_node_data['kind'] == compiler.FLOW_END: # Jump over and/or don't show flow end nodes... for pred_pred_node in graph.predecessors(pred_node): stack.append((parent, pred_pred_node)) else: child = tree.Node(pred_node, **pred_node_data) parent.add(child) # And go further backwards... stack.append((child, pred_node)) return root
def __setitem__(self, path, value): path = self._normpath(path) value = self._copier(value) try: item_node = self._fetch_node(path) item_node.metadata.update(value=value) except exc.NotFound: dirname, basename = os.path.split(path) parent_node = self._fetch_node(dirname) parent_node.add(tree.Node(basename, value=value))
def _decompose_flow(self, flow, parent=None): """Decomposes a flow into a graph, tree node + decomposed subgraphs.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) decomposed_members = {} for item in flow: subgraph, _subnode = self._deep_compiler_func(item, parent=node) decomposed_members[item] = subgraph if subgraph.number_of_nodes(): graph = gr.merge_graphs( graph, subgraph, # We can specialize this to be simpler than the default # algorithm which creates overhead that we don't # need for our purposes... overlap_detector=self._occurence_detector) return graph, node, decomposed_members
def symlink(self, src_path, dest_path): dest_path = self._normpath(dest_path) src_path = self._normpath(src_path) dirname, basename = os.path.split(dest_path) parent_node = self._fetch_node(dirname) child_node = parent_node.find(basename, only_direct=True, include_self=False) if child_node is None: child_node = tree.Node(basename, value=None) parent_node.add(child_node) child_node.metadata['target'] = src_path
def _flatten_flow(self, flow, parent): """Flattens a flow.""" graph = gr.DiGraph(name=flow.name) node = tr.Node(flow) if parent is not None: parent.add(node) if flow.retry is not None: node.add(tr.Node(flow.retry)) # Flatten all nodes into a single subgraph per item (and track origin # item to its newly expanded graph). subgraphs = {} for item in flow: subgraph = self._flatten(item, node)[0] subgraphs[item] = subgraph graph = gr.merge_graphs([graph, subgraph]) # Reconnect all items edges to their corresponding subgraphs. for (u, v, attrs) in flow.iter_links(): u_g = subgraphs[u] v_g = subgraphs[v] if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')): # Connect nodes with no predecessors in v to nodes with # no successors in u (thus maintaining the edge dependency). self._add_new_edges(graph, u_g.no_successors_iter(), v_g.no_predecessors_iter(), edge_attrs=attrs) else: # This is dependency-only edge, connect corresponding # providers and consumers. for provider in u_g: for consumer in v_g: reasons = provider.provides & consumer.requires if reasons: graph.add_edge(provider, consumer, reasons=reasons) if flow.retry is not None: self._connect_retry(flow.retry, graph) return graph, node
def ensure_path(self, path): path = self._normpath(path) # Ignore the root path as we already checked for that; and it # will always exist/can't be removed anyway... if path == self._root.item: return node = self._root for piece in self._iter_pieces(path): child_node = node.find(piece, only_direct=True, include_self=False) if child_node is None: child_node = tree.Node(piece, value=None) node.add(child_node) node = child_node
def _make_species(self): # This is the following tree: # # animal # |__mammal # | |__horse # | |__primate # | |__monkey # | |__human # |__reptile a = tree.Node("animal") m = tree.Node("mammal") r = tree.Node("reptile") a.add(m) a.add(r) m.add(tree.Node("horse")) p = tree.Node("primate") m.add(p) p.add(tree.Node("monkey")) p.add(tree.Node("human")) return a
def test_pformat(self): root = tree.Node("CEO") expected = """ CEO """ self.assertEqual(expected.strip(), root.pformat()) root.add(tree.Node("Infra")) expected = """ CEO |__Infra """ self.assertEqual(expected.strip(), root.pformat()) root[0].add(tree.Node("Infra.1")) expected = """ CEO |__Infra |__Infra.1 """ self.assertEqual(expected.strip(), root.pformat()) root.add(tree.Node("Mail")) expected = """ CEO |__Infra | |__Infra.1 |__Mail """ self.assertEqual(expected.strip(), root.pformat()) root.add(tree.Node("Search")) expected = """ CEO |__Infra | |__Infra.1 |__Mail |__Search """ self.assertEqual(expected.strip(), root.pformat()) root[-1].add(tree.Node("Search.1")) expected = """ CEO |__Infra | |__Infra.1 |__Mail |__Search |__Search.1 """ self.assertEqual(expected.strip(), root.pformat()) root[-1].add(tree.Node("Search.2")) expected = """ CEO |__Infra | |__Infra.1 |__Mail |__Search |__Search.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat()) root[0].add(tree.Node("Infra.2")) expected = """ CEO |__Infra | |__Infra.1 | |__Infra.2 |__Mail |__Search |__Search.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat()) root[0].add(tree.Node("Infra.3")) expected = """ CEO |__Infra | |__Infra.1 | |__Infra.2 | |__Infra.3 |__Mail |__Search |__Search.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat()) root[0][-1].add(tree.Node("Infra.3.1")) expected = """ CEO |__Infra | |__Infra.1 | |__Infra.2 | |__Infra.3 | |__Infra.3.1 |__Mail |__Search |__Search.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat()) root[-1][0].add(tree.Node("Search.1.1")) expected = """ CEO |__Infra | |__Infra.1 | |__Infra.2 | |__Infra.3 | |__Infra.3.1 |__Mail |__Search |__Search.1 | |__Search.1.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat()) root[1].add(tree.Node("Mail.1")) expected = """ CEO |__Infra | |__Infra.1 | |__Infra.2 | |__Infra.3 | |__Infra.3.1 |__Mail | |__Mail.1 |__Search |__Search.1 | |__Search.1.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat()) root[1][0].add(tree.Node("Mail.1.1")) expected = """ CEO |__Infra | |__Infra.1 | |__Infra.2 | |__Infra.3 | |__Infra.3.1 |__Mail | |__Mail.1 | |__Mail.1.1 |__Search |__Search.1 | |__Search.1.1 |__Search.2 """ self.assertEqual(expected.strip(), root.pformat())
def compile(self, flow, parent=None): """Decomposes a flow into a graph and scope tree hierarchy.""" graph = gr.DiGraph(name=flow.name) graph.add_node(flow, kind=FLOW, noop=True) tree_node = tr.Node(flow, kind=FLOW, noop=True) if parent is not None: parent.add(tree_node) if flow.retry is not None: tree_node.add(tr.Node(flow.retry, kind=RETRY)) decomposed = dict( (child, self._deep_compiler_func(child, parent=tree_node)[0]) for child in flow) decomposed_graphs = list(six.itervalues(decomposed)) graph = gr.merge_graphs(graph, *decomposed_graphs, overlap_detector=_overlap_occurence_detector) for u, v, attr_dict in flow.iter_links(): u_graph = decomposed[u] v_graph = decomposed[v] _add_update_edges(graph, u_graph.no_successors_iter(), list(v_graph.no_predecessors_iter()), attr_dict=attr_dict) # Insert the flow(s) retry if needed, and always make sure it # is the **immediate** successor of the flow node itself. if flow.retry is not None: graph.add_node(flow.retry, kind=RETRY) _add_update_edges(graph, [flow], [flow.retry], attr_dict={LINK_INVARIANT: True}) for node in graph.nodes_iter(): if node is not flow.retry and node is not flow: graph.node[node].setdefault(RETRY, flow.retry) from_nodes = [flow.retry] attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True} else: from_nodes = [flow] attr_dict = {LINK_INVARIANT: True} # Ensure all nodes with no predecessors are connected to this flow # or its retry node (so that the invariant that the flow node is # traversed through before its contents is maintained); this allows # us to easily know when we have entered a flow (when running) and # do special and/or smart things such as only traverse up to the # start of a flow when looking for node deciders. _add_update_edges(graph, from_nodes, [ node for node in graph.no_predecessors_iter() if node is not flow ], attr_dict=attr_dict) # Connect all nodes with no successors into a special terminator # that is used to identify the end of the flow and ensure that all # execution traversals will traverse over this node before executing # further work (this is especially useful for nesting and knowing # when we have exited a nesting level); it allows us to do special # and/or smart things such as applying deciders up to (but not # beyond) a flow termination point. # # Do note that in a empty flow this will just connect itself to # the flow node itself... and also note we can not use the flow # object itself (primarily because the underlying graph library # uses hashing to identify node uniqueness and we can easily create # a loop if we don't do this correctly, so avoid that by just # creating this special node and tagging it with a special kind); we # may be able to make this better in the future with a multidigraph # that networkx provides?? flow_term = Terminator(flow) graph.add_node(flow_term, kind=FLOW_END, noop=True) _add_update_edges(graph, [ node for node in graph.no_successors_iter() if node is not flow_term ], [flow_term], attr_dict={LINK_INVARIANT: True}) return graph, tree_node
def test_empty(self): root = tree.Node("josh") self.assertTrue(root.empty())
def __init__(self, deep_copy=True): self._root = tree.Node(self.root_path, value=None) if deep_copy: self._copier = copy.deepcopy else: self._copier = copy.copy