コード例 #1
0
ファイル: compiler.py プロジェクト: rupa108/taskflow
    def _flatten_flow(self, flow):
        """Flattens a graph flow."""
        graph = gr.DiGraph(name=flow.name)

        # Flatten all nodes into a single subgraph per node.
        subgraph_map = {}
        for item in flow:
            subgraph = self._flatten(item)
            subgraph_map[item] = subgraph
            graph = gr.merge_graphs([graph, subgraph])

        # Reconnect all node edges to their corresponding subgraphs.
        for (u, v, attrs) in flow.iter_links():
            u_g = subgraph_map[u]
            v_g = subgraph_map[v]
            if any(attrs.get(k) for k in ("invariant", "manual", "retry")):
                # Connect nodes with no predecessors in v to nodes with
                # no successors in u (thus maintaining the edge dependency).
                self._add_new_edges(graph, u_g.no_successors_iter(), v_g.no_predecessors_iter(), edge_attrs=attrs)
            else:
                # This is dependency-only edge, connect corresponding
                # providers and consumers.
                for provider in u_g:
                    for consumer in v_g:
                        reasons = provider.provides & consumer.requires
                        if reasons:
                            graph.add_edge(provider, consumer, reasons=reasons)

        if flow.retry is not None:
            self._connect_retry(flow.retry, graph)
        return graph
コード例 #2
0
ファイル: compiler.py プロジェクト: balagopalraj/clearlinux
 def apply_constraints(self, graph, flow, decomposed_members):
     # This list is used to track the links that have been previously
     # iterated over, so that when we are trying to find a entry to
     # connect to that we iterate backwards through this list, finding
     # connected nodes to the current target (lets call it v) and find
     # the first (u_n, or u_n - 1, u_n - 2...) that was decomposed into
     # a non-empty graph. We also retain all predecessors of v so that we
     # can correctly locate u_n - 1 if u_n turns out to have decomposed into
     # an empty graph (and so on).
     priors = []
     # NOTE(harlowja): u, v are flows/tasks (also graph terminology since
     # we are compiling things down into a flattened graph), the meaning
     # of this link iteration via iter_links() is that u -> v (with the
     # provided dictionary attributes, if any).
     for (u, v, attr_dict) in flow.iter_links():
         if not priors:
             priors.append((None, u))
         v_g = decomposed_members[v]
         if not v_g.number_of_nodes():
             priors.append((u, v))
             continue
         invariant = any(attr_dict.get(k) for k in _EDGE_INVARIANTS)
         if not invariant:
             # This is a symbol *only* dependency, connect
             # corresponding providers and consumers to allow the consumer
             # to be executed immediately after the provider finishes (this
             # is an optimization for these types of dependencies...)
             u_g = decomposed_members[u]
             if not u_g.number_of_nodes():
                 # This must always exist, but incase it somehow doesn't...
                 raise exc.CompilationFailure(
                     "Non-invariant link being created from '%s' ->"
                     " '%s' even though the target '%s' was found to be"
                     " decomposed into an empty graph" % (v, u, u))
             for u in u_g.nodes_iter():
                 for v in v_g.nodes_iter():
                     depends_on = u.provides & v.requires
                     if depends_on:
                         _add_update_edges(graph,
                                           [u], [v],
                                           attr_dict={
                                               _EDGE_REASONS: depends_on,
                                           })
         else:
             # Connect nodes with no predecessors in v to nodes with no
             # successors in the *first* non-empty predecessor of v (thus
             # maintaining the edge dependency).
             match = self._find_first_decomposed(u, priors,
                                                 decomposed_members,
                                                 self._is_not_empty)
             if match is not None:
                 _add_update_edges(graph,
                                   match.no_successors_iter(),
                                   list(v_g.no_predecessors_iter()),
                                   attr_dict=attr_dict)
         priors.append((u, v))
コード例 #3
0
 def apply_constraints(self, graph, flow, decomposed_members):
     # This list is used to track the links that have been previously
     # iterated over, so that when we are trying to find a entry to
     # connect to that we iterate backwards through this list, finding
     # connected nodes to the current target (lets call it v) and find
     # the first (u_n, or u_n - 1, u_n - 2...) that was decomposed into
     # a non-empty graph. We also retain all predecessors of v so that we
     # can correctly locate u_n - 1 if u_n turns out to have decomposed into
     # an empty graph (and so on).
     priors = []
     # NOTE(harlowja): u, v are flows/tasks (also graph terminology since
     # we are compiling things down into a flattened graph), the meaning
     # of this link iteration via iter_links() is that u -> v (with the
     # provided dictionary attributes, if any).
     for (u, v, attr_dict) in flow.iter_links():
         if not priors:
             priors.append((None, u))
         v_g = decomposed_members[v]
         if not v_g.number_of_nodes():
             priors.append((u, v))
             continue
         invariant = any(attr_dict.get(k) for k in _EDGE_INVARIANTS)
         if not invariant:
             # This is a symbol *only* dependency, connect
             # corresponding providers and consumers to allow the consumer
             # to be executed immediately after the provider finishes (this
             # is an optimization for these types of dependencies...)
             u_g = decomposed_members[u]
             if not u_g.number_of_nodes():
                 # This must always exist, but incase it somehow doesn't...
                 raise exc.CompilationFailure(
                     "Non-invariant link being created from '%s' ->"
                     " '%s' even though the target '%s' was found to be"
                     " decomposed into an empty graph" % (v, u, u))
             for u in u_g.nodes_iter():
                 for v in v_g.nodes_iter():
                     depends_on = u.provides & v.requires
                     if depends_on:
                         _add_update_edges(graph, [u], [v],
                                           attr_dict={
                                               _EDGE_REASONS: depends_on,
                                           })
         else:
             # Connect nodes with no predecessors in v to nodes with no
             # successors in the *first* non-empty predecessor of v (thus
             # maintaining the edge dependency).
             match = self._find_first_decomposed(u, priors,
                                                 decomposed_members,
                                                 self._is_not_empty)
             if match is not None:
                 _add_update_edges(graph,
                                   match.no_successors_iter(),
                                   list(v_g.no_predecessors_iter()),
                                   attr_dict=attr_dict)
         priors.append((u, v))
コード例 #4
0
 def compile(self, flow, parent=None):
     """Decomposes a flow into a graph and scope tree hierarchy."""
     graph = gr.DiGraph(name=flow.name)
     graph.add_node(flow, kind=FLOW, noop=True)
     tree_node = tr.Node(flow, kind=FLOW, noop=True)
     if parent is not None:
         parent.add(tree_node)
     if flow.retry is not None:
         tree_node.add(tr.Node(flow.retry, kind=RETRY))
     decomposed = dict(
         (child, self._deep_compiler_func(child, parent=tree_node)[0])
         for child in flow)
     decomposed_graphs = list(six.itervalues(decomposed))
     graph = gr.merge_graphs(graph,
                             *decomposed_graphs,
                             overlap_detector=_overlap_occurence_detector)
     for u, v, attr_dict in flow.iter_links():
         u_graph = decomposed[u]
         v_graph = decomposed[v]
         _add_update_edges(graph,
                           u_graph.no_successors_iter(),
                           list(v_graph.no_predecessors_iter()),
                           attr_dict=attr_dict)
     if flow.retry is not None:
         graph.add_node(flow.retry, kind=RETRY)
         _add_update_edges(graph, [flow], [flow.retry],
                           attr_dict={LINK_INVARIANT: True})
         for node in graph.nodes_iter():
             if node is not flow.retry and node is not flow:
                 graph.node[node].setdefault(RETRY, flow.retry)
         from_nodes = [flow.retry]
         connected_attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
     else:
         from_nodes = [flow]
         connected_attr_dict = {LINK_INVARIANT: True}
     connected_to = [
         node for node in graph.no_predecessors_iter() if node is not flow
     ]
     if connected_to:
         # Ensure all nodes in this graph(s) that have no
         # predecessors depend on this flow (or this flow's retry) so that
         # we can depend on the flow being traversed before its
         # children (even though at the current time it will be skipped).
         _add_update_edges(graph,
                           from_nodes,
                           connected_to,
                           attr_dict=connected_attr_dict)
     return graph, tree_node
コード例 #5
0
ファイル: compiler.py プロジェクト: TheSriram/taskflow
    def _flatten_flow(self, flow, parent):
        """Flattens a flow."""
        graph = gr.DiGraph(name=flow.name)
        node = tr.Node(flow)
        if parent is not None:
            parent.add(node)
        if flow.retry is not None:
            node.add(tr.Node(flow.retry))

        # Flatten all nodes into a single subgraph per item (and track origin
        # item to its newly expanded graph).
        subgraphs = {}
        for item in flow:
            subgraph = self._flatten(item, node)[0]
            subgraphs[item] = subgraph
            graph = gr.merge_graphs([graph, subgraph])

        # Reconnect all items edges to their corresponding subgraphs.
        for (u, v, attrs) in flow.iter_links():
            u_g = subgraphs[u]
            v_g = subgraphs[v]
            if any(attrs.get(k) for k in _EDGE_INVARIANTS):
                # Connect nodes with no predecessors in v to nodes with
                # no successors in u (thus maintaining the edge dependency).
                self._add_new_edges(graph,
                                    u_g.no_successors_iter(),
                                    v_g.no_predecessors_iter(),
                                    edge_attrs=attrs)
            else:
                # This is symbol dependency edge, connect corresponding
                # providers and consumers.
                for provider in u_g:
                    for consumer in v_g:
                        reasons = provider.provides & consumer.requires
                        if reasons:
                            graph.add_edge(provider, consumer, reasons=reasons)

        if flow.retry is not None:
            self._connect_retry(flow.retry, graph)
        return graph, node
コード例 #6
0
ファイル: compiler.py プロジェクト: ygkim93/OpenStackInAction
    def _flatten_flow(self, flow, parent):
        """Flattens a flow."""
        graph = gr.DiGraph(name=flow.name)
        node = tr.Node(flow)
        if parent is not None:
            parent.add(node)
        if flow.retry is not None:
            node.add(tr.Node(flow.retry))

        # Flatten all nodes into a single subgraph per item (and track origin
        # item to its newly expanded graph).
        subgraphs = {}
        for item in flow:
            subgraph = self._flatten(item, node)[0]
            subgraphs[item] = subgraph
            graph = gr.merge_graphs([graph, subgraph])

        # Reconnect all items edges to their corresponding subgraphs.
        for (u, v, attrs) in flow.iter_links():
            u_g = subgraphs[u]
            v_g = subgraphs[v]
            if any(attrs.get(k) for k in ('invariant', 'manual', 'retry')):
                # Connect nodes with no predecessors in v to nodes with
                # no successors in u (thus maintaining the edge dependency).
                self._add_new_edges(graph,
                                    u_g.no_successors_iter(),
                                    v_g.no_predecessors_iter(),
                                    edge_attrs=attrs)
            else:
                # This is dependency-only edge, connect corresponding
                # providers and consumers.
                for provider in u_g:
                    for consumer in v_g:
                        reasons = provider.provides & consumer.requires
                        if reasons:
                            graph.add_edge(provider, consumer, reasons=reasons)

        if flow.retry is not None:
            self._connect_retry(flow.retry, graph)
        return graph, node
コード例 #7
0
 def compile(self, flow, parent=None):
     """Decomposes a flow into a graph and scope tree hierarchy."""
     graph = gr.DiGraph(name=flow.name)
     graph.add_node(flow, kind=FLOW, noop=True)
     tree_node = tr.Node(flow, kind=FLOW, noop=True)
     if parent is not None:
         parent.add(tree_node)
     if flow.retry is not None:
         tree_node.add(tr.Node(flow.retry, kind=RETRY))
     decomposed = dict(
         (child, self._deep_compiler_func(child, parent=tree_node)[0])
         for child in flow)
     decomposed_graphs = list(six.itervalues(decomposed))
     graph = gr.merge_graphs(graph, *decomposed_graphs,
                             overlap_detector=_overlap_occurence_detector)
     for u, v, attr_dict in flow.iter_links():
         u_graph = decomposed[u]
         v_graph = decomposed[v]
         _add_update_edges(graph, u_graph.no_successors_iter(),
                           list(v_graph.no_predecessors_iter()),
                           attr_dict=attr_dict)
     # Insert the flow(s) retry if needed, and always make sure it
     # is the **immediate** successor of the flow node itself.
     if flow.retry is not None:
         graph.add_node(flow.retry, kind=RETRY)
         _add_update_edges(graph, [flow], [flow.retry],
                           attr_dict={LINK_INVARIANT: True})
         for node in graph.nodes_iter():
             if node is not flow.retry and node is not flow:
                 graph.node[node].setdefault(RETRY, flow.retry)
         from_nodes = [flow.retry]
         attr_dict = {LINK_INVARIANT: True, LINK_RETRY: True}
     else:
         from_nodes = [flow]
         attr_dict = {LINK_INVARIANT: True}
     # Ensure all nodes with no predecessors are connected to this flow
     # or its retry node (so that the invariant that the flow node is
     # traversed through before its contents is maintained); this allows
     # us to easily know when we have entered a flow (when running) and
     # do special and/or smart things such as only traverse up to the
     # start of a flow when looking for node deciders.
     _add_update_edges(graph, from_nodes, [
         node for node in graph.no_predecessors_iter()
         if node is not flow
     ], attr_dict=attr_dict)
     # Connect all nodes with no successors into a special terminator
     # that is used to identify the end of the flow and ensure that all
     # execution traversals will traverse over this node before executing
     # further work (this is especially useful for nesting and knowing
     # when we have exited a nesting level); it allows us to do special
     # and/or smart things such as applying deciders up to (but not
     # beyond) a flow termination point.
     #
     # Do note that in a empty flow this will just connect itself to
     # the flow node itself... and also note we can not use the flow
     # object itself (primarily because the underlying graph library
     # uses hashing to identify node uniqueness and we can easily create
     # a loop if we don't do this correctly, so avoid that by just
     # creating this special node and tagging it with a special kind); we
     # may be able to make this better in the future with a multidigraph
     # that networkx provides??
     flow_term = Terminator(flow)
     graph.add_node(flow_term, kind=FLOW_END, noop=True)
     _add_update_edges(graph, [
         node for node in graph.no_successors_iter()
         if node is not flow_term
     ], [flow_term], attr_dict={LINK_INVARIANT: True})
     return graph, tree_node