Пример #1
0
 def apply_constraints(self, graph, flow, decomposed_members):
     # This list is used to track the links that have been previously
     # iterated over, so that when we are trying to find a entry to
     # connect to that we iterate backwards through this list, finding
     # connected nodes to the current target (lets call it v) and find
     # the first (u_n, or u_n - 1, u_n - 2...) that was decomposed into
     # a non-empty graph. We also retain all predecessors of v so that we
     # can correctly locate u_n - 1 if u_n turns out to have decomposed into
     # an empty graph (and so on).
     priors = []
     # NOTE(harlowja): u, v are flows/tasks (also graph terminology since
     # we are compiling things down into a flattened graph), the meaning
     # of this link iteration via iter_links() is that u -> v (with the
     # provided dictionary attributes, if any).
     for (u, v, attr_dict) in flow.iter_links():
         if not priors:
             priors.append((None, u))
         v_g = decomposed_members[v]
         if not v_g.number_of_nodes():
             priors.append((u, v))
             continue
         invariant = any(attr_dict.get(k) for k in _EDGE_INVARIANTS)
         if not invariant:
             # This is a symbol *only* dependency, connect
             # corresponding providers and consumers to allow the consumer
             # to be executed immediately after the provider finishes (this
             # is an optimization for these types of dependencies...)
             u_g = decomposed_members[u]
             if not u_g.number_of_nodes():
                 # This must always exist, but incase it somehow doesn't...
                 raise exc.CompilationFailure(
                     "Non-invariant link being created from '%s' ->"
                     " '%s' even though the target '%s' was found to be"
                     " decomposed into an empty graph" % (v, u, u))
             for u in u_g.nodes_iter():
                 for v in v_g.nodes_iter():
                     depends_on = u.provides & v.requires
                     if depends_on:
                         _add_update_edges(graph, [u], [v],
                                           attr_dict={
                                               _EDGE_REASONS: depends_on,
                                           })
         else:
             # Connect nodes with no predecessors in v to nodes with no
             # successors in the *first* non-empty predecessor of v (thus
             # maintaining the edge dependency).
             match = self._find_first_decomposed(u, priors,
                                                 decomposed_members,
                                                 self._is_not_empty)
             if match is not None:
                 _add_update_edges(graph,
                                   match.no_successors_iter(),
                                   list(v_g.no_predecessors_iter()),
                                   attr_dict=attr_dict)
         priors.append((u, v))
Пример #2
0
    def compile(self):
        """Compiles & caches frequently used execution helper objects.

        Build out a cache of commonly used item that are associated
        with the contained atoms (by name), and are useful to have for
        quick lookup on (for example, the change state handler function for
        each atom, the scope walker object for each atom, the task or retry
        specific scheduler and so-on).
        """
        change_state_handlers = {
            com.TASK:
            functools.partial(self.task_action.change_state, progress=0.0),
            com.RETRY:
            self.retry_action.change_state,
        }
        schedulers = {
            com.RETRY: self.retry_scheduler,
            com.TASK: self.task_scheduler,
        }
        check_transition_handlers = {
            com.TASK: st.check_task_transition,
            com.RETRY: st.check_retry_transition,
        }
        actions = {
            com.TASK: self.task_action,
            com.RETRY: self.retry_action,
        }
        graph = self._compilation.execution_graph
        for node, node_data in graph.nodes_iter(data=True):
            node_kind = node_data['kind']
            if node_kind in com.FLOWS:
                continue
            elif node_kind in com.ATOMS:
                check_transition_handler = check_transition_handlers[node_kind]
                change_state_handler = change_state_handlers[node_kind]
                scheduler = schedulers[node_kind]
                action = actions[node_kind]
            else:
                raise exc.CompilationFailure("Unknown node kind '%s'"
                                             " encountered" % node_kind)
            metadata = {}
            deciders_it = self._walk_edge_deciders(graph, node)
            walker = sc.ScopeWalker(self.compilation, node, names_only=True)
            metadata['scope_walker'] = walker
            metadata['check_transition_handler'] = check_transition_handler
            metadata['change_state_handler'] = change_state_handler
            metadata['scheduler'] = scheduler
            metadata['edge_deciders'] = tuple(deciders_it)
            metadata['action'] = action
            LOG.trace("Compiled %s metadata for node %s (%s)", metadata,
                      node.name, node_kind)
            self._atom_cache[node.name] = metadata
Пример #3
0
    def compile(self):
        """Compiles & caches frequently used execution helper objects.

        Build out a cache of commonly used item that are associated
        with the contained atoms (by name), and are useful to have for
        quick lookup on (for example, the change state handler function for
        each atom, the scope walker object for each atom, the task or retry
        specific scheduler and so-on).
        """
        change_state_handlers = {
            com.TASK:
            functools.partial(self.task_action.change_state, progress=0.0),
            com.RETRY:
            self.retry_action.change_state,
        }
        schedulers = {
            com.RETRY: self.retry_scheduler,
            com.TASK: self.task_scheduler,
        }
        check_transition_handlers = {
            com.TASK: st.check_task_transition,
            com.RETRY: st.check_retry_transition,
        }
        graph = self._compilation.execution_graph
        for node, node_data in graph.nodes_iter(data=True):
            node_kind = node_data['kind']
            if node_kind == com.FLOW:
                continue
            elif node_kind in com.ATOMS:
                check_transition_handler = check_transition_handlers[node_kind]
                change_state_handler = change_state_handlers[node_kind]
                scheduler = schedulers[node_kind]
            else:
                raise exc.CompilationFailure("Unknown node kind '%s'"
                                             " encountered" % node_kind)
            metadata = {}
            walker = sc.ScopeWalker(self.compilation, node, names_only=True)
            edge_deciders = {}
            for prev_node in graph.predecessors_iter(node):
                # If there is any link function that says if this connection
                # is able to run (or should not) ensure we retain it and use
                # it later as needed.
                u_v_data = graph.adj[prev_node][node]
                u_v_decider = u_v_data.get(LINK_DECIDER)
                if u_v_decider is not None:
                    edge_deciders[prev_node.name] = u_v_decider
            metadata['scope_walker'] = walker
            metadata['check_transition_handler'] = check_transition_handler
            metadata['change_state_handler'] = change_state_handler
            metadata['scheduler'] = scheduler
            metadata['edge_deciders'] = edge_deciders
            self._atom_cache[node.name] = metadata