def apply_constraints(self, graph, flow, decomposed_members): # This list is used to track the links that have been previously # iterated over, so that when we are trying to find a entry to # connect to that we iterate backwards through this list, finding # connected nodes to the current target (lets call it v) and find # the first (u_n, or u_n - 1, u_n - 2...) that was decomposed into # a non-empty graph. We also retain all predecessors of v so that we # can correctly locate u_n - 1 if u_n turns out to have decomposed into # an empty graph (and so on). priors = [] # NOTE(harlowja): u, v are flows/tasks (also graph terminology since # we are compiling things down into a flattened graph), the meaning # of this link iteration via iter_links() is that u -> v (with the # provided dictionary attributes, if any). for (u, v, attr_dict) in flow.iter_links(): if not priors: priors.append((None, u)) v_g = decomposed_members[v] if not v_g.number_of_nodes(): priors.append((u, v)) continue invariant = any(attr_dict.get(k) for k in _EDGE_INVARIANTS) if not invariant: # This is a symbol *only* dependency, connect # corresponding providers and consumers to allow the consumer # to be executed immediately after the provider finishes (this # is an optimization for these types of dependencies...) u_g = decomposed_members[u] if not u_g.number_of_nodes(): # This must always exist, but incase it somehow doesn't... raise exc.CompilationFailure( "Non-invariant link being created from '%s' ->" " '%s' even though the target '%s' was found to be" " decomposed into an empty graph" % (v, u, u)) for u in u_g.nodes_iter(): for v in v_g.nodes_iter(): depends_on = u.provides & v.requires if depends_on: _add_update_edges(graph, [u], [v], attr_dict={ _EDGE_REASONS: depends_on, }) else: # Connect nodes with no predecessors in v to nodes with no # successors in the *first* non-empty predecessor of v (thus # maintaining the edge dependency). match = self._find_first_decomposed(u, priors, decomposed_members, self._is_not_empty) if match is not None: _add_update_edges(graph, match.no_successors_iter(), list(v_g.no_predecessors_iter()), attr_dict=attr_dict) priors.append((u, v))
def compile(self): """Compiles & caches frequently used execution helper objects. Build out a cache of commonly used item that are associated with the contained atoms (by name), and are useful to have for quick lookup on (for example, the change state handler function for each atom, the scope walker object for each atom, the task or retry specific scheduler and so-on). """ change_state_handlers = { com.TASK: functools.partial(self.task_action.change_state, progress=0.0), com.RETRY: self.retry_action.change_state, } schedulers = { com.RETRY: self.retry_scheduler, com.TASK: self.task_scheduler, } check_transition_handlers = { com.TASK: st.check_task_transition, com.RETRY: st.check_retry_transition, } actions = { com.TASK: self.task_action, com.RETRY: self.retry_action, } graph = self._compilation.execution_graph for node, node_data in graph.nodes_iter(data=True): node_kind = node_data['kind'] if node_kind in com.FLOWS: continue elif node_kind in com.ATOMS: check_transition_handler = check_transition_handlers[node_kind] change_state_handler = change_state_handlers[node_kind] scheduler = schedulers[node_kind] action = actions[node_kind] else: raise exc.CompilationFailure("Unknown node kind '%s'" " encountered" % node_kind) metadata = {} deciders_it = self._walk_edge_deciders(graph, node) walker = sc.ScopeWalker(self.compilation, node, names_only=True) metadata['scope_walker'] = walker metadata['check_transition_handler'] = check_transition_handler metadata['change_state_handler'] = change_state_handler metadata['scheduler'] = scheduler metadata['edge_deciders'] = tuple(deciders_it) metadata['action'] = action LOG.trace("Compiled %s metadata for node %s (%s)", metadata, node.name, node_kind) self._atom_cache[node.name] = metadata
def compile(self): """Compiles & caches frequently used execution helper objects. Build out a cache of commonly used item that are associated with the contained atoms (by name), and are useful to have for quick lookup on (for example, the change state handler function for each atom, the scope walker object for each atom, the task or retry specific scheduler and so-on). """ change_state_handlers = { com.TASK: functools.partial(self.task_action.change_state, progress=0.0), com.RETRY: self.retry_action.change_state, } schedulers = { com.RETRY: self.retry_scheduler, com.TASK: self.task_scheduler, } check_transition_handlers = { com.TASK: st.check_task_transition, com.RETRY: st.check_retry_transition, } graph = self._compilation.execution_graph for node, node_data in graph.nodes_iter(data=True): node_kind = node_data['kind'] if node_kind == com.FLOW: continue elif node_kind in com.ATOMS: check_transition_handler = check_transition_handlers[node_kind] change_state_handler = change_state_handlers[node_kind] scheduler = schedulers[node_kind] else: raise exc.CompilationFailure("Unknown node kind '%s'" " encountered" % node_kind) metadata = {} walker = sc.ScopeWalker(self.compilation, node, names_only=True) edge_deciders = {} for prev_node in graph.predecessors_iter(node): # If there is any link function that says if this connection # is able to run (or should not) ensure we retain it and use # it later as needed. u_v_data = graph.adj[prev_node][node] u_v_decider = u_v_data.get(LINK_DECIDER) if u_v_decider is not None: edge_deciders[prev_node.name] = u_v_decider metadata['scope_walker'] = walker metadata['check_transition_handler'] = check_transition_handler metadata['change_state_handler'] = change_state_handler metadata['scheduler'] = scheduler metadata['edge_deciders'] = edge_deciders self._atom_cache[node.name] = metadata