def _fetch_scopes_for(self, atom): """Fetches a tuple of the visible scopes for the given atom.""" try: return self._scopes[atom] except KeyError: walker = sc.ScopeWalker(self.compilation, atom, names_only=True) visible_to = tuple(walker) self._scopes[atom] = visible_to return visible_to
def test_empty(self): r = lf.Flow("root") r_1 = test_utils.TaskOneReturn("root.1") r.add(r_1) c = compiler.PatternCompiler(r).compile() self.assertIn(r_1, c.execution_graph) self.assertIsNotNone(c.hierarchy.find(r_1)) walker = sc.ScopeWalker(c, r_1) scopes = list(walker) self.assertEqual([], scopes)
def compile(self): """Compiles & caches frequently used execution helper objects. Build out a cache of commonly used item that are associated with the contained atoms (by name), and are useful to have for quick lookup on (for example, the change state handler function for each atom, the scope walker object for each atom, the task or retry specific scheduler and so-on). """ change_state_handlers = { com.TASK: functools.partial(self.task_action.change_state, progress=0.0), com.RETRY: self.retry_action.change_state, } schedulers = { com.RETRY: self.retry_scheduler, com.TASK: self.task_scheduler, } check_transition_handlers = { com.TASK: st.check_task_transition, com.RETRY: st.check_retry_transition, } actions = { com.TASK: self.task_action, com.RETRY: self.retry_action, } graph = self._compilation.execution_graph for node, node_data in graph.nodes_iter(data=True): node_kind = node_data['kind'] if node_kind in com.FLOWS: continue elif node_kind in com.ATOMS: check_transition_handler = check_transition_handlers[node_kind] change_state_handler = change_state_handlers[node_kind] scheduler = schedulers[node_kind] action = actions[node_kind] else: raise exc.CompilationFailure("Unknown node kind '%s'" " encountered" % node_kind) metadata = {} deciders_it = self._walk_edge_deciders(graph, node) walker = sc.ScopeWalker(self.compilation, node, names_only=True) metadata['scope_walker'] = walker metadata['check_transition_handler'] = check_transition_handler metadata['change_state_handler'] = change_state_handler metadata['scheduler'] = scheduler metadata['edge_deciders'] = tuple(deciders_it) metadata['action'] = action LOG.trace("Compiled %s metadata for node %s (%s)", metadata, node.name, node_kind) self._atom_cache[node.name] = metadata
def compile(self): """Compiles & caches frequently used execution helper objects. Build out a cache of commonly used item that are associated with the contained atoms (by name), and are useful to have for quick lookup on (for example, the change state handler function for each atom, the scope walker object for each atom, the task or retry specific scheduler and so-on). """ change_state_handlers = { com.TASK: functools.partial(self.task_action.change_state, progress=0.0), com.RETRY: self.retry_action.change_state, } schedulers = { com.RETRY: self.retry_scheduler, com.TASK: self.task_scheduler, } check_transition_handlers = { com.TASK: st.check_task_transition, com.RETRY: st.check_retry_transition, } graph = self._compilation.execution_graph for node, node_data in graph.nodes_iter(data=True): node_kind = node_data['kind'] if node_kind == com.FLOW: continue elif node_kind in com.ATOMS: check_transition_handler = check_transition_handlers[node_kind] change_state_handler = change_state_handlers[node_kind] scheduler = schedulers[node_kind] else: raise exc.CompilationFailure("Unknown node kind '%s'" " encountered" % node_kind) metadata = {} walker = sc.ScopeWalker(self.compilation, node, names_only=True) edge_deciders = {} for prev_node in graph.predecessors_iter(node): # If there is any link function that says if this connection # is able to run (or should not) ensure we retain it and use # it later as needed. u_v_data = graph.adj[prev_node][node] u_v_decider = u_v_data.get(LINK_DECIDER) if u_v_decider is not None: edge_deciders[prev_node.name] = u_v_decider metadata['scope_walker'] = walker metadata['check_transition_handler'] = check_transition_handler metadata['change_state_handler'] = change_state_handler metadata['scheduler'] = scheduler metadata['edge_deciders'] = edge_deciders self._atom_cache[node.name] = metadata
def compile(self): """Compiles & caches frequently used execution helper objects. Build out a cache of commonly used item that are associated with the contained atoms (by name), and are useful to have for quick lookup on (for example, the change state handler function for each atom, the scope walker object for each atom, the task or retry specific scheduler and so-on). """ change_state_handlers = { 'task': functools.partial(self.task_action.change_state, progress=0.0), 'retry': self.retry_action.change_state, } schedulers = { 'retry': self.retry_scheduler, 'task': self.task_scheduler, } execution_graph = self._compilation.execution_graph all_retry_atoms = [] all_task_atoms = [] for atom in self.analyzer.iterate_all_nodes(): metadata = {} walker = sc.ScopeWalker(self.compilation, atom, names_only=True) if isinstance(atom, task.BaseTask): check_transition_handler = st.check_task_transition change_state_handler = change_state_handlers['task'] scheduler = schedulers['task'] all_task_atoms.append(atom) else: check_transition_handler = st.check_retry_transition change_state_handler = change_state_handlers['retry'] scheduler = schedulers['retry'] all_retry_atoms.append(atom) edge_deciders = {} for previous_atom in execution_graph.predecessors(atom): # If there is any link function that says if this connection # is able to run (or should not) ensure we retain it and use # it later as needed. u_v_data = execution_graph.adj[previous_atom][atom] u_v_decider = u_v_data.get(flow.LINK_DECIDER) if u_v_decider is not None: edge_deciders[previous_atom.name] = u_v_decider metadata['scope_walker'] = walker metadata['check_transition_handler'] = check_transition_handler metadata['change_state_handler'] = change_state_handler metadata['scheduler'] = scheduler metadata['edge_deciders'] = edge_deciders self._atom_cache[atom.name] = metadata self._atoms_by_kind['retry'] = all_retry_atoms self._atoms_by_kind['task'] = all_task_atoms
def fetch_scopes_for(self, atom_name): """Fetches a walker of the visible scopes for the given atom.""" try: return self._walkers_to_names[atom_name] except KeyError: atom = None for node in self.analyzer.iterate_all_nodes(): if node.name == atom_name: atom = node break if atom is not None: walker = sc.ScopeWalker(self.compilation, atom, names_only=True) self._walkers_to_names[atom_name] = walker else: walker = None return walker
def task_action(self): return ta.TaskAction( self._storage, self._task_executor, self._task_notifier, lambda atom: sc.ScopeWalker(self.compilation, atom, names_only=True))
def retry_action(self): return ra.RetryAction( self._storage, self._task_notifier, lambda atom: sc.ScopeWalker( self.compilation, atom, names_only=True))
def _get_scopes(compilation, atom, names_only=True): walker = sc.ScopeWalker(compilation, atom, names_only=names_only) return list(iter(walker))