def _fetch_scopes_for(self, atom):
"""Fetches a tuple of the visible scopes for the given atom."""
try:
return self._scopes[atom]
except KeyError:
walker = sc.ScopeWalker(self.compilation, atom,
names_only=True)
visible_to = tuple(walker)
self._scopes[atom] = visible_to
return visible_to
def test_empty(self):
r = lf.Flow("root")
r_1 = test_utils.TaskOneReturn("root.1")
r.add(r_1)
c = compiler.PatternCompiler(r).compile()
self.assertIn(r_1, c.execution_graph)
self.assertIsNotNone(c.hierarchy.find(r_1))
walker = sc.ScopeWalker(c, r_1)
scopes = list(walker)
self.assertEqual([], scopes)
def compile(self):
"""Compiles & caches frequently used execution helper objects.
Build out a cache of commonly used item that are associated
with the contained atoms (by name), and are useful to have for
quick lookup on (for example, the change state handler function for
each atom, the scope walker object for each atom, the task or retry
specific scheduler and so-on).
"""
change_state_handlers = {
com.TASK:
functools.partial(self.task_action.change_state, progress=0.0),
com.RETRY:
self.retry_action.change_state,
}
schedulers = {
com.RETRY: self.retry_scheduler,
com.TASK: self.task_scheduler,
}
check_transition_handlers = {
com.TASK: st.check_task_transition,
com.RETRY: st.check_retry_transition,
}
actions = {
com.TASK: self.task_action,
com.RETRY: self.retry_action,
}
graph = self._compilation.execution_graph
for node, node_data in graph.nodes_iter(data=True):
node_kind = node_data['kind']
if node_kind in com.FLOWS:
continue
elif node_kind in com.ATOMS:
check_transition_handler = check_transition_handlers[node_kind]
change_state_handler = change_state_handlers[node_kind]
scheduler = schedulers[node_kind]
action = actions[node_kind]
else:
raise exc.CompilationFailure("Unknown node kind '%s'"
" encountered" % node_kind)
metadata = {}
deciders_it = self._walk_edge_deciders(graph, node)
walker = sc.ScopeWalker(self.compilation, node, names_only=True)
metadata['scope_walker'] = walker
metadata['check_transition_handler'] = check_transition_handler
metadata['change_state_handler'] = change_state_handler
metadata['scheduler'] = scheduler
metadata['edge_deciders'] = tuple(deciders_it)
metadata['action'] = action
LOG.trace("Compiled %s metadata for node %s (%s)", metadata,
node.name, node_kind)
self._atom_cache[node.name] = metadata
def compile(self):
"""Compiles & caches frequently used execution helper objects.
Build out a cache of commonly used item that are associated
with the contained atoms (by name), and are useful to have for
quick lookup on (for example, the change state handler function for
each atom, the scope walker object for each atom, the task or retry
specific scheduler and so-on).
"""
change_state_handlers = {
com.TASK:
functools.partial(self.task_action.change_state, progress=0.0),
com.RETRY:
self.retry_action.change_state,
}
schedulers = {
com.RETRY: self.retry_scheduler,
com.TASK: self.task_scheduler,
}
check_transition_handlers = {
com.TASK: st.check_task_transition,
com.RETRY: st.check_retry_transition,
}
graph = self._compilation.execution_graph
for node, node_data in graph.nodes_iter(data=True):
node_kind = node_data['kind']
if node_kind == com.FLOW:
continue
elif node_kind in com.ATOMS:
check_transition_handler = check_transition_handlers[node_kind]
change_state_handler = change_state_handlers[node_kind]
scheduler = schedulers[node_kind]
else:
raise exc.CompilationFailure("Unknown node kind '%s'"
" encountered" % node_kind)
metadata = {}
walker = sc.ScopeWalker(self.compilation, node, names_only=True)
edge_deciders = {}
for prev_node in graph.predecessors_iter(node):
# If there is any link function that says if this connection
# is able to run (or should not) ensure we retain it and use
# it later as needed.
u_v_data = graph.adj[prev_node][node]
u_v_decider = u_v_data.get(LINK_DECIDER)
if u_v_decider is not None:
edge_deciders[prev_node.name] = u_v_decider
metadata['scope_walker'] = walker
metadata['check_transition_handler'] = check_transition_handler
metadata['change_state_handler'] = change_state_handler
metadata['scheduler'] = scheduler
metadata['edge_deciders'] = edge_deciders
self._atom_cache[node.name] = metadata
def compile(self):
"""Compiles & caches frequently used execution helper objects.
Build out a cache of commonly used item that are associated
with the contained atoms (by name), and are useful to have for
quick lookup on (for example, the change state handler function for
each atom, the scope walker object for each atom, the task or retry
specific scheduler and so-on).
"""
change_state_handlers = {
'task': functools.partial(self.task_action.change_state,
progress=0.0),
'retry': self.retry_action.change_state,
}
schedulers = {
'retry': self.retry_scheduler,
'task': self.task_scheduler,
}
execution_graph = self._compilation.execution_graph
all_retry_atoms = []
all_task_atoms = []
for atom in self.analyzer.iterate_all_nodes():
metadata = {}
walker = sc.ScopeWalker(self.compilation, atom, names_only=True)
if isinstance(atom, task.BaseTask):
check_transition_handler = st.check_task_transition
change_state_handler = change_state_handlers['task']
scheduler = schedulers['task']
all_task_atoms.append(atom)
else:
check_transition_handler = st.check_retry_transition
change_state_handler = change_state_handlers['retry']
scheduler = schedulers['retry']
all_retry_atoms.append(atom)
edge_deciders = {}
for previous_atom in execution_graph.predecessors(atom):
# If there is any link function that says if this connection
# is able to run (or should not) ensure we retain it and use
# it later as needed.
u_v_data = execution_graph.adj[previous_atom][atom]
u_v_decider = u_v_data.get(flow.LINK_DECIDER)
if u_v_decider is not None:
edge_deciders[previous_atom.name] = u_v_decider
metadata['scope_walker'] = walker
metadata['check_transition_handler'] = check_transition_handler
metadata['change_state_handler'] = change_state_handler
metadata['scheduler'] = scheduler
metadata['edge_deciders'] = edge_deciders
self._atom_cache[atom.name] = metadata
self._atoms_by_kind['retry'] = all_retry_atoms
self._atoms_by_kind['task'] = all_task_atoms
def fetch_scopes_for(self, atom_name):
"""Fetches a walker of the visible scopes for the given atom."""
try:
return self._walkers_to_names[atom_name]
except KeyError:
atom = None
for node in self.analyzer.iterate_all_nodes():
if node.name == atom_name:
atom = node
break
if atom is not None:
walker = sc.ScopeWalker(self.compilation,
atom,
names_only=True)
self._walkers_to_names[atom_name] = walker
else:
walker = None
return walker
def task_action(self):
return ta.TaskAction(
self._storage, self._task_executor, self._task_notifier, lambda
atom: sc.ScopeWalker(self.compilation, atom, names_only=True))
def retry_action(self):
return ra.RetryAction(
self._storage, self._task_notifier, lambda atom: sc.ScopeWalker(
self.compilation, atom, names_only=True))
def _get_scopes(compilation, atom, names_only=True):
walker = sc.ScopeWalker(compilation, atom, names_only=names_only)
return list(iter(walker))