def validate(self):
# At this point we can check to ensure all dependencies are either
# flow/task provided or storage provided, if there are still missing
# dependencies then this flow will fail at runtime (which we can avoid
# by failing at validation time).
if LOG.isEnabledFor(logging.TRACE):
execution_graph = self._compilation.execution_graph
LOG.trace(
"Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f", execution_graph.number_of_nodes(),
execution_graph.number_of_edges(), nx.density(execution_graph))
missing = set()
# Attempt to retain a chain of what was missing (so that the final
# raised exception for the flow has the nodes that had missing
# dependencies).
last_cause = None
last_node = None
missing_nodes = 0
for atom in self._runtime.iterate_nodes(compiler.ATOMS):
exec_missing = self.storage.fetch_unsatisfied_args(
atom.name, atom.rebind, optional_args=atom.optional)
revert_missing = self.storage.fetch_unsatisfied_args(
atom.name,
atom.revert_rebind,
optional_args=atom.revert_optional)
atom_missing = (('execute', exec_missing), ('revert',
revert_missing))
for method, method_missing in atom_missing:
if method_missing:
cause = exc.MissingDependencies(atom,
sorted(method_missing),
cause=last_cause,
method=method)
last_cause = cause
last_node = atom
missing_nodes += 1
missing.update(method_missing)
if missing:
# For when a task is provided (instead of a flow) and that
# task is the only item in the graph and its missing deps, avoid
# re-wrapping it in yet another exception...
if missing_nodes == 1 and last_node is self._flow:
raise last_cause
else:
raise exc.MissingDependencies(self._flow,
sorted(missing),
cause=last_cause)
self._validated = True
def validate(self):
self._check('validate', True, True)
# At this point we can check to ensure all dependencies are either
# flow/task provided or storage provided, if there are still missing
# dependencies then this flow will fail at runtime (which we can avoid
# by failing at validation time).
execution_graph = self._compilation.execution_graph
if LOG.isEnabledFor(logging.BLATHER):
LOG.blather(
"Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f", execution_graph.number_of_nodes(),
execution_graph.number_of_edges(), nx.density(execution_graph))
missing = set()
# Attempt to retain a chain of what was missing (so that the final
# raised exception for the flow has the nodes that had missing
# dependencies).
last_cause = None
last_node = None
missing_nodes = 0
fetch_func = self.storage.fetch_unsatisfied_args
for node in execution_graph.nodes_iter():
node_missing = fetch_func(node.name,
node.rebind,
optional_args=node.optional)
if node_missing:
cause = exc.MissingDependencies(node,
sorted(node_missing),
cause=last_cause)
last_cause = cause
last_node = node
missing_nodes += 1
missing.update(node_missing)
if missing:
# For when a task is provided (instead of a flow) and that
# task is the only item in the graph and its missing deps, avoid
# re-wrapping it in yet another exception...
if missing_nodes == 1 and last_node is self._flow:
raise last_cause
else:
raise exc.MissingDependencies(self._flow,
sorted(missing),
cause=last_cause)
def connect(graph, infer_key='infer', auto_reason='auto', discard_func=None):
"""Connects a graphs runners to other runners in the graph which provide
outputs for each runners requirements.
"""
if len(graph) == 0:
return
if discard_func:
for (u, v, e_data) in graph.edges(data=True):
if discard_func(u, v, e_data):
graph.remove_edge(u, v)
for (r, r_data) in graph.nodes_iter(data=True):
requires = set(r.requires)
# Find the ones that have already been attached manually.
manual_providers = {}
if requires:
incoming = [e[0] for e in graph.in_edges_iter([r])]
for r2 in incoming:
fulfills = requires & r2.provides
if fulfills:
LOG.debug("%s is a manual provider of %s for %s", r2,
fulfills, r)
for k in fulfills:
manual_providers[k] = r2
requires.remove(k)
# Anything leftover that we must find providers for??
auto_providers = {}
if requires and r_data.get(infer_key):
for r2 in graph.nodes_iter():
if r is r2:
continue
fulfills = requires & r2.provides
if fulfills:
graph.add_edge(r2, r, reason=auto_reason)
LOG.debug(
"Connecting %s as a automatic provider for"
" %s for %s", r2, fulfills, r)
for k in fulfills:
auto_providers[k] = r2
requires.remove(k)
if not requires:
break
# Anything still leftover??
if requires:
# Ensure its in string format, since join will puke on
# things that are not strings.
missing = ", ".join(sorted([str(s) for s in requires]))
raise exc.MissingDependencies(r, missing)
else:
r.providers = {}
r.providers.update(auto_providers)
r.providers.update(manual_providers)
def run(self):
"""Runs the flow in the engine to completion."""
if self.storage.get_flow_state() == states.REVERTED:
self._reset()
self.compile()
external_provides = set(self.storage.fetch_all().keys())
missing = self._flow.requires - external_provides
if missing:
raise exc.MissingDependencies(self._flow, sorted(missing))
if self._failures:
self._revert()
else:
self._run()
def _associate_providers(self, runner):
# Ensure that some previous task provides this input.
who_provides = {}
task_requires = runner.requires
for r in task_requires:
provider = None
for before_me in runner.runs_before:
if r in before_me.provides:
provider = before_me
break
if provider:
who_provides[r] = provider
# Ensure that the last task provides all the needed input for this
# task to run correctly.
missing_requires = task_requires - set(who_provides.keys())
if missing_requires:
raise exc.MissingDependencies(runner, sorted(missing_requires))
runner.providers.update(who_provides)
def prepare(self):
if not self._compiled:
raise exc.InvalidState("Can not prepare an engine"
" which has not been compiled")
if not self._storage_ensured:
self._ensure_storage()
self._storage_ensured = True
# At this point we can check to ensure all dependencies are either
# flow/task provided or storage provided, if there are still missing
# dependencies then this flow will fail at runtime (which we can avoid
# by failing at preparation time).
external_provides = set(self.storage.fetch_all().keys())
missing = self._flow.requires - external_provides
if missing:
raise exc.MissingDependencies(self._flow, sorted(missing))
# Reset everything back to pending (if we were previously reverted).
if self.storage.get_flow_state() == states.REVERTED:
self._runtime.reset_all()
self._change_state(states.PENDING)
def validate(self):
if not self._storage_ensured:
raise exc.InvalidState("Can not validate an engine"
" which has not has its storage"
" populated")
# At this point we can check to ensure all dependencies are either
# flow/task provided or storage provided, if there are still missing
# dependencies then this flow will fail at runtime (which we can avoid
# by failing at validation time).
execution_graph = self._compilation.execution_graph
if LOG.isEnabledFor(logging.BLATHER):
LOG.blather(
"Validating scoping and argument visibility for"
" execution graph with %s nodes and %s edges with"
" density %0.3f", execution_graph.number_of_nodes(),
execution_graph.number_of_edges(), nx.density(execution_graph))
missing = set()
fetch = self.storage.fetch_unsatisfied_args
for node in execution_graph.nodes_iter():
missing.update(
fetch(node.name, node.rebind, optional_args=node.optional))
if missing:
raise exc.MissingDependencies(self._flow, sorted(missing))