def validate(self): # At this point we can check to ensure all dependencies are either # flow/task provided or storage provided, if there are still missing # dependencies then this flow will fail at runtime (which we can avoid # by failing at validation time). if LOG.isEnabledFor(logging.TRACE): execution_graph = self._compilation.execution_graph LOG.trace( "Validating scoping and argument visibility for" " execution graph with %s nodes and %s edges with" " density %0.3f", execution_graph.number_of_nodes(), execution_graph.number_of_edges(), nx.density(execution_graph)) missing = set() # Attempt to retain a chain of what was missing (so that the final # raised exception for the flow has the nodes that had missing # dependencies). last_cause = None last_node = None missing_nodes = 0 for atom in self._runtime.iterate_nodes(compiler.ATOMS): exec_missing = self.storage.fetch_unsatisfied_args( atom.name, atom.rebind, optional_args=atom.optional) revert_missing = self.storage.fetch_unsatisfied_args( atom.name, atom.revert_rebind, optional_args=atom.revert_optional) atom_missing = (('execute', exec_missing), ('revert', revert_missing)) for method, method_missing in atom_missing: if method_missing: cause = exc.MissingDependencies(atom, sorted(method_missing), cause=last_cause, method=method) last_cause = cause last_node = atom missing_nodes += 1 missing.update(method_missing) if missing: # For when a task is provided (instead of a flow) and that # task is the only item in the graph and its missing deps, avoid # re-wrapping it in yet another exception... if missing_nodes == 1 and last_node is self._flow: raise last_cause else: raise exc.MissingDependencies(self._flow, sorted(missing), cause=last_cause) self._validated = True
def validate(self): self._check('validate', True, True) # At this point we can check to ensure all dependencies are either # flow/task provided or storage provided, if there are still missing # dependencies then this flow will fail at runtime (which we can avoid # by failing at validation time). execution_graph = self._compilation.execution_graph if LOG.isEnabledFor(logging.BLATHER): LOG.blather( "Validating scoping and argument visibility for" " execution graph with %s nodes and %s edges with" " density %0.3f", execution_graph.number_of_nodes(), execution_graph.number_of_edges(), nx.density(execution_graph)) missing = set() # Attempt to retain a chain of what was missing (so that the final # raised exception for the flow has the nodes that had missing # dependencies). last_cause = None last_node = None missing_nodes = 0 fetch_func = self.storage.fetch_unsatisfied_args for node in execution_graph.nodes_iter(): node_missing = fetch_func(node.name, node.rebind, optional_args=node.optional) if node_missing: cause = exc.MissingDependencies(node, sorted(node_missing), cause=last_cause) last_cause = cause last_node = node missing_nodes += 1 missing.update(node_missing) if missing: # For when a task is provided (instead of a flow) and that # task is the only item in the graph and its missing deps, avoid # re-wrapping it in yet another exception... if missing_nodes == 1 and last_node is self._flow: raise last_cause else: raise exc.MissingDependencies(self._flow, sorted(missing), cause=last_cause)
def connect(graph, infer_key='infer', auto_reason='auto', discard_func=None): """Connects a graphs runners to other runners in the graph which provide outputs for each runners requirements. """ if len(graph) == 0: return if discard_func: for (u, v, e_data) in graph.edges(data=True): if discard_func(u, v, e_data): graph.remove_edge(u, v) for (r, r_data) in graph.nodes_iter(data=True): requires = set(r.requires) # Find the ones that have already been attached manually. manual_providers = {} if requires: incoming = [e[0] for e in graph.in_edges_iter([r])] for r2 in incoming: fulfills = requires & r2.provides if fulfills: LOG.debug("%s is a manual provider of %s for %s", r2, fulfills, r) for k in fulfills: manual_providers[k] = r2 requires.remove(k) # Anything leftover that we must find providers for?? auto_providers = {} if requires and r_data.get(infer_key): for r2 in graph.nodes_iter(): if r is r2: continue fulfills = requires & r2.provides if fulfills: graph.add_edge(r2, r, reason=auto_reason) LOG.debug( "Connecting %s as a automatic provider for" " %s for %s", r2, fulfills, r) for k in fulfills: auto_providers[k] = r2 requires.remove(k) if not requires: break # Anything still leftover?? if requires: # Ensure its in string format, since join will puke on # things that are not strings. missing = ", ".join(sorted([str(s) for s in requires])) raise exc.MissingDependencies(r, missing) else: r.providers = {} r.providers.update(auto_providers) r.providers.update(manual_providers)
def run(self): """Runs the flow in the engine to completion.""" if self.storage.get_flow_state() == states.REVERTED: self._reset() self.compile() external_provides = set(self.storage.fetch_all().keys()) missing = self._flow.requires - external_provides if missing: raise exc.MissingDependencies(self._flow, sorted(missing)) if self._failures: self._revert() else: self._run()
def _associate_providers(self, runner): # Ensure that some previous task provides this input. who_provides = {} task_requires = runner.requires for r in task_requires: provider = None for before_me in runner.runs_before: if r in before_me.provides: provider = before_me break if provider: who_provides[r] = provider # Ensure that the last task provides all the needed input for this # task to run correctly. missing_requires = task_requires - set(who_provides.keys()) if missing_requires: raise exc.MissingDependencies(runner, sorted(missing_requires)) runner.providers.update(who_provides)
def prepare(self): if not self._compiled: raise exc.InvalidState("Can not prepare an engine" " which has not been compiled") if not self._storage_ensured: self._ensure_storage() self._storage_ensured = True # At this point we can check to ensure all dependencies are either # flow/task provided or storage provided, if there are still missing # dependencies then this flow will fail at runtime (which we can avoid # by failing at preparation time). external_provides = set(self.storage.fetch_all().keys()) missing = self._flow.requires - external_provides if missing: raise exc.MissingDependencies(self._flow, sorted(missing)) # Reset everything back to pending (if we were previously reverted). if self.storage.get_flow_state() == states.REVERTED: self._runtime.reset_all() self._change_state(states.PENDING)
def validate(self): if not self._storage_ensured: raise exc.InvalidState("Can not validate an engine" " which has not has its storage" " populated") # At this point we can check to ensure all dependencies are either # flow/task provided or storage provided, if there are still missing # dependencies then this flow will fail at runtime (which we can avoid # by failing at validation time). execution_graph = self._compilation.execution_graph if LOG.isEnabledFor(logging.BLATHER): LOG.blather( "Validating scoping and argument visibility for" " execution graph with %s nodes and %s edges with" " density %0.3f", execution_graph.number_of_nodes(), execution_graph.number_of_edges(), nx.density(execution_graph)) missing = set() fetch = self.storage.fetch_unsatisfied_args for node in execution_graph.nodes_iter(): missing.update( fetch(node.name, node.rebind, optional_args=node.optional)) if missing: raise exc.MissingDependencies(self._flow, sorted(missing))