Exemplo n.º 1
0
 def validate(self):
     # At this point we can check to ensure all dependencies are either
     # flow/task provided or storage provided, if there are still missing
     # dependencies then this flow will fail at runtime (which we can avoid
     # by failing at validation time).
     if LOG.isEnabledFor(logging.TRACE):
         execution_graph = self._compilation.execution_graph
         LOG.trace(
             "Validating scoping and argument visibility for"
             " execution graph with %s nodes and %s edges with"
             " density %0.3f", execution_graph.number_of_nodes(),
             execution_graph.number_of_edges(), nx.density(execution_graph))
     missing = set()
     # Attempt to retain a chain of what was missing (so that the final
     # raised exception for the flow has the nodes that had missing
     # dependencies).
     last_cause = None
     last_node = None
     missing_nodes = 0
     for atom in self._runtime.iterate_nodes(compiler.ATOMS):
         exec_missing = self.storage.fetch_unsatisfied_args(
             atom.name, atom.rebind, optional_args=atom.optional)
         revert_missing = self.storage.fetch_unsatisfied_args(
             atom.name,
             atom.revert_rebind,
             optional_args=atom.revert_optional)
         atom_missing = (('execute', exec_missing), ('revert',
                                                     revert_missing))
         for method, method_missing in atom_missing:
             if method_missing:
                 cause = exc.MissingDependencies(atom,
                                                 sorted(method_missing),
                                                 cause=last_cause,
                                                 method=method)
                 last_cause = cause
                 last_node = atom
                 missing_nodes += 1
                 missing.update(method_missing)
     if missing:
         # For when a task is provided (instead of a flow) and that
         # task is the only item in the graph and its missing deps, avoid
         # re-wrapping it in yet another exception...
         if missing_nodes == 1 and last_node is self._flow:
             raise last_cause
         else:
             raise exc.MissingDependencies(self._flow,
                                           sorted(missing),
                                           cause=last_cause)
     self._validated = True
Exemplo n.º 2
0
 def validate(self):
     self._check('validate', True, True)
     # At this point we can check to ensure all dependencies are either
     # flow/task provided or storage provided, if there are still missing
     # dependencies then this flow will fail at runtime (which we can avoid
     # by failing at validation time).
     execution_graph = self._compilation.execution_graph
     if LOG.isEnabledFor(logging.BLATHER):
         LOG.blather(
             "Validating scoping and argument visibility for"
             " execution graph with %s nodes and %s edges with"
             " density %0.3f", execution_graph.number_of_nodes(),
             execution_graph.number_of_edges(), nx.density(execution_graph))
     missing = set()
     # Attempt to retain a chain of what was missing (so that the final
     # raised exception for the flow has the nodes that had missing
     # dependencies).
     last_cause = None
     last_node = None
     missing_nodes = 0
     fetch_func = self.storage.fetch_unsatisfied_args
     for node in execution_graph.nodes_iter():
         node_missing = fetch_func(node.name,
                                   node.rebind,
                                   optional_args=node.optional)
         if node_missing:
             cause = exc.MissingDependencies(node,
                                             sorted(node_missing),
                                             cause=last_cause)
             last_cause = cause
             last_node = node
             missing_nodes += 1
             missing.update(node_missing)
     if missing:
         # For when a task is provided (instead of a flow) and that
         # task is the only item in the graph and its missing deps, avoid
         # re-wrapping it in yet another exception...
         if missing_nodes == 1 and last_node is self._flow:
             raise last_cause
         else:
             raise exc.MissingDependencies(self._flow,
                                           sorted(missing),
                                           cause=last_cause)
Exemplo n.º 3
0
def connect(graph, infer_key='infer', auto_reason='auto', discard_func=None):
    """Connects a graphs runners to other runners in the graph which provide
    outputs for each runners requirements.
    """

    if len(graph) == 0:
        return
    if discard_func:
        for (u, v, e_data) in graph.edges(data=True):
            if discard_func(u, v, e_data):
                graph.remove_edge(u, v)
    for (r, r_data) in graph.nodes_iter(data=True):
        requires = set(r.requires)

        # Find the ones that have already been attached manually.
        manual_providers = {}
        if requires:
            incoming = [e[0] for e in graph.in_edges_iter([r])]
            for r2 in incoming:
                fulfills = requires & r2.provides
                if fulfills:
                    LOG.debug("%s is a manual provider of %s for %s", r2,
                              fulfills, r)
                    for k in fulfills:
                        manual_providers[k] = r2
                        requires.remove(k)

        # Anything leftover that we must find providers for??
        auto_providers = {}
        if requires and r_data.get(infer_key):
            for r2 in graph.nodes_iter():
                if r is r2:
                    continue
                fulfills = requires & r2.provides
                if fulfills:
                    graph.add_edge(r2, r, reason=auto_reason)
                    LOG.debug(
                        "Connecting %s as a automatic provider for"
                        " %s for %s", r2, fulfills, r)
                    for k in fulfills:
                        auto_providers[k] = r2
                        requires.remove(k)
                if not requires:
                    break

        # Anything still leftover??
        if requires:
            # Ensure its in string format, since join will puke on
            # things that are not strings.
            missing = ", ".join(sorted([str(s) for s in requires]))
            raise exc.MissingDependencies(r, missing)
        else:
            r.providers = {}
            r.providers.update(auto_providers)
            r.providers.update(manual_providers)
Exemplo n.º 4
0
 def run(self):
     """Runs the flow in the engine to completion."""
     if self.storage.get_flow_state() == states.REVERTED:
         self._reset()
     self.compile()
     external_provides = set(self.storage.fetch_all().keys())
     missing = self._flow.requires - external_provides
     if missing:
         raise exc.MissingDependencies(self._flow, sorted(missing))
     if self._failures:
         self._revert()
     else:
         self._run()
Exemplo n.º 5
0
 def _associate_providers(self, runner):
     # Ensure that some previous task provides this input.
     who_provides = {}
     task_requires = runner.requires
     for r in task_requires:
         provider = None
         for before_me in runner.runs_before:
             if r in before_me.provides:
                 provider = before_me
                 break
         if provider:
             who_provides[r] = provider
     # Ensure that the last task provides all the needed input for this
     # task to run correctly.
     missing_requires = task_requires - set(who_provides.keys())
     if missing_requires:
         raise exc.MissingDependencies(runner, sorted(missing_requires))
     runner.providers.update(who_provides)
Exemplo n.º 6
0
 def prepare(self):
     if not self._compiled:
         raise exc.InvalidState("Can not prepare an engine"
                                " which has not been compiled")
     if not self._storage_ensured:
         self._ensure_storage()
         self._storage_ensured = True
     # At this point we can check to ensure all dependencies are either
     # flow/task provided or storage provided, if there are still missing
     # dependencies then this flow will fail at runtime (which we can avoid
     # by failing at preparation time).
     external_provides = set(self.storage.fetch_all().keys())
     missing = self._flow.requires - external_provides
     if missing:
         raise exc.MissingDependencies(self._flow, sorted(missing))
     # Reset everything back to pending (if we were previously reverted).
     if self.storage.get_flow_state() == states.REVERTED:
         self._runtime.reset_all()
         self._change_state(states.PENDING)
Exemplo n.º 7
0
 def validate(self):
     if not self._storage_ensured:
         raise exc.InvalidState("Can not validate an engine"
                                " which has not has its storage"
                                " populated")
     # At this point we can check to ensure all dependencies are either
     # flow/task provided or storage provided, if there are still missing
     # dependencies then this flow will fail at runtime (which we can avoid
     # by failing at validation time).
     execution_graph = self._compilation.execution_graph
     if LOG.isEnabledFor(logging.BLATHER):
         LOG.blather(
             "Validating scoping and argument visibility for"
             " execution graph with %s nodes and %s edges with"
             " density %0.3f", execution_graph.number_of_nodes(),
             execution_graph.number_of_edges(), nx.density(execution_graph))
     missing = set()
     fetch = self.storage.fetch_unsatisfied_args
     for node in execution_graph.nodes_iter():
         missing.update(
             fetch(node.name, node.rebind, optional_args=node.optional))
     if missing:
         raise exc.MissingDependencies(self._flow, sorted(missing))