def connect_and_verify(): """Do basic sanity tests on the graph structure.""" if len(self._graph) == 0: return self._connect() degrees = [g[1] for g in self._graph.in_degree_iter()] zero_degrees = [d for d in degrees if d == 0] if not zero_degrees: # If every task depends on something else to produce its input # then we will be in a deadlock situation. raise exc.InvalidStateException("No task has an in-degree" " of zero") self_loops = self._graph.nodes_with_selfloops() if self_loops: # A task that has a dependency on itself will never be able # to run. raise exc.InvalidStateException("%s tasks have been detected" " with dependencies on" " themselves" % len(self_loops)) simple_cycles = len(cycles.recursive_simple_cycles(self._graph)) if simple_cycles: # A task loop will never be able to run, unless it somehow # breaks that loop. raise exc.InvalidStateException("%s tasks have been detected" " with dependency loops" % simple_cycles)
def run_it(runner, failed=False, result=None, simulate_run=False): try: # Add the task to be rolled back *immediately* so that even if # the task fails while producing results it will be given a # chance to rollback. rb = utils.RollbackTask(context, runner.task, result=None) self._accumulator.add(rb) self.task_notifier.notify(states.STARTED, details={ 'context': context, 'flow': self, 'runner': runner, }) if not simulate_run: result = runner(context, *args, **kwargs) else: if failed: # TODO(harlowja): make this configurable?? # If we previously failed, we want to fail again at # the same place. if not result: # If no exception or exception message was provided # or captured from the previous run then we need to # form one for this task. result = "%s failed running." % (runner.task) if isinstance(result, basestring): result = exc.InvalidStateException(result) if not isinstance(result, Exception): LOG.warn( "Can not raise a non-exception" " object: %s", result) result = exc.InvalidStateException() raise result # Adjust the task result in the accumulator before # notifying others that the task has finished to # avoid the case where a listener might throw an # exception. rb.result = result runner.result = result self.results[runner.uuid] = result self.task_notifier.notify(states.SUCCESS, details={ 'context': context, 'flow': self, 'runner': runner, }) except Exception as e: runner.result = e cause = utils.FlowFailure(runner, self, e) with excutils.save_and_reraise_exception(): # Notify any listeners that the task has errored. self.task_notifier.notify(states.FAILURE, details={ 'context': context, 'flow': self, 'runner': runner, }) self.rollback(context, cause)
def connect(self): """Connects the nodes & edges of the graph together.""" if self._connected or len(self._graph) == 0: return # Figure out the provider of items and the requirers of items. provides_what = collections.defaultdict(list) requires_what = collections.defaultdict(list) for t in self._graph.nodes_iter(): for r in getattr(t, 'requires', []): requires_what[r].append(t) for p in getattr(t, 'provides', []): provides_what[p].append(t) def get_providers(node, want_what): providers = [] for (producer, me) in self._graph.in_edges_iter(node): providing_what = self._graph.get_edge_data(producer, me) if want_what in providing_what: providers.append(producer) return providers # Link providers to consumers of items. for (want_what, who_wants) in requires_what.iteritems(): who_provided = 0 for p in provides_what[want_what]: # P produces for N so thats why we link P->N and not N->P for n in who_wants: if p is n: # No self-referencing allowed. continue if (len(get_providers(n, want_what)) and not self._allow_same_inputs): msg = "Multiple providers of %s not allowed." raise exc.InvalidStateException(msg % (want_what)) self._graph.add_edge(p, n, attr_dict={ want_what: True, }) who_provided += 1 if not who_provided: who_wants = ", ".join([str(a) for a in who_wants]) raise exc.InvalidStateException("%s requires input %s " "but no other task produces " "said output." % (who_wants, want_what)) self._connected = True
def _ordering(self): try: return iter(self._connect()) except g_exc.NetworkXUnfeasible: raise exc.InvalidStateException("Unable to correctly determine " "the path through the provided " "flow which will satisfy the " "tasks needed inputs and outputs.")
def soft_reset(self): """Partially resets the internal state of this flow, allowing for the flow to be ran again from an interrupted state only. """ if self.state not in self.SOFT_RESETTABLE_STATES: raise exc.InvalidStateException(("Can not soft reset when" " in state %s") % (self.state)) self._change_state(None, states.PENDING)
def order(self): self.connect() try: return dag.topological_sort(self._graph) except g_exc.NetworkXUnfeasible: raise exc.InvalidStateException("Unable to correctly determine " "the path through the provided " "flow which will satisfy the " "tasks needed inputs and outputs.")
def check_and_fetch(): if self.state not in self.MUTABLE_STATES: raise exc.InvalidStateException("Flow is currently in a" " non-mutable state %s" % (self.state)) provider = self._find_uuid(provider_uuid) if not provider or not self._graph.has_node(provider): raise exc.InvalidStateException("Can not add a dependency " "from unknown uuid %s" % (provider_uuid)) consumer = self._find_uuid(consumer_uuid) if not consumer or not self._graph.has_node(consumer): raise exc.InvalidStateException("Can not add a dependency " "to unknown uuid %s" % (consumer_uuid)) if provider is consumer: raise exc.InvalidStateException("Can not add a dependency " "to loop via uuid %s" % (consumer_uuid)) return (provider, consumer)
def reset(self): """Fully resets the internal state of this flow, allowing for the flow to be ran again. Note: Listeners are also reset. """ if self.state not in self.RESETTABLE_STATES: raise exc.InvalidStateException(("Can not reset when" " in state %s") % (self.state)) self.notifier.reset() self.task_notifier.reset() self._change_state(None, states.PENDING)
def erase(self, job): with self._lock.acquire(read=False): # Ensure that we even have said job in the first place. exists = False for (d, j) in self._board: if j == job: exists = True break if not exists: raise exc.JobNotFound() if job.state not in (states.SUCCESS, states.FAILURE): raise exc.InvalidStateException("Can not delete a job in " "state %s" % (job.state)) self._board = [(d, j) for (d, j) in self._board if j != job] self._notify_erased(job)
def _validate_provides(self, task): # Ensure that some previous task provides this input. missing_requires = [] for r in getattr(task, 'requires', []): found_provider = False for prev_task in reversed(self._tasks): if r in getattr(prev_task, 'provides', []): found_provider = True break if not found_provider: missing_requires.append(r) # Ensure that the last task provides all the needed input for this # task to run correctly. if len(missing_requires): msg = ("There is no previous task providing the outputs %s" " for %s to correctly execute.") % (missing_requires, task) raise exc.InvalidStateException(msg)
def check_task_transition(old_state, new_state): """Check that task can transition from old_state to new_state. If transition can be performed, it returns True. If transition should be ignored, it returns False. If transition is not valid, it raises InvalidStateException. """ pair = (old_state, new_state) if pair in _ALLOWED_TASK_TRANSITIONS: return True if pair in _IGNORED_TASK_TRANSITIONS: return False # TODO(harlowja): Should we check/allow for 3rd party states to be # triggered during RUNNING by having a concept of a sub-state that we also # verify against?? raise exc.InvalidStateException( "Task transition from %s to %s is not allowed" % pair)
def check_flow_transition(old_state, new_state): """Check that flow can transition from old_state to new_state. If transition can be performed, it returns True. If transition should be ignored, it returns False. If transition is not valid, it raises InvalidStateException. """ if old_state == new_state: return False pair = (old_state, new_state) if pair in _ALLOWED_FLOW_TRANSITIONS: return True if pair in _IGNORED_FLOW_TRANSITIONS: return False if new_state == RESUMING: return True raise exc.InvalidStateException( "Flow transition from %s to %s is not allowed" % pair)
def run(self, flow, *args, **kwargs): already_associated = [] def associate_all(a_flow): if a_flow in already_associated: return # Associate with the flow. self.associate(a_flow) already_associated.append(a_flow) # Ensure we are associated with all the flows parents. if a_flow.parents: for p in a_flow.parents: associate_all(p) if flow.state != states.PENDING: raise exc.InvalidStateException("Unable to run %s when in" " state %s" % (flow, flow.state)) associate_all(flow) return flow.run(self.context, *args, **kwargs)
def interrupt(self): """Attempts to interrupt the current flow and any tasks that are currently not running in the flow. Returns how many tasks were interrupted (if any). """ if self.state in self.UNINTERRUPTIBLE_STATES: raise exc.InvalidStateException(("Can not interrupt when" " in state %s") % (self.state)) # Note(harlowja): Do *not* acquire the lock here so that the flow may # be interrupted while running. This does mean the the above check may # not be valid but we can worry about that if it becomes an issue. old_state = self.state if old_state != states.INTERRUPTED: self._state = states.INTERRUPTED self.notifier.notify(self.state, details={ 'context': None, 'flow': self, 'old_state': old_state, }) return 0
def check(): if self.state not in self.MUTABLE_STATES: raise exc.InvalidStateException("Flow is currently in a" " non-mutable state %s" % (self.state))
def run(self, context, *args, **kwargs): """Executes the workflow.""" if self.state not in self.RUNNABLE_STATES: raise exc.InvalidStateException("Unable to run flow when " "in state %s" % (self.state))
def check(): if self.state not in self.CANCELLABLE_STATES: raise exc.InvalidStateException("Can not attempt cancellation" " when in state %s" % self.state)
def check(): if self.state not in self.RESETTABLE_STATES: raise exc.InvalidStateException("Runner not in a resettable" " state: %s" % (self.state))
def check(): if self.state not in self.CANCELABLE_STATES: raise exc.InvalidStateException("Runner not in a cancelable" " state: %s" % (self.state))
def check(): if self.state not in self.REVERTABLE_STATES: raise exc.InvalidStateException("Flow is currently unable " "to be rolled back in " "state %s" % (self.state))
def check(): if self.state not in self.RUNNABLE_STATES: raise exc.InvalidStateException("Flow is currently unable " "to be ran in state %s" % (self.state))