def connect_and_verify():
"""Do basic sanity tests on the graph structure."""
if len(self._graph) == 0:
return
self._connect()
degrees = [g[1] for g in self._graph.in_degree_iter()]
zero_degrees = [d for d in degrees if d == 0]
if not zero_degrees:
# If every task depends on something else to produce its input
# then we will be in a deadlock situation.
raise exc.InvalidStateException("No task has an in-degree"
" of zero")
self_loops = self._graph.nodes_with_selfloops()
if self_loops:
# A task that has a dependency on itself will never be able
# to run.
raise exc.InvalidStateException("%s tasks have been detected"
" with dependencies on"
" themselves" %
len(self_loops))
simple_cycles = len(cycles.recursive_simple_cycles(self._graph))
if simple_cycles:
# A task loop will never be able to run, unless it somehow
# breaks that loop.
raise exc.InvalidStateException("%s tasks have been detected"
" with dependency loops" %
simple_cycles)
def run_it(runner, failed=False, result=None, simulate_run=False):
try:
# Add the task to be rolled back *immediately* so that even if
# the task fails while producing results it will be given a
# chance to rollback.
rb = utils.RollbackTask(context, runner.task, result=None)
self._accumulator.add(rb)
self.task_notifier.notify(states.STARTED,
details={
'context': context,
'flow': self,
'runner': runner,
})
if not simulate_run:
result = runner(context, *args, **kwargs)
else:
if failed:
# TODO(harlowja): make this configurable??
# If we previously failed, we want to fail again at
# the same place.
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (runner.task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn(
"Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Adjust the task result in the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
rb.result = result
runner.result = result
self.results[runner.uuid] = result
self.task_notifier.notify(states.SUCCESS,
details={
'context': context,
'flow': self,
'runner': runner,
})
except Exception as e:
runner.result = e
cause = utils.FlowFailure(runner, self, e)
with excutils.save_and_reraise_exception():
# Notify any listeners that the task has errored.
self.task_notifier.notify(states.FAILURE,
details={
'context': context,
'flow': self,
'runner': runner,
})
self.rollback(context, cause)
def connect(self):
"""Connects the nodes & edges of the graph together."""
if self._connected or len(self._graph) == 0:
return
# Figure out the provider of items and the requirers of items.
provides_what = collections.defaultdict(list)
requires_what = collections.defaultdict(list)
for t in self._graph.nodes_iter():
for r in getattr(t, 'requires', []):
requires_what[r].append(t)
for p in getattr(t, 'provides', []):
provides_what[p].append(t)
def get_providers(node, want_what):
providers = []
for (producer, me) in self._graph.in_edges_iter(node):
providing_what = self._graph.get_edge_data(producer, me)
if want_what in providing_what:
providers.append(producer)
return providers
# Link providers to consumers of items.
for (want_what, who_wants) in requires_what.iteritems():
who_provided = 0
for p in provides_what[want_what]:
# P produces for N so thats why we link P->N and not N->P
for n in who_wants:
if p is n:
# No self-referencing allowed.
continue
if (len(get_providers(n, want_what))
and not self._allow_same_inputs):
msg = "Multiple providers of %s not allowed."
raise exc.InvalidStateException(msg % (want_what))
self._graph.add_edge(p, n, attr_dict={
want_what: True,
})
who_provided += 1
if not who_provided:
who_wants = ", ".join([str(a) for a in who_wants])
raise exc.InvalidStateException("%s requires input %s "
"but no other task produces "
"said output." %
(who_wants, want_what))
self._connected = True
def _ordering(self):
try:
return iter(self._connect())
except g_exc.NetworkXUnfeasible:
raise exc.InvalidStateException("Unable to correctly determine "
"the path through the provided "
"flow which will satisfy the "
"tasks needed inputs and outputs.")
def soft_reset(self):
"""Partially resets the internal state of this flow, allowing for the
flow to be ran again from an interrupted state only.
"""
if self.state not in self.SOFT_RESETTABLE_STATES:
raise exc.InvalidStateException(("Can not soft reset when"
" in state %s") % (self.state))
self._change_state(None, states.PENDING)
def order(self):
self.connect()
try:
return dag.topological_sort(self._graph)
except g_exc.NetworkXUnfeasible:
raise exc.InvalidStateException("Unable to correctly determine "
"the path through the provided "
"flow which will satisfy the "
"tasks needed inputs and outputs.")
def check_and_fetch():
if self.state not in self.MUTABLE_STATES:
raise exc.InvalidStateException("Flow is currently in a"
" non-mutable state %s"
% (self.state))
provider = self._find_uuid(provider_uuid)
if not provider or not self._graph.has_node(provider):
raise exc.InvalidStateException("Can not add a dependency "
"from unknown uuid %s" %
(provider_uuid))
consumer = self._find_uuid(consumer_uuid)
if not consumer or not self._graph.has_node(consumer):
raise exc.InvalidStateException("Can not add a dependency "
"to unknown uuid %s"
% (consumer_uuid))
if provider is consumer:
raise exc.InvalidStateException("Can not add a dependency "
"to loop via uuid %s"
% (consumer_uuid))
return (provider, consumer)
def reset(self):
"""Fully resets the internal state of this flow, allowing for the flow
to be ran again.
Note: Listeners are also reset.
"""
if self.state not in self.RESETTABLE_STATES:
raise exc.InvalidStateException(("Can not reset when"
" in state %s") % (self.state))
self.notifier.reset()
self.task_notifier.reset()
self._change_state(None, states.PENDING)
def erase(self, job):
with self._lock.acquire(read=False):
# Ensure that we even have said job in the first place.
exists = False
for (d, j) in self._board:
if j == job:
exists = True
break
if not exists:
raise exc.JobNotFound()
if job.state not in (states.SUCCESS, states.FAILURE):
raise exc.InvalidStateException("Can not delete a job in "
"state %s" % (job.state))
self._board = [(d, j) for (d, j) in self._board if j != job]
self._notify_erased(job)
def _validate_provides(self, task):
# Ensure that some previous task provides this input.
missing_requires = []
for r in getattr(task, 'requires', []):
found_provider = False
for prev_task in reversed(self._tasks):
if r in getattr(prev_task, 'provides', []):
found_provider = True
break
if not found_provider:
missing_requires.append(r)
# Ensure that the last task provides all the needed input for this
# task to run correctly.
if len(missing_requires):
msg = ("There is no previous task providing the outputs %s"
" for %s to correctly execute.") % (missing_requires, task)
raise exc.InvalidStateException(msg)
def check_task_transition(old_state, new_state):
"""Check that task can transition from old_state to new_state.
If transition can be performed, it returns True. If transition
should be ignored, it returns False. If transition is not
valid, it raises InvalidStateException.
"""
pair = (old_state, new_state)
if pair in _ALLOWED_TASK_TRANSITIONS:
return True
if pair in _IGNORED_TASK_TRANSITIONS:
return False
# TODO(harlowja): Should we check/allow for 3rd party states to be
# triggered during RUNNING by having a concept of a sub-state that we also
# verify against??
raise exc.InvalidStateException(
"Task transition from %s to %s is not allowed" % pair)
def check_flow_transition(old_state, new_state):
"""Check that flow can transition from old_state to new_state.
If transition can be performed, it returns True. If transition
should be ignored, it returns False. If transition is not
valid, it raises InvalidStateException.
"""
if old_state == new_state:
return False
pair = (old_state, new_state)
if pair in _ALLOWED_FLOW_TRANSITIONS:
return True
if pair in _IGNORED_FLOW_TRANSITIONS:
return False
if new_state == RESUMING:
return True
raise exc.InvalidStateException(
"Flow transition from %s to %s is not allowed" % pair)
def run(self, flow, *args, **kwargs):
already_associated = []
def associate_all(a_flow):
if a_flow in already_associated:
return
# Associate with the flow.
self.associate(a_flow)
already_associated.append(a_flow)
# Ensure we are associated with all the flows parents.
if a_flow.parents:
for p in a_flow.parents:
associate_all(p)
if flow.state != states.PENDING:
raise exc.InvalidStateException("Unable to run %s when in"
" state %s" % (flow, flow.state))
associate_all(flow)
return flow.run(self.context, *args, **kwargs)
def interrupt(self):
"""Attempts to interrupt the current flow and any tasks that are
currently not running in the flow.
Returns how many tasks were interrupted (if any).
"""
if self.state in self.UNINTERRUPTIBLE_STATES:
raise exc.InvalidStateException(("Can not interrupt when"
" in state %s") % (self.state))
# Note(harlowja): Do *not* acquire the lock here so that the flow may
# be interrupted while running. This does mean the the above check may
# not be valid but we can worry about that if it becomes an issue.
old_state = self.state
if old_state != states.INTERRUPTED:
self._state = states.INTERRUPTED
self.notifier.notify(self.state, details={
'context': None,
'flow': self,
'old_state': old_state,
})
return 0
def check():
if self.state not in self.MUTABLE_STATES:
raise exc.InvalidStateException("Flow is currently in a"
" non-mutable state %s"
% (self.state))
def run(self, context, *args, **kwargs):
"""Executes the workflow."""
if self.state not in self.RUNNABLE_STATES:
raise exc.InvalidStateException("Unable to run flow when "
"in state %s" % (self.state))
def check():
if self.state not in self.CANCELLABLE_STATES:
raise exc.InvalidStateException("Can not attempt cancellation"
" when in state %s" %
self.state)
def check():
if self.state not in self.RESETTABLE_STATES:
raise exc.InvalidStateException("Runner not in a resettable"
" state: %s" % (self.state))
def check():
if self.state not in self.CANCELABLE_STATES:
raise exc.InvalidStateException("Runner not in a cancelable"
" state: %s" % (self.state))
def check():
if self.state not in self.REVERTABLE_STATES:
raise exc.InvalidStateException("Flow is currently unable "
"to be rolled back in "
"state %s" % (self.state))
def check():
if self.state not in self.RUNNABLE_STATES:
raise exc.InvalidStateException("Flow is currently unable "
"to be ran in state %s"
% (self.state))