def post(self, name, book, details=None):
# Didn't work, clean it up.
def try_clean(path):
with self._job_mutate:
self._remove_job(path)
# NOTE(harlowja): Jobs are not ephemeral, they will persist until they
# are consumed (this may change later, but seems safer to do this until
# further notice).
job = ZookeeperJob(name, self,
self._client, self._persistence,
book=book, details=details)
job_path, _lock_path = _get_paths(self.path, job.uuid)
# NOTE(harlowja): This avoids the watcher thread from attempting to
# overwrite or delete this job which is not yet ready but is in the
# process of being posted.
with self._job_mutate:
self._known_jobs[job_path] = (job, _POSTING)
with self._wrap(job.uuid, "Posting failure: %s", ensure_known=False):
try:
self._client.create(job_path, value=self._format_job(job))
with self._job_mutate:
self._known_jobs[job_path] = (job, _READY)
return job
except k_exceptions.NodeExistsException:
try_clean()
raise excp.JobAlreadyExists("Duplicate job %s already posted"
% job.uuid)
except Exception:
with excutils.save_and_reraise_exception():
try_clean()
def _perform_reconcilation(self, context, task, excp):
# Attempt to reconcile the given exception that occured while applying
# the given task and either reconcile said task and its associated
# failure, so that the workflow can continue or abort and perform
# some type of undo of the tasks already completed.
try:
self._change_state(context, states.REVERTING)
except Exception:
LOG.exception("Dropping exception catched when"
" changing state to reverting while performing"
" reconcilation on a tasks exception.")
cause = exc.TaskException(task, self, excp)
with excutils.save_and_reraise_exception():
try:
self._on_task_error(context, task)
except Exception:
LOG.exception("Dropping exception catched when"
" notifying about existing task"
" exception.")
# The default strategy will be to rollback all the contained
# tasks by calling there reverting methods, and then calling
# any parent workflows rollbacks (and so-on).
try:
self.rollback(context, cause)
finally:
try:
self._change_state(context, states.FAILURE)
except Exception:
LOG.exception("Dropping exception catched when"
" changing state to failure while performing"
" reconcilation on a tasks exception.")
def post(self, name, book, details=None):
# Didn't work, clean it up.
def try_clean(path):
with self._job_mutate:
self._remove_job(path)
# NOTE(harlowja): Jobs are not ephemeral, they will persist until they
# are consumed (this may change later, but seems safer to do this until
# further notice).
job = ZookeeperJob(name,
self,
self._client,
self._persistence,
book=book,
details=details)
job_path, _lock_path = _get_paths(self.path, job.uuid)
# NOTE(harlowja): This avoids the watcher thread from attempting to
# overwrite or delete this job which is not yet ready but is in the
# process of being posted.
with self._job_mutate:
self._known_jobs[job_path] = (job, _POSTING)
with self._wrap(job.uuid, "Posting failure: %s", ensure_known=False):
try:
self._client.create(job_path, value=self._format_job(job))
with self._job_mutate:
self._known_jobs[job_path] = (job, _READY)
return job
except k_exceptions.NodeExistsException:
try_clean()
raise excp.JobAlreadyExists("Duplicate job %s already posted" %
job.uuid)
except Exception:
with excutils.save_and_reraise_exception():
try_clean()
def connect(self, timeout=10.0):
def try_clean():
# Attempt to do the needed cleanup if post-connection setup does
# not succeed (maybe the connection is lost right after it is
# obtained).
try:
self.close()
except k_exceptions.KazooException:
LOG.exception("Failed cleaning-up after post-connection"
" initialization failed")
try:
if timeout is not None:
timeout = float(timeout)
self._client.start(timeout=timeout)
except (self._client.handler.timeout_exception,
k_exceptions.KazooException) as e:
raise excp.ConnectionFailure("Failed to connect to"
" zookeeper due to: %s" % (e))
try:
kazoo_utils.check_compatible(self._client, MIN_ZK_VERSION)
self._client.ensure_path(self.path)
self._job_watcher = watchers.ChildrenWatch(
self._client,
self.path,
func=self._on_job_posting,
allow_session_lost=False)
except excp.IncompatibleVersion:
with excutils.save_and_reraise_exception():
try_clean()
except (self._client.handler.timeout_exception,
k_exceptions.KazooException) as e:
try_clean()
raise excp.ConnectionFailure("Failed to do post-connection"
" initialization due to: %s" % (e))
def run_it(runner, failed=False, result=None, simulate_run=False):
try:
# Add the task to be rolled back *immediately* so that even if
# the task fails while producing results it will be given a
# chance to rollback.
rb = utils.RollbackTask(context, runner.task, result=None)
self._accumulator.add(rb)
self.task_notifier.notify(states.STARTED,
details={
'context': context,
'flow': self,
'runner': runner,
})
if not simulate_run:
result = runner(context, *args, **kwargs)
else:
if failed:
# TODO(harlowja): make this configurable??
# If we previously failed, we want to fail again at
# the same place.
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (runner.task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn(
"Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Adjust the task result in the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
rb.result = result
runner.result = result
self.results[runner.uuid] = result
self.task_notifier.notify(states.SUCCESS,
details={
'context': context,
'flow': self,
'runner': runner,
})
except Exception as e:
runner.result = e
cause = utils.FlowFailure(runner, self, e)
with excutils.save_and_reraise_exception():
# Notify any listeners that the task has errored.
self.task_notifier.notify(states.FAILURE,
details={
'context': context,
'flow': self,
'runner': runner,
})
self.rollback(context, cause)
def run_it(runner, failed=False, result=None, simulate_run=False):
try:
# Add the task to be rolled back *immediately* so that even if
# the task fails while producing results it will be given a
# chance to rollback.
rb = utils.RollbackTask(context, runner.task, result=None)
self._accumulator.add(rb)
self.task_notifier.notify(states.STARTED, details={
'context': context,
'flow': self,
'runner': runner,
})
if not simulate_run:
result = runner(context, *args, **kwargs)
else:
if failed:
# TODO(harlowja): make this configurable??
# If we previously failed, we want to fail again at
# the same place.
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (runner.task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn("Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Adjust the task result in the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
rb.result = result
runner.result = result
self.results[runner.uuid] = result
self.task_notifier.notify(states.SUCCESS, details={
'context': context,
'flow': self,
'runner': runner,
})
except Exception as e:
runner.result = e
cause = utils.FlowFailure(runner, self, e)
with excutils.save_and_reraise_exception():
# Notify any listeners that the task has errored.
self.task_notifier.notify(states.FAILURE, details={
'context': context,
'flow': self,
'runner': runner,
})
self.rollback(context, cause)
def _register(watch_states, notifier, cb):
registered = []
try:
for s in watch_states:
if not notifier.is_registered(s, cb):
notifier.register(s, cb)
registered.append((s, cb))
except ValueError:
with excutils.save_and_reraise_exception():
for (s, cb) in registered:
notifier.deregister(s, cb)
def remove_path_on_error(path):
"""Protect code that wants to operate on PATH atomically.
Any exception will cause PATH to be removed.
:param path: File to work with
"""
try:
yield
except Exception:
with excutils.save_and_reraise_exception():
delete_if_exists(path)
def _register(watch_states, notifier, cb):
registered = []
try:
for s in watch_states:
if not notifier.is_registered(s, cb):
notifier.register(s, cb)
registered.append((s, cb))
except ValueError:
with excutils.save_and_reraise_exception():
for (s, cb) in registered:
notifier.deregister(s, cb)
def revert(self, task):
if not self._change_state(task, states.REVERTING, progress=0.0):
return
with _autobind(task, 'update_progress', self._on_update_progress):
kwargs = self._storage.fetch_mapped_args(task.rebind)
kwargs['result'] = self._storage.get(task.name)
kwargs['flow_failures'] = self._storage.get_failures()
try:
task.revert(**kwargs)
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(task, states.FAILURE)
self._change_state(task, states.REVERTED, progress=1.0)
def _revert(self, current_failure=None):
self._change_state(states.REVERTING)
try:
state = self._root.revert(self)
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(states.FAILURE)
self._change_state(state)
if state == states.SUSPENDED:
return
misc.Failure.reraise_if_any(self._failures.values())
if current_failure:
current_failure.reraise()
def _revert(self, current_failure=None):
self._change_state(states.REVERTING)
try:
state = self._root.revert(self)
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(states.FAILURE)
self._change_state(state)
if state == states.SUSPENDED:
return
misc.Failure.reraise_if_any(self._failures.values())
if current_failure:
current_failure.reraise()
def run_iter(self, timeout=None):
"""Runs the engine using iteration (or die trying).
:param timeout: timeout to wait for any tasks to complete (this timeout
will be used during the waiting period that occurs after the
waiting state is yielded when unfinished tasks are being waited
for).
Instead of running to completion in a blocking manner, this will
return a generator which will yield back the various states that the
engine is going through (and can be used to run multiple engines at
once using a generator per engine). the iterator returned also
responds to the send() method from pep-0342 and will attempt to suspend
itself if a truthy value is sent in (the suspend may be delayed until
all active tasks have finished).
NOTE(harlowja): using the run_iter method will **not** retain the
engine lock while executing so the user should ensure that there is
only one entity using a returned engine iterator (one per engine) at a
given time.
"""
self.compile()
self.prepare()
self._task_executor.start()
state = None
runner = self._runtime.runner
try:
self._change_state(states.RUNNING)
for state in runner.run_iter(timeout=timeout):
try:
try_suspend = yield state
except GeneratorExit:
break
else:
if try_suspend:
self.suspend()
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(states.FAILURE)
else:
ignorable_states = getattr(runner, 'ignorable_states', [])
if state and state not in ignorable_states:
self._change_state(state)
if state != states.SUSPENDED and state != states.SUCCESS:
failures = self.storage.get_failures()
misc.Failure.reraise_if_any(failures.values())
finally:
self._task_executor.stop()
def revert(self, engine):
if engine.storage.get_task_state(self.uuid) == states.PENDING:
# NOTE(imelnikov): in all the other states, the task
# execution was at least attempted, so we should give
# task a chance for cleanup
return
kwargs = engine.storage.fetch_mapped_args(self._args_mapping)
self._change_state(engine, states.REVERTING)
try:
self._task.revert(result=engine.storage.get(self._id), **kwargs)
self._change_state(engine, states.REVERTED)
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(engine, states.FAILURE)
else:
self._update_result(engine, states.PENDING)
def revert(self, engine):
if not self._change_state_update_task(engine, states.REVERTING, 0.0):
# NOTE(imelnikov): in all the other states, the task
# execution was at least attempted, so we should give
# task a chance for cleanup
return
with _autobind(self._task,
'update_progress', self._on_update_progress,
engine=engine):
kwargs = engine.storage.fetch_mapped_args(self._task.rebind)
kwargs['result'] = engine.storage.get(self._id)
try:
self._task.revert(**kwargs)
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(engine, states.FAILURE)
self._change_state_update_task(engine, states.REVERTED, 1.0)
def run_task(task, failed=False, result=None, simulate_run=False):
try:
self._on_task_start(context, task)
if not simulate_run:
inputs = self._fetch_task_inputs(task)
if not inputs:
inputs = {}
inputs.update(kwargs)
result = task(context, *args, **inputs)
else:
if failed:
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn("Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Keep a pristine copy of the result
# so that if said result is altered by other further
# states the one here will not be. This ensures that
# if rollback occurs that the task gets exactly the
# result it returned and not a modified one.
self.results.append((task, result))
# Add the task result to the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
self._accumulator.add(RollbackTask(context, task,
copy.deepcopy(result)))
self._on_task_finish(context, task, result)
except Exception as e:
cause = FlowFailure(task, self, e)
with excutils.save_and_reraise_exception():
try:
self._on_task_error(context, task, e)
except Exception:
LOG.exception("Dropping exception catched when"
" notifying about task failure.")
self.rollback(context, cause)
def revert(self, engine):
if not self._change_state_update_task(engine, states.REVERTING, 0.0):
# NOTE(imelnikov): in all the other states, the task
# execution was at least attempted, so we should give
# task a chance for cleanup
return
with _autobind(self._task,
'update_progress',
self._on_update_progress,
engine=engine):
kwargs = engine.storage.fetch_mapped_args(self._task.rebind)
kwargs['result'] = engine.storage.get(self._id)
try:
self._task.revert(**kwargs)
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(engine, states.FAILURE)
self._change_state_update_task(engine, states.REVERTED, 1.0)
def connect(self, timeout=10.0):
def try_clean():
# Attempt to do the needed cleanup if post-connection setup does
# not succeed (maybe the connection is lost right after it is
# obtained).
try:
self.close()
except k_exceptions.KazooException:
LOG.exception("Failed cleaning-up after post-connection"
" initialization failed")
try:
if timeout is not None:
timeout = float(timeout)
self._client.start(timeout=timeout)
except (self._client.handler.timeout_exception,
k_exceptions.KazooException) as e:
raise excp.JobFailure("Failed to connect to zookeeper", e)
try:
kazoo_utils.check_compatible(self._client, MIN_ZK_VERSION)
if self._worker is None and self._emit_notifications:
self._worker = futures.ThreadPoolExecutor(max_workers=1)
self._client.ensure_path(self.path)
if self._job_watcher is None:
self._job_watcher = watchers.ChildrenWatch(
self._client,
self.path,
func=self._on_job_posting,
allow_session_lost=True)
except excp.IncompatibleVersion:
with excutils.save_and_reraise_exception():
try_clean()
except (self._client.handler.timeout_exception,
k_exceptions.KazooException) as e:
try_clean()
raise excp.JobFailure("Failed to do post-connection"
" initialization", e)
def connect(self, timeout=10.0):
def try_clean():
# Attempt to do the needed cleanup if post-connection setup does
# not succeed (maybe the connection is lost right after it is
# obtained).
try:
self.close()
except k_exceptions.KazooException:
LOG.exception("Failed cleaning-up after post-connection"
" initialization failed")
try:
if timeout is not None:
timeout = float(timeout)
self._client.start(timeout=timeout)
except (self._client.handler.timeout_exception,
k_exceptions.KazooException) as e:
raise excp.ConnectionFailure("Failed to connect to"
" zookeeper due to: %s" % (e))
try:
kazoo_utils.check_compatible(self._client, MIN_ZK_VERSION)
self._client.ensure_path(self.path)
self._job_watcher = watchers.ChildrenWatch(
self._client,
self.path,
func=self._on_job_posting,
allow_session_lost=False)
except excp.IncompatibleVersion:
with excutils.save_and_reraise_exception():
try_clean()
except (self._client.handler.timeout_exception,
k_exceptions.KazooException) as e:
try_clean()
raise excp.ConnectionFailure("Failed to do post-connection"
" initialization due to: %s" % (e))
def run(self, context, *args, **kwargs):
super(Flow, self).run(context, *args, **kwargs)
def resume_it():
if self._leftoff_at is not None:
return ([], self._leftoff_at)
if self.resumer:
(finished,
leftover) = self.resumer.resume(self, self._ordering())
else:
finished = []
leftover = self._ordering()
return (finished, leftover)
self._change_state(context, states.STARTED)
try:
those_finished, leftover = resume_it()
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(context, states.FAILURE)
def run_it(runner, failed=False, result=None, simulate_run=False):
try:
# Add the task to be rolled back *immediately* so that even if
# the task fails while producing results it will be given a
# chance to rollback.
rb = utils.RollbackTask(context, runner.task, result=None)
self._accumulator.add(rb)
self.task_notifier.notify(states.STARTED,
details={
'context': context,
'flow': self,
'runner': runner,
})
if not simulate_run:
result = runner(context, *args, **kwargs)
else:
if failed:
# TODO(harlowja): make this configurable??
# If we previously failed, we want to fail again at
# the same place.
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (runner.task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn(
"Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Adjust the task result in the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
rb.result = result
runner.result = result
self.results[runner.uuid] = result
self.task_notifier.notify(states.SUCCESS,
details={
'context': context,
'flow': self,
'runner': runner,
})
except Exception as e:
runner.result = e
cause = utils.FlowFailure(runner, self, e)
with excutils.save_and_reraise_exception():
# Notify any listeners that the task has errored.
self.task_notifier.notify(states.FAILURE,
details={
'context': context,
'flow': self,
'runner': runner,
})
self.rollback(context, cause)
if len(those_finished):
self._change_state(context, states.RESUMING)
for (r, details) in those_finished:
# Fake running the task so that we trigger the same
# notifications and state changes (and rollback that
# would have happened in a normal flow).
failed = states.FAILURE in details.get('states', [])
result = details.get('result')
run_it(r, failed=failed, result=result, simulate_run=True)
self._leftoff_at = leftover
self._change_state(context, states.RUNNING)
if self.state == states.INTERRUPTED:
return
was_interrupted = False
for r in leftover:
r.reset()
run_it(r)
if self.state == states.INTERRUPTED:
was_interrupted = True
break
if not was_interrupted:
# Only gets here if everything went successfully.
self._change_state(context, states.SUCCESS)
self._leftoff_at = None
def run(self, context, *args, **kwargs):
super(Flow, self).run(context, *args, **kwargs)
def resume_it():
if self._leftoff_at is not None:
return ([], self._leftoff_at)
if self.resumer:
(finished, leftover) = self.resumer.resume(self,
self._ordering())
else:
finished = []
leftover = self._ordering()
return (finished, leftover)
self._change_state(context, states.STARTED)
try:
those_finished, leftover = resume_it()
except Exception:
with excutils.save_and_reraise_exception():
self._change_state(context, states.FAILURE)
def run_it(runner, failed=False, result=None, simulate_run=False):
try:
# Add the task to be rolled back *immediately* so that even if
# the task fails while producing results it will be given a
# chance to rollback.
rb = utils.RollbackTask(context, runner.task, result=None)
self._accumulator.add(rb)
self.task_notifier.notify(states.STARTED, details={
'context': context,
'flow': self,
'runner': runner,
})
if not simulate_run:
result = runner(context, *args, **kwargs)
else:
if failed:
# TODO(harlowja): make this configurable??
# If we previously failed, we want to fail again at
# the same place.
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (runner.task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn("Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Adjust the task result in the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
rb.result = result
runner.result = result
self.results[runner.uuid] = result
self.task_notifier.notify(states.SUCCESS, details={
'context': context,
'flow': self,
'runner': runner,
})
except Exception as e:
runner.result = e
cause = utils.FlowFailure(runner, self, e)
with excutils.save_and_reraise_exception():
# Notify any listeners that the task has errored.
self.task_notifier.notify(states.FAILURE, details={
'context': context,
'flow': self,
'runner': runner,
})
self.rollback(context, cause)
if len(those_finished):
self._change_state(context, states.RESUMING)
for (r, details) in those_finished:
# Fake running the task so that we trigger the same
# notifications and state changes (and rollback that
# would have happened in a normal flow).
failed = states.FAILURE in details.get('states', [])
result = details.get('result')
run_it(r, failed=failed, result=result, simulate_run=True)
self._leftoff_at = leftover
self._change_state(context, states.RUNNING)
if self.state == states.INTERRUPTED:
return
was_interrupted = False
for r in leftover:
r.reset()
run_it(r)
if self.state == states.INTERRUPTED:
was_interrupted = True
break
if not was_interrupted:
# Only gets here if everything went successfully.
self._change_state(context, states.SUCCESS)
self._leftoff_at = None
def run(self, context, *args, **kwargs):
if self.state != states.PENDING:
raise exc.InvalidStateException("Unable to run flow when "
"in state %s" % (self.state))
if self.result_fetcher:
result_fetcher = functools.partial(self.result_fetcher, context)
else:
result_fetcher = None
self._change_state(context, states.STARTED)
try:
task_order = self.order()
except Exception:
with excutils.save_and_reraise_exception():
try:
self._change_state(context, states.FAILURE)
except Exception:
LOG.exception("Dropping exception catched when"
" notifying about ordering failure.")
def run_task(task, failed=False, result=None, simulate_run=False):
try:
self._on_task_start(context, task)
if not simulate_run:
inputs = self._fetch_task_inputs(task)
if not inputs:
inputs = {}
inputs.update(kwargs)
result = task(context, *args, **inputs)
else:
if failed:
if not result:
# If no exception or exception message was provided
# or captured from the previous run then we need to
# form one for this task.
result = "%s failed running." % (task)
if isinstance(result, basestring):
result = exc.InvalidStateException(result)
if not isinstance(result, Exception):
LOG.warn("Can not raise a non-exception"
" object: %s", result)
result = exc.InvalidStateException()
raise result
# Keep a pristine copy of the result
# so that if said result is altered by other further
# states the one here will not be. This ensures that
# if rollback occurs that the task gets exactly the
# result it returned and not a modified one.
self.results.append((task, result))
# Add the task result to the accumulator before
# notifying others that the task has finished to
# avoid the case where a listener might throw an
# exception.
self._accumulator.add(RollbackTask(context, task,
copy.deepcopy(result)))
self._on_task_finish(context, task, result)
except Exception as e:
cause = FlowFailure(task, self, e)
with excutils.save_and_reraise_exception():
try:
self._on_task_error(context, task, e)
except Exception:
LOG.exception("Dropping exception catched when"
" notifying about task failure.")
self.rollback(context, cause)
last_task = 0
was_interrupted = False
if result_fetcher:
self._change_state(context, states.RESUMING)
for (i, task) in enumerate(task_order):
if self.state == states.INTERRUPTED:
was_interrupted = True
break
(has_result, was_error, result) = result_fetcher(self, task)
if not has_result:
break
# Fake running the task so that we trigger the same
# notifications and state changes (and rollback that
# would have happened in a normal flow).
last_task = i + 1
run_task(task, failed=was_error, result=result,
simulate_run=True)
if was_interrupted:
return
self._change_state(context, states.RUNNING)
for task in task_order[last_task:]:
if self.state == states.INTERRUPTED:
was_interrupted = True
break
run_task(task)
if not was_interrupted:
# Only gets here if everything went successfully.
self._change_state(context, states.SUCCESS)