class Workflow(object):
"""
The engine that executes a workflow.
It is a essentially a facility for managing all branches.
A Workflow is also the place that holds the data of a running workflow.
"""
def __init__(self,
workflow_spec,
deserializing=False,
task_class=None,
**kwargs):
"""
Constructor.
:param deserializing: set to true when deserializing to avoid
generating tasks twice (and associated problems with multiple
hierarchies of tasks)
"""
assert workflow_spec is not None
LOG.debug("__init__ Workflow instance: %s" % self.__str__())
self.spec = workflow_spec
self.data = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
if task_class:
self.task_class = task_class
else:
if 'parent' in kwargs:
self.task_class = kwargs['parent'].task_class
else:
self.task_class = Task
if deserializing:
assert 'Root' in workflow_spec.task_specs
root = workflow_spec.task_specs['Root'] # Probably deserialized
else:
if 'Root' in workflow_spec.task_specs:
root = workflow_spec.task_specs['Root']
else:
root = specs.Simple(workflow_spec, 'Root')
self.task_tree = self.task_class(self, root)
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE)
self.spec.start._predict(start)
if 'parent' not in kwargs:
start.task_spec._update_state(start)
#start.dump()
def is_completed(self):
"""
Returns True if the entire Workflow is completed, False otherwise.
"""
mask = Task.NOT_FINISHED_MASK
iter = Task.Iterator(self.task_tree, mask)
try:
iter.next()
except:
# No waiting tasks found.
return True
return False
def has_failed(self):
mask = Task.FAILED
iter = Task.Iterator(self.task_tree, mask)
try:
iter.next()
return True
except:
return False
def _get_waiting_tasks(self):
waiting = Task.Iterator(self.task_tree, Task.WAITING)
return [w for w in waiting]
def _task_completed_notify(self, task):
# Commented cause Workflow and Tasks data are separated now
#if task.get_name() == 'End':
# self.data.update(task.data)
#self.data.update(task.data) # Update workflow data with each completed task's data
#print '_task_completed_notify. task: %s, n_subscribers: %s, is_completed: %s' % (
# task, self.completed_event.n_subscribers(), self.is_completed())
#LOG.debug('data(%s): %s', self.spec.name, self.data)
# Update the state of every WAITING task.
for thetask in self._get_waiting_tasks():
thetask.task_spec._update_state(thetask)
if self.completed_event.n_subscribers() == 0:
# Since is_completed() is expensive it makes sense to bail
# out if calling it is not necessary.
return
if self.is_completed():
self.completed_event(self)
def _get_mutex(self, name):
if name not in self.locks:
self.locks[name] = mutex()
return self.locks[name]
def get_data(self, name, default=None):
"""
Returns the value of the data field with the given name, or the given
default value if the data field does not exist.
:type name: string
:param name: A data field name.
:type default: obj
:param default: Return this value if the data field does not exist.
:rtype: obj
:returns: The value of the data field.
"""
return self.data.get(name, default)
def get_name(self):
return self.spec.name
def cancel(self, success=False):
"""
Cancels all open tasks in the workflow.
:type success: boolean
:param success: Whether the Workflow should be marked as successfully
completed.
"""
self.success = success
cancel = []
mask = Task.NOT_FINISHED_MASK
for task in Task.Iterator(self.task_tree, mask):
cancel.append(task)
for task in cancel:
task.cancel()
def get_task_spec_from_name(self, name):
"""
Returns the task spec with the given name.
:type name: string
:param name: The name of the task.
:rtype: TaskSpec
:returns: The task spec with the given name.
"""
return self.spec.get_task_spec_from_name(name)
def get_task(self, id):
"""
Returns the task with the given id.
:type id:integer
:param id: The id of a task.
:rtype: Task
:returns: The task with the given id.
"""
tasks = [task for task in self.get_tasks() if task.id == id]
return tasks[0] if len(tasks) == 1 else None
def get_tasks_from_spec_name(self, name):
"""
Returns all tasks whose spec has the given name.
@type name: str
@param name: The name of a task spec.
@rtype: Task
@return: The task that relates to the spec with the given name.
"""
return [
task for task in self.get_tasks() if task.task_spec.name == name
]
def get_tasks(self, state=Task.ANY_MASK):
"""
Returns a list of Task objects with the given state.
:type state: integer
:param state: A bitmask of states.
:rtype: list[Task]
:returns: A list of tasks.
"""
return [t for t in Task.Iterator(self.task_tree, state)]
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
:type task_id: integer
:param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def complete_next(self, pick_up=True):
"""
Runs the next task.
Returns True if completed, False otherwise.
:type pick_up: boolean
:param pick_up: When True, this method attempts to choose the next
task not by searching beginning at the root, but by
searching from the position at which the last call
of complete_next() left off.
:rtype: boolean
:returns: True if all tasks were completed, False otherwise.
"""
# Try to pick up where we left off.
blacklist = []
if pick_up and self.last_task is not None:
try:
iter = Task.Iterator(self.last_task, Task.READY)
next = iter.next()
except StopIteration:
next = None
self.last_task = None
if next is not None:
if next.complete():
self.last_task = next
return True
blacklist.append(next)
# Walk through all ready tasks.
for task in Task.Iterator(self.task_tree, Task.READY):
for blacklisted_task in blacklist:
if task._is_descendant_of(blacklisted_task):
continue
if task.complete():
self.last_task = task
return True
blacklist.append(task)
# Walk through all waiting tasks.
for task in Task.Iterator(self.task_tree, Task.WAITING):
task.task_spec._update_state(task)
if not task._has_state(Task.WAITING):
self.last_task = task
# XXX: without this children of WAITING celery task never predicted
task._sync_children(task.task_spec.outputs)
return True
return False
def complete_all(self, pick_up=True):
"""
Runs all branches until completion. This is a convenience wrapper
around complete_next(), and the pick_up argument is passed along.
:type pick_up: boolean
:param pick_up: Passed on to each call of complete_next().
"""
while self.complete_next(pick_up):
pass
def get_dump(self):
"""
Returns a complete dump of the current internal task tree for
debugging.
:rtype: string
:returns: The debug information.
"""
return self.task_tree.get_dump()
def dump(self):
"""
Like get_dump(), but prints the output to the terminal instead of
returning it.
"""
print self.task_tree.dump()
def serialize(self, serializer, **kwargs):
"""
Serializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: object
:returns: The serialized workflow.
"""
return serializer.serialize_workflow(self, **kwargs)
@classmethod
def deserialize(cls, serializer, s_state, **kwargs):
"""
Deserializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type s_state: object
:param s_state: The serialized workflow.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: Workflow
:returns: The workflow instance.
"""
return serializer.deserialize_workflow(s_state, **kwargs)
class Workflow(object):
"""
The engine that executes a workflow.
It is a essentially a facility for managing all branches.
A Workflow is also the place that holds the attributes of a running workflow.
"""
def __init__(self, workflow_spec, **kwargs):
"""
Constructor.
"""
assert workflow_spec is not None
self.spec = workflow_spec
self.task_id_assigner = TaskIdAssigner()
self.attributes = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
self.task_tree = Task(self, specs.Simple(workflow_spec, 'Root'))
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start)
self.spec.start._predict(start)
if not kwargs.has_key('parent'):
start.task_spec._update_state(start)
#start.dump()
def is_completed(self):
"""
Returns True if the entire Workflow is completed, False otherwise.
"""
mask = Task.NOT_FINISHED_MASK
iter = Task.Iterator(self.task_tree, mask)
try:
next = iter.next()
except:
# No waiting tasks found.
return True
return False
def _get_waiting_tasks(self):
waiting = Task.Iterator(self.task_tree, Task.WAITING)
return [w for w in waiting]
def _task_completed_notify(self, task):
if task.get_name() == 'End':
self.attributes.update(task.get_attributes())
# Update the state of every WAITING task.
for thetask in self._get_waiting_tasks():
thetask.task_spec._update_state(thetask)
if self.completed_event.n_subscribers() == 0:
# Since is_completed() is expensive it makes sense to bail
# out if calling it is not necessary.
return
if self.is_completed():
self.completed_event(self)
def _get_mutex(self, name):
if not self.locks.has_key(name):
self.locks[name] = mutex()
return self.locks[name]
def get_attribute(self, name, default=None):
"""
Returns the value of the attribute with the given name, or the given
default value if the attribute does not exist.
@type name: string
@param name: An attribute name.
@type default: obj
@param default: Return this value if the attribute does not exist.
@rtype: obj
@return: The value of the attribute.
"""
return self.attributes.get(name, default)
def cancel(self, success=False):
"""
Cancels all open tasks in the workflow.
@type success: boolean
@param success: Whether the Workflow should be marked as successfully
completed.
"""
self.success = success
cancel = []
mask = Task.NOT_FINISHED_MASK
for task in Task.Iterator(self.task_tree, mask):
cancel.append(task)
for task in cancel:
task.cancel()
def get_task_spec_from_name(self, name):
"""
Returns the task spec with the given name.
@type name: string
@param name: The name of the task.
@rtype: TaskSpec
@return: The task spec with the given name.
"""
return self.spec.get_task_spec_from_name(name)
def get_task(self, id):
"""
Returns the task with the given id.
@type id:integer
@param id: The id of a state.
@rtype: Task
@return: The task with the given id.
"""
tasks = [task for task in self.get_tasks() if task.id == id]
return tasks[0] if len(tasks) == 1 else None
def get_tasks(self, state=Task.ANY_MASK):
"""
Returns a list of Task objects with the given state.
@type state: integer
@param state: A bitmask of states.
@rtype: list[Task]
@return: A list of tasks.
"""
return [t for t in Task.Iterator(self.task_tree, state)]
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
@type task_id: integer
@param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def complete_next(self, pick_up=True):
"""
Runs the next task.
Returns True if completed, False otherwise.
@type pick_up: boolean
@param pick_up: When True, this method attempts to choose the next
task not by searching beginning at the root, but by
searching from the position at which the last call
of complete_next() left off.
@rtype: boolean
@return: True if all tasks were completed, False otherwise.
"""
# Try to pick up where we left off.
blacklist = []
if pick_up and self.last_task is not None:
try:
iter = Task.Iterator(self.last_task, Task.READY)
next = iter.next()
except:
next = None
self.last_task = None
if next is not None:
if next.complete():
self.last_task = next
return True
blacklist.append(next)
# Walk through all waiting tasks.
for task in Task.Iterator(self.task_tree, Task.READY):
for blacklisted_task in blacklist:
if task._is_descendant_of(blacklisted_task):
continue
if task.complete():
self.last_task = task
return True
blacklist.append(task)
return False
def complete_all(self, pick_up=True):
"""
Runs all branches until completion. This is a convenience wrapper
around complete_next(), and the pick_up argument is passed along.
@type pick_up: boolean
@param pick_up: Passed on to each call of complete_next().
"""
while self.complete_next(pick_up):
pass
def get_dump(self):
"""
Returns a complete dump of the current internal task tree for
debugging.
@rtype: string
@return: The debug information.
"""
return self.task_tree.get_dump()
def dump(self):
"""
Like get_dump(), but prints the output to the terminal instead of
returning it.
"""
return self.task_tree.dump()
def serialize(self, serializer):
"""
Serializes a Workflow instance using the provided serializer.
"""
return serializer.serialize_workflow(self)
@classmethod
def deserialize(cls, serializer, s_state):
"""
Deserializes a Workflow instance using the provided serializer.
"""
return serializer.deserialize_workflow(s_state)
class Workflow(object):
"""
The engine that executes a workflow.
It is a essentially a facility for managing all branches.
A Workflow is also the place that holds the attributes of a running workflow.
"""
def __init__(self, workflow_spec, deserializing=False, **kwargs):
"""
Constructor.
:param deserializing: set to true when deserializing to avoid
generating tasks twice (and associated problems with multiple
hierarchies of tasks)
"""
assert workflow_spec is not None
LOG.debug("__init__ Workflow instance: %s" % self.__str__())
self.spec = workflow_spec
self.task_id_assigner = TaskIdAssigner()
self.attributes = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
if deserializing:
assert 'Root' in workflow_spec.task_specs
root = workflow_spec.task_specs['Root'] # Probably deserialized
else:
if 'Root' in workflow_spec.task_specs:
root = workflow_spec.task_specs['Root']
else:
root = specs.Simple(workflow_spec, 'Root')
self.task_tree = Task(self, root)
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE)
self.spec.start._predict(start)
if 'parent' not in kwargs:
start.task_spec._update_state(start)
#start.dump()
def is_completed(self):
"""
Returns True if the entire Workflow is completed, False otherwise.
"""
mask = Task.NOT_FINISHED_MASK
iter = Task.Iterator(self.task_tree, mask)
try:
iter.next()
except:
# No waiting tasks found.
return True
return False
def _get_waiting_tasks(self):
waiting = Task.Iterator(self.task_tree, Task.WAITING)
return [w for w in waiting]
def _task_completed_notify(self, task):
if task.get_name() == 'End':
self.attributes.update(task.get_attributes())
# Update the state of every WAITING task.
for thetask in self._get_waiting_tasks():
thetask.task_spec._update_state(thetask)
if self.completed_event.n_subscribers() == 0:
# Since is_completed() is expensive it makes sense to bail
# out if calling it is not necessary.
return
if self.is_completed():
self.completed_event(self)
def _get_mutex(self, name):
if name not in self.locks:
self.locks[name] = mutex()
return self.locks[name]
def get_attribute(self, name, default=None):
"""
Returns the value of the attribute with the given name, or the given
default value if the attribute does not exist.
:type name: string
:param name: An attribute name.
:type default: obj
:param default: Return this value if the attribute does not exist.
:rtype: obj
:returns: The value of the attribute.
"""
return self.attributes.get(name, default)
def cancel(self, success=False):
"""
Cancels all open tasks in the workflow.
:type success: boolean
:param success: Whether the Workflow should be marked as successfully
completed.
"""
self.success = success
cancel = []
mask = Task.NOT_FINISHED_MASK
for task in Task.Iterator(self.task_tree, mask):
cancel.append(task)
for task in cancel:
task.cancel()
def get_task_spec_from_name(self, name):
"""
Returns the task spec with the given name.
:type name: string
:param name: The name of the task.
:rtype: TaskSpec
:returns: The task spec with the given name.
"""
return self.spec.get_task_spec_from_name(name)
def get_task(self, id):
"""
Returns the task with the given id.
:type id:integer
:param id: The id of a task.
:rtype: Task
:returns: The task with the given id.
"""
tasks = [task for task in self.get_tasks() if task.id == id]
return tasks[0] if len(tasks) == 1 else None
def get_tasks(self, state=Task.ANY_MASK):
"""
Returns a list of Task objects with the given state.
:type state: integer
:param state: A bitmask of states.
:rtype: list[Task]
:returns: A list of tasks.
"""
return [t for t in Task.Iterator(self.task_tree, state)]
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
:type task_id: integer
:param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def complete_next(self, pick_up=True):
"""
Runs the next task.
Returns True if completed, False otherwise.
:type pick_up: boolean
:param pick_up: When True, this method attempts to choose the next
task not by searching beginning at the root, but by
searching from the position at which the last call
of complete_next() left off.
:rtype: boolean
:returns: True if all tasks were completed, False otherwise.
"""
# Try to pick up where we left off.
blacklist = []
if pick_up and self.last_task is not None:
try:
iter = Task.Iterator(self.last_task, Task.READY)
next = iter.next()
except:
next = None
self.last_task = None
if next is not None:
if next.complete():
self.last_task = next
return True
blacklist.append(next)
# Walk through all ready tasks.
for task in Task.Iterator(self.task_tree, Task.READY):
for blacklisted_task in blacklist:
if task._is_descendant_of(blacklisted_task):
continue
if task.complete():
self.last_task = task
return True
blacklist.append(task)
# Walk through all waiting tasks.
for task in Task.Iterator(self.task_tree, Task.WAITING):
task.task_spec._update_state(task)
if not task._has_state(Task.WAITING):
self.last_task = task
return True
return False
def complete_all(self, pick_up=True):
"""
Runs all branches until completion. This is a convenience wrapper
around complete_next(), and the pick_up argument is passed along.
:type pick_up: boolean
:param pick_up: Passed on to each call of complete_next().
"""
while self.complete_next(pick_up):
pass
def get_dump(self):
"""
Returns a complete dump of the current internal task tree for
debugging.
:rtype: string
:returns: The debug information.
"""
return self.task_tree.get_dump()
def dump(self):
"""
Like get_dump(), but prints the output to the terminal instead of
returning it.
"""
print self.task_tree.dump()
def serialize(self, serializer, **kwargs):
"""
Serializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: object
:returns: The serialized workflow.
"""
return serializer.serialize_workflow(self, **kwargs)
@classmethod
def deserialize(cls, serializer, s_state, **kwargs):
"""
Deserializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type s_state: object
:param s_state: The serialized workflow.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: Workflow
:returns: The workflow instance.
"""
return serializer.deserialize_workflow(s_state, **kwargs)
