def __init__(self, workflow_spec, **kwargs):
"""
Constructor.
"""
assert workflow_spec is not None
self.spec = workflow_spec
self.task_id_assigner = TaskIdAssigner()
self.attributes = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
self.task_tree = Task(self, specs.Simple(workflow_spec, 'Root'))
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start)
self.spec.start._predict(start)
if not kwargs.has_key('parent'):
start.task_spec._update_state(start)
def __init__(self, parent, name, **kwargs):
"""
Constructor. May also have properties/attributes passed.
The difference between the assignment of a property using
property_assign versus pre_assign and post_assign is that
changes made using property_assign are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are properties that, once defined, can no
longer be modified.
@type parent: L{SpiffWorkflow.specs.WorkflowSpec}
@param parent: A reference to the parent (usually a workflow).
@type name: string
@param name: A name for the task.
@type lock: list(str)
@keyword lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
@type property_assign: list((str, object))
@keyword property_assign: a list of name/value pairs
@type pre_assign: list((str, object))
@keyword pre_assign: a list of name/value pairs
@type post_assign: list((str, object))
@keyword post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = str(name)
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.cancelled = False
self.properties = kwargs.get('properties', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self._parent._add_notify(self)
self.properties.update(self.defines)
assert self.id is not None
def __init__(self,
workflow_spec,
deserializing=False,
task_class=None,
**kwargs):
"""
Constructor.
:param deserializing: set to true when deserializing to avoid
generating tasks twice (and associated problems with multiple
hierarchies of tasks)
"""
assert workflow_spec is not None
LOG.debug("__init__ Workflow instance: %s" % self.__str__())
self.spec = workflow_spec
self.data = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
if task_class:
self.task_class = task_class
else:
if 'parent' in kwargs:
self.task_class = kwargs['parent'].task_class
else:
self.task_class = Task
if deserializing:
assert 'Root' in workflow_spec.task_specs
root = workflow_spec.task_specs['Root'] # Probably deserialized
else:
if 'Root' in workflow_spec.task_specs:
root = workflow_spec.task_specs['Root']
else:
root = specs.Simple(workflow_spec, 'Root')
self.task_tree = self.task_class(self, root)
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE)
self.spec.start._predict(start)
if 'parent' not in kwargs:
start.task_spec._update_state(start)
def __init__(self, workflow_spec, **kwargs):
"""
Constructor.
"""
assert workflow_spec is not None
self.spec = workflow_spec
self.task_id_assigner = TaskIdAssigner()
self.attributes = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
self.task_tree = Task(self, specs.Simple(workflow_spec, 'Root'))
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start)
self.spec.start._predict(start)
if not kwargs.has_key('parent'):
start.task_spec._update_state(start)
def __init__(self, parent, name, **kwargs):
"""
Constructor. May also have properties/attributes passed.
The difference between the assignment of a property using
property_assign versus pre_assign and post_assign is that
changes made using property_assign are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are properties that, once defined, can no
longer be modified.
@type parent: L{SpiffWorkflow.specs.WorkflowSpec}
@param parent: A reference to the parent (usually a workflow).
@type name: string
@param name: A name for the task.
@type lock: list(str)
@keyword lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
@type property_assign: list((str, object))
@keyword property_assign: a list of name/value pairs
@type pre_assign: list((str, object))
@keyword pre_assign: a list of name/value pairs
@type post_assign: list((str, object))
@keyword post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = str(name)
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.cancelled = False
self.properties = kwargs.get('properties', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self._parent._add_notify(self)
self.properties.update(self.defines)
assert self.id is not None
def __init__(self, parent, name, **kwargs):
"""
Constructor.
The difference between the assignment of a data value using
the data argument versus pre_assign and post_assign is that
changes made using data are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are spec data fields that, once defined, can
no longer be modified.
:type parent: L{SpiffWorkflow.specs.WorkflowSpec}
:param parent: A reference to the parent (usually a workflow).
:type name: string
:param name: A name for the task.
:type lock: list(str)
:param lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
:type data: dict((str, object))
:param data: name/value pairs
:type defines: dict((str, object))
:param defines: name/value pairs
:type pre_assign: list((str, object))
:param pre_assign: a list of name/value pairs
:type post_assign: list((str, object))
:param post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = name
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.data = kwargs.get('data', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self.cancelled_event = Event()
self.finished_event = Event()
# Error handling
self.error_handlers = []
self._parent._add_notify(self)
self.data.update(self.defines)
assert self.id is not None
def __init__(self, workflow_spec, deserializing=False, task_class=None, **kwargs):
"""
Constructor.
:param deserializing: set to true when deserializing to avoid
generating tasks twice (and associated problems with multiple
hierarchies of tasks)
"""
assert workflow_spec is not None
LOG.debug("__init__ Workflow instance: %s" % self.__str__())
self.spec = workflow_spec
self.data = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
if task_class:
self.task_class = task_class
else:
if 'parent' in kwargs:
self.task_class = kwargs['parent'].task_class
else:
self.task_class = Task
if deserializing:
assert 'Root' in workflow_spec.task_specs
root = workflow_spec.task_specs['Root'] # Probably deserialized
else:
if 'Root' in workflow_spec.task_specs:
root = workflow_spec.task_specs['Root']
else:
root = specs.Simple(workflow_spec, 'Root')
self.task_tree = self.task_class(self, root)
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE)
self.spec.start._predict(start)
if 'parent' not in kwargs:
start.task_spec._update_state(start)
class TaskSpec(object):
"""
This class implements an abstract base type for all tasks.
Tasks provide the following signals:
- B{entered}: called when the state changes to READY or WAITING, at a
time where properties are not yet initialized.
- B{reached}: called when the state changes to READY or WAITING, at a
time where properties are already initialized using property_assign
and pre-assign.
- B{ready}: called when the state changes to READY, at a time where
properties are already initialized using property_assign and
pre-assign.
- B{completed}: called when the state changes to COMPLETED, at a time
before the post-assign variables are assigned.
- B{cancelled}: called when the state changes to CANCELLED, at a time
before the post-assign variables are assigned.
- B{finished}: called when the state changes to COMPLETED or CANCELLED,
at the last possible time and after the post-assign variables are
assigned.
"""
def __init__(self, parent, name, **kwargs):
"""
Constructor. May also have properties/attributes passed.
The difference between the assignment of a property using
property_assign versus pre_assign and post_assign is that
changes made using property_assign are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are properties that, once defined, can no
longer be modified.
@type parent: L{SpiffWorkflow.specs.WorkflowSpec}
@param parent: A reference to the parent (usually a workflow).
@type name: string
@param name: A name for the task.
@type lock: list(str)
@keyword lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
@type property_assign: list((str, object))
@keyword property_assign: a list of name/value pairs
@type pre_assign: list((str, object))
@keyword pre_assign: a list of name/value pairs
@type post_assign: list((str, object))
@keyword post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = str(name)
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.cancelled = False
self.properties = kwargs.get('properties', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self._parent._add_notify(self)
self.properties.update(self.defines)
assert self.id is not None
def _connect_notify(self, taskspec):
"""
Called by the previous task to let us know that it exists.
@type taskspec: TaskSpec
@param taskspec: The task by which this method is executed.
"""
self.inputs.append(taskspec)
def _get_activated_tasks(self, my_task, destination):
"""
Returns the list of tasks that were activated in the previous
call of execute(). Only returns tasks that point towards the
destination task, i.e. those which have destination as a
descendant.
@type my_task: Task
@param my_task: The associated task in the task tree.
@type destination: Task
@param destination: The destination task.
"""
return my_task.children
def _get_activated_threads(self, my_task):
"""
Returns the list of threads that were activated in the previous
call of execute().
@type my_task: Task
@param my_task: The associated task in the task tree.
"""
return my_task.children
def set_property(self, **kwargs):
"""
Defines the given property name/value pairs.
"""
for key in kwargs:
if self.defines.has_key(key):
msg = "Property %s can not be modified" % key
raise Exception.WorkflowException(msg)
self.properties.update(kwargs)
def get_property(self, name, default = None):
"""
Returns the value of the property with the given name, or the given
default value if the property does not exist.
@type name: string
@param name: A property name.
@type default: string
@param default: This value is returned if the property does not exist.
"""
return self.properties.get(name, default)
def connect(self, taskspec):
"""
Connect the *following* task to this one. In other words, the
given task is added as an output task.
@type taskspec: TaskSpec
@param taskspec: The new output task.
"""
self.outputs.append(taskspec)
taskspec._connect_notify(self)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise Exception.WorkflowException(self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise Exception.WorkflowException(self, 'No input task connected.')
def _predict(self, my_task, seen = None, looked_ahead = 0):
"""
Updates the branch such that all possible future routes are added
with the LIKELY flag.
Should NOT be overwritten! Instead, overwrite the hook (_predict_hook).
@type my_task: Task
@param my_task: The associated task in the task tree.
@type seen: list[taskspec]
@param seen: A list of already visited tasks.
@type looked_ahead: integer
@param looked_ahead: The depth of the predicted path so far.
"""
if seen is None:
seen = []
elif self in seen:
return
if not my_task._is_definite():
seen.append(self)
if my_task._has_state(Task.MAYBE):
looked_ahead += 1
if looked_ahead >= self.lookahead:
return
if not my_task._is_finished():
self._predict_hook(my_task)
for child in my_task.children:
child.task_spec._predict(child, seen[:], looked_ahead)
def _predict_hook(self, my_task):
if my_task._is_definite():
child_state = Task.FUTURE
else:
child_state = Task.LIKELY
my_task._update_children(self.outputs, child_state)
def _update_state(self, my_task):
my_task._inherit_attributes()
if not self._update_state_hook(my_task):
return
self.entered_event.emit(my_task.workflow, my_task)
my_task._ready()
def _update_state_hook(self, my_task):
was_predicted = my_task._is_predicted()
if not my_task.parent._is_finished():
my_task.state = Task.FUTURE
if was_predicted:
self._predict(my_task)
if my_task.parent._is_finished():
return True
return False
def _on_ready(self, my_task):
"""
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
assert not self.cancelled
self.test()
# Acquire locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
if not mutex.testandset():
return False
# Assign variables, if so requested.
for assignment in self.pre_assign:
assignment.assign(my_task, my_task)
# Run task-specific code.
result = self._on_ready_before_hook(my_task)
self.reached_event.emit(my_task.workflow, my_task)
if result:
result = self._on_ready_hook(my_task)
# Run user code, if any.
if result:
result = self.ready_event.emit(my_task.workflow, my_task)
if result:
# Assign variables, if so requested.
for assignment in self.post_assign:
assignment.assign(my_task, my_task)
# Release locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
mutex.unlock()
return result
def _on_ready_before_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_ready_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_cancel(self, my_task):
"""
May be called by another task to cancel the operation before it was
completed.
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_trigger(self, my_task):
"""
May be called by another task to trigger a task-specific
event.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
raise NotImplementedError("Trigger not supported by this task.")
def _on_complete(self, my_task):
"""
Return True on success, False otherwise. Should not be overwritten,
overwrite _on_complete_hook() instead.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
assert not self.cancelled
if my_task.workflow.debug:
print "Executing task:", my_task.get_name()
if not self._on_complete_hook(my_task):
return False
# Notify the Workflow.
my_task.workflow._task_completed_notify(my_task)
if my_task.workflow.debug:
my_task.workflow.outer_job.task_tree.dump()
self.completed_event.emit(my_task.workflow, my_task)
return True
def _on_complete_hook(self, my_task):
"""
A hook into _on_complete() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: bool
@return: True on success, False otherwise.
"""
# If we have more than one output, implicitly split.
my_task._update_children(self.outputs)
return True
class Workflow(object):
"""
The engine that executes a workflow.
It is a essentially a facility for managing all branches.
A Workflow is also the place that holds the attributes of a running workflow.
"""
def __init__(self, workflow_spec, **kwargs):
"""
Constructor.
"""
assert workflow_spec is not None
self.spec = workflow_spec
self.task_id_assigner = TaskIdAssigner()
self.attributes = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
self.task_tree = Task(self, specs.Simple(workflow_spec, 'Root'))
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start)
self.spec.start._predict(start)
if not kwargs.has_key('parent'):
start.task_spec._update_state(start)
#start.dump()
def is_completed(self):
"""
Returns True if the entire Workflow is completed, False otherwise.
"""
mask = Task.NOT_FINISHED_MASK
iter = Task.Iterator(self.task_tree, mask)
try:
next = iter.next()
except:
# No waiting tasks found.
return True
return False
def _get_waiting_tasks(self):
waiting = Task.Iterator(self.task_tree, Task.WAITING)
return [w for w in waiting]
def _task_completed_notify(self, task):
if task.get_name() == 'End':
self.attributes.update(task.get_attributes())
# Update the state of every WAITING task.
for thetask in self._get_waiting_tasks():
thetask.task_spec._update_state(thetask)
if self.completed_event.n_subscribers() == 0:
# Since is_completed() is expensive it makes sense to bail
# out if calling it is not necessary.
return
if self.is_completed():
self.completed_event(self)
def _get_mutex(self, name):
if not self.locks.has_key(name):
self.locks[name] = mutex()
return self.locks[name]
def get_attribute(self, name, default=None):
"""
Returns the value of the attribute with the given name, or the given
default value if the attribute does not exist.
@type name: string
@param name: An attribute name.
@type default: obj
@param default: Return this value if the attribute does not exist.
@rtype: obj
@return: The value of the attribute.
"""
return self.attributes.get(name, default)
def cancel(self, success=False):
"""
Cancels all open tasks in the workflow.
@type success: boolean
@param success: Whether the Workflow should be marked as successfully
completed.
"""
self.success = success
cancel = []
mask = Task.NOT_FINISHED_MASK
for task in Task.Iterator(self.task_tree, mask):
cancel.append(task)
for task in cancel:
task.cancel()
def get_task_spec_from_name(self, name):
"""
Returns the task spec with the given name.
@type name: string
@param name: The name of the task.
@rtype: TaskSpec
@return: The task spec with the given name.
"""
return self.spec.get_task_spec_from_name(name)
def get_task(self, id):
"""
Returns the task with the given id.
@type id:integer
@param id: The id of a state.
@rtype: Task
@return: The task with the given id.
"""
tasks = [task for task in self.get_tasks() if task.id == id]
return tasks[0] if len(tasks) == 1 else None
def get_tasks(self, state=Task.ANY_MASK):
"""
Returns a list of Task objects with the given state.
@type state: integer
@param state: A bitmask of states.
@rtype: list[Task]
@return: A list of tasks.
"""
return [t for t in Task.Iterator(self.task_tree, state)]
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
@type task_id: integer
@param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def complete_next(self, pick_up=True):
"""
Runs the next task.
Returns True if completed, False otherwise.
@type pick_up: boolean
@param pick_up: When True, this method attempts to choose the next
task not by searching beginning at the root, but by
searching from the position at which the last call
of complete_next() left off.
@rtype: boolean
@return: True if all tasks were completed, False otherwise.
"""
# Try to pick up where we left off.
blacklist = []
if pick_up and self.last_task is not None:
try:
iter = Task.Iterator(self.last_task, Task.READY)
next = iter.next()
except:
next = None
self.last_task = None
if next is not None:
if next.complete():
self.last_task = next
return True
blacklist.append(next)
# Walk through all waiting tasks.
for task in Task.Iterator(self.task_tree, Task.READY):
for blacklisted_task in blacklist:
if task._is_descendant_of(blacklisted_task):
continue
if task.complete():
self.last_task = task
return True
blacklist.append(task)
return False
def complete_all(self, pick_up=True):
"""
Runs all branches until completion. This is a convenience wrapper
around complete_next(), and the pick_up argument is passed along.
@type pick_up: boolean
@param pick_up: Passed on to each call of complete_next().
"""
while self.complete_next(pick_up):
pass
def get_dump(self):
"""
Returns a complete dump of the current internal task tree for
debugging.
@rtype: string
@return: The debug information.
"""
return self.task_tree.get_dump()
def dump(self):
"""
Like get_dump(), but prints the output to the terminal instead of
returning it.
"""
return self.task_tree.dump()
def serialize(self, serializer):
"""
Serializes a Workflow instance using the provided serializer.
"""
return serializer.serialize_workflow(self)
@classmethod
def deserialize(cls, serializer, s_state):
"""
Deserializes a Workflow instance using the provided serializer.
"""
return serializer.deserialize_workflow(s_state)
class TaskSpec(object):
"""
This class implements an abstract base type for all tasks.
Tasks provide the following signals:
- B{entered}: called when the state changes to READY or WAITING, at a
time where properties are not yet initialized.
- B{reached}: called when the state changes to READY or WAITING, at a
time where properties are already initialized using property_assign
and pre-assign.
- B{ready}: called when the state changes to READY, at a time where
properties are already initialized using property_assign and
pre-assign.
- B{completed}: called when the state changes to COMPLETED, at a time
before the post-assign variables are assigned.
- B{cancelled}: called when the state changes to CANCELLED, at a time
before the post-assign variables are assigned.
- B{finished}: called when the state changes to COMPLETED or CANCELLED,
at the last possible time after the post-assign variables are
assigned and mutexes are released.
Event sequence is: entered -> reached -> ready -> completed -> finished
(cancelled may happen at any time)
The only events where implementing something other than state tracking
may be useful are the following:
- Reached: You could mess with the pre-assign variables here, for
example. Other then that, there is probably no need in a real
application.
- Ready: This is where a task could implement custom code, for example
for triggering an external system. This is also the only event where a
return value has a meaning (returning non-True will mean that the
post-assign procedure is skipped.)
"""
def __init__(self, parent, name, **kwargs):
"""
Constructor. May also have properties/attributes passed.
The difference between the assignment of a property using
properties versus pre_assign and post_assign is that
changes made using properties are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are properties that, once defined, can no
longer be modified.
@type parent: L{SpiffWorkflow.specs.WorkflowSpec}
@param parent: A reference to the parent (usually a workflow).
@type name: string
@param name: A name for the task.
@type lock: list(str)
@keyword lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
@type properties: dict((str, object))
@keyword properties: name/value pairs
@type defines: dict((str, object))
@keyword defines: name/value pairs
@type pre_assign: list((str, object))
@keyword pre_assign: a list of name/value pairs
@type post_assign: list((str, object))
@keyword post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
if __debug__:
from SpiffWorkflow.specs import WorkflowSpec # Can't import above
assert isinstance(parent, WorkflowSpec)
self._parent = parent
self.id = None
self.name = str(name)
self.description = kwargs.pop('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.properties = kwargs.pop('properties', {})
self.defines = kwargs.pop('defines', {})
self.pre_assign = kwargs.pop('pre_assign', [])
self.post_assign = kwargs.pop('post_assign', [])
self.locks = kwargs.pop('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self.cancelled_event = Event()
self.finished_event = Event()
self._parent._add_notify(self)
self.properties.update(self.defines)
assert self.id is not None
def _connect_notify(self, taskspec):
"""
Called by the previous task to let us know that it exists.
@type taskspec: TaskSpec
@param taskspec: The task by which this method is executed.
"""
self.inputs.append(taskspec)
def ancestors(self):
"""Returns list of ancestor task specs based on inputs"""
results = []
def recursive_find_ancestors(task, stack):
for input in task.inputs:
if input not in stack:
stack.append(input)
recursive_find_ancestors(input, stack)
recursive_find_ancestors(self, results)
return results
def _get_activated_tasks(self, my_task, destination):
"""
Returns the list of tasks that were activated in the previous
call of execute(). Only returns tasks that point towards the
destination task, i.e. those which have destination as a
descendant.
@type my_task: Task
@param my_task: The associated task in the task tree.
@type destination: Task
@param destination: The destination task.
"""
return my_task.children
def _get_activated_threads(self, my_task):
"""
Returns the list of threads that were activated in the previous
call of execute().
@type my_task: Task
@param my_task: The associated task in the task tree.
"""
return my_task.children
def set_property(self, **kwargs):
"""
Defines the given property name/value pairs.
"""
for key in kwargs:
if key in self.defines:
msg = "Property %s can not be modified" % key
raise WorkflowException(self, msg)
self.properties.update(kwargs)
def get_property(self, name, default=None):
"""
Returns the value of the property with the given name, or the given
default value if the property does not exist.
@type name: string
@param name: A property name.
@type default: string
@param default: This value is returned if the property does not exist.
"""
return self.properties.get(name, default)
def connect(self, taskspec):
"""
Connect the *following* task to this one. In other words, the
given task is added as an output task.
@type taskspec: TaskSpec
@param taskspec: The new output task.
"""
if taskspec not in self.outputs:
self.outputs.append(taskspec)
taskspec._connect_notify(self)
else:
LOG.debug("Attempt to insert '%s' as an output to '%s' when it "
"already exists. Ignorning request" % (taskspec.name,
self.name))
def follow(self, taskspec):
"""
Make this task follow the provided one. In other words, this task is
added to the given task outputs.
This is an alias to connect, just easier to understand when reading
code - ex: my_task.follow(the_other_task)
Adding it after being confused by .connect one times too many!
@type taskspec: TaskSpec
@param taskspec: The task to follow.
"""
taskspec.connect(self)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise WorkflowException(self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise WorkflowException(self, 'No input task connected.')
def _predict(self, my_task, seen=None, looked_ahead=0):
"""
Updates the branch such that all possible future routes are added
with the LIKELY flag.
Should NOT be overwritten! Instead, overwrite the hook (_predict_hook).
@type my_task: Task
@param my_task: The associated task in the task tree.
@type seen: list[taskspec]
@param seen: A list of already visited tasks.
@type looked_ahead: integer
@param looked_ahead: The depth of the predicted path so far.
"""
if seen is None:
seen = []
elif self in seen:
return
if not my_task._is_definite():
seen.append(self)
if my_task._has_state(Task.MAYBE):
looked_ahead += 1
if looked_ahead >= self.lookahead:
return
if not my_task._is_finished():
self._predict_hook(my_task)
for child in my_task.children:
child.task_spec._predict(child, seen[:], looked_ahead)
def _predict_hook(self, my_task):
if my_task._is_definite():
child_state = Task.FUTURE
else:
child_state = Task.LIKELY
my_task._update_children(self.outputs, child_state)
def _update_state(self, my_task):
"""
Called whenever any event happens that may affect the
state of this task in the workflow. For example, if a predecessor
completes it makes sure to call this method so we can react.
"""
my_task._inherit_attributes()
if not self._update_state_hook(my_task):
LOG.debug("_update_state_hook for %s was not positive, so not "
"going to READY state" % my_task.get_name())
return
self.entered_event.emit(my_task.workflow, my_task)
my_task._ready()
def _update_state_hook(self, my_task):
"""
Typically this method should perform the following actions::
- Update the state of the corresponding task.
- Update the predictions for its successors.
Returning non-False will cause the task to go into READY.
Returning any other value will cause no action.
"""
if not my_task.parent._is_finished():
assert my_task.state != Task.WAITING
my_task.state = Task.FUTURE
if my_task._is_predicted():
self._predict(my_task)
LOG.debug("'%s'._update_state_hook says parent (%s, state=%s) "
"is_finished=%s" % (self.name, my_task.parent.get_name(),
my_task.parent.get_state_name(),
my_task.parent._is_finished()))
if my_task.parent._is_finished():
return True
return False
def _on_ready(self, my_task):
"""
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
self.test()
# Acquire locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
if not mutex.testandset():
return False
# Assign variables, if so requested.
for assignment in self.pre_assign:
assignment.assign(my_task, my_task)
# Run task-specific code.
result = self._on_ready_before_hook(my_task)
self.reached_event.emit(my_task.workflow, my_task)
if result:
result = self._on_ready_hook(my_task)
# Run user code, if any.
if result:
result = self.ready_event.emit(my_task.workflow, my_task)
if result:
# Assign variables, if so requested.
for assignment in self.post_assign:
assignment.assign(my_task, my_task)
# Release locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
mutex.unlock()
self.finished_event.emit(my_task.workflow, my_task)
return result
def _on_ready_before_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_ready_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_cancel(self, my_task):
"""
May be called by another task to cancel the operation before it was
completed.
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
self.cancelled_event.emit(my_task.workflow, my_task)
return True
def _on_trigger(self, my_task):
"""
May be called by another task to trigger a task-specific
event.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
raise NotImplementedError("Trigger not supported by this task.")
def _on_complete(self, my_task):
"""
Return True on success, False otherwise. Should not be overwritten,
overwrite _on_complete_hook() instead.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
if my_task.workflow.debug:
print "Executing task:", my_task.get_name()
if not self._on_complete_hook(my_task):
return False
# Notify the Workflow.
my_task.workflow._task_completed_notify(my_task)
if my_task.workflow.debug:
if hasattr(my_task.workflow, "outer_workflow"):
my_task.workflow.outer_workflow.task_tree.dump()
self.completed_event.emit(my_task.workflow, my_task)
return True
def _on_complete_hook(self, my_task):
"""
A hook into _on_complete() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: bool
@return: True on success, False otherwise.
"""
# If we have more than one output, implicitly split.
my_task._update_children(self.outputs)
return True
def serialize(self, serializer, **kwargs):
"""
Serializes the instance using the provided serializer.
@note: The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
@type serializer: L{SpiffWorkflow.storage.Serializer}
@param serializer: The serializer to use.
@type kwargs: dict
@param kwargs: Passed to the serializer.
@rtype: object
@return: The serialized object.
"""
return serializer._serialize_task_spec(self, **kwargs)
@classmethod
def deserialize(cls, serializer, wf_spec, s_state, **kwargs):
"""
Deserializes the instance using the provided serializer.
@note: The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
@type serializer: L{SpiffWorkflow.storage.Serializer}
@param serializer: The serializer to use.
@type wf_spec: L{SpiffWorkflow.spec.WorkflowSpec}
@param wf_spec: An instance of the WorkflowSpec.
@type s_state: object
@param s_state: The serialized task specification object.
@type kwargs: dict
@param kwargs: Passed to the serializer.
@rtype: TaskSpec
@return: The task specification instance.
"""
return serializer._deserialize_task_spec(wf_spec, s_state,
cls(wf_spec, s_state['name']), **kwargs)
class TaskSpec(object):
"""
This class implements an abstract base type for all tasks.
Tasks provide the following signals:
- B{entered}: called when the state changes to READY or WAITING, at a
time where properties are not yet initialized.
- B{reached}: called when the state changes to READY or WAITING, at a
time where properties are already initialized using property_assign
and pre-assign.
- B{ready}: called when the state changes to READY, at a time where
properties are already initialized using property_assign and
pre-assign.
- B{completed}: called when the state changes to COMPLETED, at a time
before the post-assign variables are assigned.
- B{cancelled}: called when the state changes to CANCELLED, at a time
before the post-assign variables are assigned.
- B{finished}: called when the state changes to COMPLETED or CANCELLED,
at the last possible time after the post-assign variables are
assigned and mutexes are released.
Event sequence is: entered -> reached -> ready -> completed -> finished
(cancelled may happen at any time)
The only events where implementing something other than state tracking
may be useful are the following:
- Reached: You could mess with the pre-assign variables here, for
example. Other then that, there is probably no need in a real
application.
- Ready: This is where a task could implement custom code, for example
for triggering an external system. This is also the only event where a
return value has a meaning (returning non-True will mean that the
post-assign procedure is skipped.)
"""
def __init__(self, parent, name, **kwargs):
"""
Constructor. May also have properties/attributes passed.
The difference between the assignment of a property using
properties versus pre_assign and post_assign is that
changes made using properties are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are properties that, once defined, can no
longer be modified.
@type parent: L{SpiffWorkflow.specs.WorkflowSpec}
@param parent: A reference to the parent (usually a workflow).
@type name: string
@param name: A name for the task.
@type lock: list(str)
@keyword lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
@type properties: dict((str, object))
@keyword properties: name/value pairs
@type defines: dict((str, object))
@keyword defines: name/value pairs
@type pre_assign: list((str, object))
@keyword pre_assign: a list of name/value pairs
@type post_assign: list((str, object))
@keyword post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
if __debug__:
from SpiffWorkflow.specs import WorkflowSpec # Can't import above
assert isinstance(parent, WorkflowSpec)
self._parent = parent
self.id = None
self.name = str(name)
self.description = kwargs.pop('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.properties = kwargs.pop('properties', {})
self.defines = kwargs.pop('defines', {})
self.pre_assign = kwargs.pop('pre_assign', [])
self.post_assign = kwargs.pop('post_assign', [])
self.locks = kwargs.pop('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self.cancelled_event = Event()
self.finished_event = Event()
self._parent._add_notify(self)
self.properties.update(self.defines)
assert self.id is not None
def _connect_notify(self, taskspec):
"""
Called by the previous task to let us know that it exists.
@type taskspec: TaskSpec
@param taskspec: The task by which this method is executed.
"""
self.inputs.append(taskspec)
def ancestors(self):
"""Returns list of ancestor task specs based on inputs"""
results = []
def recursive_find_ancestors(task, stack):
for input in task.inputs:
if input not in stack:
stack.append(input)
recursive_find_ancestors(input, stack)
recursive_find_ancestors(self, results)
return results
def _get_activated_tasks(self, my_task, destination):
"""
Returns the list of tasks that were activated in the previous
call of execute(). Only returns tasks that point towards the
destination task, i.e. those which have destination as a
descendant.
@type my_task: Task
@param my_task: The associated task in the task tree.
@type destination: Task
@param destination: The destination task.
"""
return my_task.children
def _get_activated_threads(self, my_task):
"""
Returns the list of threads that were activated in the previous
call of execute().
@type my_task: Task
@param my_task: The associated task in the task tree.
"""
return my_task.children
def set_property(self, **kwargs):
"""
Defines the given property name/value pairs.
"""
for key in kwargs:
if key in self.defines:
msg = "Property %s can not be modified" % key
raise WorkflowException(self, msg)
self.properties.update(kwargs)
def get_property(self, name, default=None):
"""
Returns the value of the property with the given name, or the given
default value if the property does not exist.
@type name: string
@param name: A property name.
@type default: string
@param default: This value is returned if the property does not exist.
"""
return self.properties.get(name, default)
def connect(self, taskspec):
"""
Connect the *following* task to this one. In other words, the
given task is added as an output task.
@type taskspec: TaskSpec
@param taskspec: The new output task.
"""
if taskspec not in self.outputs:
self.outputs.append(taskspec)
taskspec._connect_notify(self)
else:
LOG.debug("Attempt to insert '%s' as an output to '%s' when it "
"already exists. Ignorning request" %
(taskspec.name, self.name))
def follow(self, taskspec):
"""
Make this task follow the provided one. In other words, this task is
added to the given task outputs.
This is an alias to connect, just easier to understand when reading
code - ex: my_task.follow(the_other_task)
Adding it after being confused by .connect one times too many!
@type taskspec: TaskSpec
@param taskspec: The task to follow.
"""
taskspec.connect(self)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise WorkflowException(self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise WorkflowException(self, 'No input task connected.')
def _predict(self, my_task, seen=None, looked_ahead=0):
"""
Updates the branch such that all possible future routes are added
with the LIKELY flag.
Should NOT be overwritten! Instead, overwrite the hook (_predict_hook).
@type my_task: Task
@param my_task: The associated task in the task tree.
@type seen: list[taskspec]
@param seen: A list of already visited tasks.
@type looked_ahead: integer
@param looked_ahead: The depth of the predicted path so far.
"""
if seen is None:
seen = []
elif self in seen:
return
if not my_task._is_definite():
seen.append(self)
if my_task._has_state(Task.MAYBE):
looked_ahead += 1
if looked_ahead >= self.lookahead:
return
if not my_task._is_finished():
self._predict_hook(my_task)
for child in my_task.children:
child.task_spec._predict(child, seen[:], looked_ahead)
def _predict_hook(self, my_task):
if my_task._is_definite():
child_state = Task.FUTURE
else:
child_state = Task.LIKELY
my_task._update_children(self.outputs, child_state)
def _update_state(self, my_task):
"""
Called whenever any event happens that may affect the
state of this task in the workflow. For example, if a predecessor
completes it makes sure to call this method so we can react.
"""
my_task._inherit_attributes()
if not self._update_state_hook(my_task):
LOG.debug("_update_state_hook for %s was not positive, so not "
"going to READY state" % my_task.get_name())
return
self.entered_event.emit(my_task.workflow, my_task)
my_task._ready()
def _update_state_hook(self, my_task):
"""
Typically this method should perform the following actions::
- Update the state of the corresponding task.
- Update the predictions for its successors.
Returning non-False will cause the task to go into READY.
Returning any other value will cause no action.
"""
if not my_task.parent._is_finished():
assert my_task.state != Task.WAITING
my_task.state = Task.FUTURE
if my_task._is_predicted():
self._predict(my_task)
LOG.debug(
"'%s'._update_state_hook says parent (%s, state=%s) "
"is_finished=%s" %
(self.name, my_task.parent.get_name(),
my_task.parent.get_state_name(), my_task.parent._is_finished()))
if my_task.parent._is_finished():
return True
return False
def _on_ready(self, my_task):
"""
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
self.test()
# Acquire locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
if not mutex.testandset():
return False
# Assign variables, if so requested.
for assignment in self.pre_assign:
assignment.assign(my_task, my_task)
# Run task-specific code.
result = self._on_ready_before_hook(my_task)
self.reached_event.emit(my_task.workflow, my_task)
if result:
result = self._on_ready_hook(my_task)
# Run user code, if any.
if result:
result = self.ready_event.emit(my_task.workflow, my_task)
if result:
# Assign variables, if so requested.
for assignment in self.post_assign:
assignment.assign(my_task, my_task)
# Release locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
mutex.unlock()
self.finished_event.emit(my_task.workflow, my_task)
return result
def _on_ready_before_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_ready_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_cancel(self, my_task):
"""
May be called by another task to cancel the operation before it was
completed.
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
self.cancelled_event.emit(my_task.workflow, my_task)
return True
def _on_trigger(self, my_task):
"""
May be called by another task to trigger a task-specific
event.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
raise NotImplementedError("Trigger not supported by this task.")
def _on_complete(self, my_task):
"""
Return True on success, False otherwise. Should not be overwritten,
overwrite _on_complete_hook() instead.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
if my_task.workflow.debug:
print "Executing task:", my_task.get_name()
if not self._on_complete_hook(my_task):
return False
# Notify the Workflow.
my_task.workflow._task_completed_notify(my_task)
if my_task.workflow.debug:
if hasattr(my_task.workflow, "outer_workflow"):
my_task.workflow.outer_workflow.task_tree.dump()
self.completed_event.emit(my_task.workflow, my_task)
return True
def _on_complete_hook(self, my_task):
"""
A hook into _on_complete() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: bool
@return: True on success, False otherwise.
"""
# If we have more than one output, implicitly split.
my_task._update_children(self.outputs)
return True
def serialize(self, serializer, **kwargs):
"""
Serializes the instance using the provided serializer.
@note: The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
@type serializer: L{SpiffWorkflow.storage.Serializer}
@param serializer: The serializer to use.
@type kwargs: dict
@param kwargs: Passed to the serializer.
@rtype: object
@return: The serialized object.
"""
return serializer._serialize_task_spec(self, **kwargs)
@classmethod
def deserialize(cls, serializer, wf_spec, s_state, **kwargs):
"""
Deserializes the instance using the provided serializer.
@note: The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
@type serializer: L{SpiffWorkflow.storage.Serializer}
@param serializer: The serializer to use.
@type wf_spec: L{SpiffWorkflow.spec.WorkflowSpec}
@param wf_spec: An instance of the WorkflowSpec.
@type s_state: object
@param s_state: The serialized task specification object.
@type kwargs: dict
@param kwargs: Passed to the serializer.
@rtype: TaskSpec
@return: The task specification instance.
"""
return serializer._deserialize_task_spec(wf_spec, s_state,
cls(wf_spec, s_state['name']),
**kwargs)
class TaskSpec(object):
"""
This class implements an abstract base type for all tasks.
Tasks provide the following signals:
- B{entered}: called when the state changes to READY or WAITING, at a
time where properties are not yet initialized.
- B{reached}: called when the state changes to READY or WAITING, at a
time where properties are already initialized using property_assign
and pre-assign.
- B{ready}: called when the state changes to READY, at a time where
properties are already initialized using property_assign and
pre-assign.
- B{completed}: called when the state changes to COMPLETED, at a time
before the post-assign variables are assigned.
- B{cancelled}: called when the state changes to CANCELLED, at a time
before the post-assign variables are assigned.
- B{finished}: called when the state changes to COMPLETED or CANCELLED,
at the last possible time and after the post-assign variables are
assigned.
"""
def __init__(self, parent, name, **kwargs):
"""
Constructor. May also have properties/attributes passed.
The difference between the assignment of a property using
property_assign versus pre_assign and post_assign is that
changes made using property_assign are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are properties that, once defined, can no
longer be modified.
@type parent: L{SpiffWorkflow.specs.WorkflowSpec}
@param parent: A reference to the parent (usually a workflow).
@type name: string
@param name: A name for the task.
@type lock: list(str)
@keyword lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
@type property_assign: list((str, object))
@keyword property_assign: a list of name/value pairs
@type pre_assign: list((str, object))
@keyword pre_assign: a list of name/value pairs
@type post_assign: list((str, object))
@keyword post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = str(name)
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.cancelled = False
self.properties = kwargs.get('properties', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self._parent._add_notify(self)
self.properties.update(self.defines)
assert self.id is not None
def _connect_notify(self, taskspec):
"""
Called by the previous task to let us know that it exists.
@type taskspec: TaskSpec
@param taskspec: The task by which this method is executed.
"""
self.inputs.append(taskspec)
def _get_activated_tasks(self, my_task, destination):
"""
Returns the list of tasks that were activated in the previous
call of execute(). Only returns tasks that point towards the
destination task, i.e. those which have destination as a
descendant.
@type my_task: Task
@param my_task: The associated task in the task tree.
@type destination: Task
@param destination: The destination task.
"""
return my_task.children
def _get_activated_threads(self, my_task):
"""
Returns the list of threads that were activated in the previous
call of execute().
@type my_task: Task
@param my_task: The associated task in the task tree.
"""
return my_task.children
def set_property(self, **kwargs):
"""
Defines the given property name/value pairs.
"""
for key in kwargs:
if self.defines.has_key(key):
msg = "Property %s can not be modified" % key
raise Exception.WorkflowException(msg)
self.properties.update(kwargs)
def get_property(self, name, default=None):
"""
Returns the value of the property with the given name, or the given
default value if the property does not exist.
@type name: string
@param name: A property name.
@type default: string
@param default: This value is returned if the property does not exist.
"""
return self.properties.get(name, default)
def connect(self, taskspec):
"""
Connect the *following* task to this one. In other words, the
given task is added as an output task.
@type taskspec: TaskSpec
@param taskspec: The new output task.
"""
self.outputs.append(taskspec)
taskspec._connect_notify(self)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise Exception.WorkflowException(
self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise Exception.WorkflowException(self, 'No input task connected.')
def _predict(self, my_task, seen=None, looked_ahead=0):
"""
Updates the branch such that all possible future routes are added
with the LIKELY flag.
Should NOT be overwritten! Instead, overwrite the hook (_predict_hook).
@type my_task: Task
@param my_task: The associated task in the task tree.
@type seen: list[taskspec]
@param seen: A list of already visited tasks.
@type looked_ahead: integer
@param looked_ahead: The depth of the predicted path so far.
"""
if seen is None:
seen = []
elif self in seen:
return
if not my_task._is_definite():
seen.append(self)
if my_task._has_state(Task.MAYBE):
looked_ahead += 1
if looked_ahead >= self.lookahead:
return
if not my_task._is_finished():
self._predict_hook(my_task)
for child in my_task.children:
child.task_spec._predict(child, seen[:], looked_ahead)
def _predict_hook(self, my_task):
if my_task._is_definite():
child_state = Task.FUTURE
else:
child_state = Task.LIKELY
my_task._update_children(self.outputs, child_state)
def _update_state(self, my_task):
my_task._inherit_attributes()
if not self._update_state_hook(my_task):
return
self.entered_event.emit(my_task.workflow, my_task)
my_task._ready()
def _update_state_hook(self, my_task):
was_predicted = my_task._is_predicted()
if not my_task.parent._is_finished():
my_task.state = Task.FUTURE
if was_predicted:
self._predict(my_task)
if my_task.parent._is_finished():
return True
return False
def _on_ready(self, my_task):
"""
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
assert not self.cancelled
self.test()
# Acquire locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
if not mutex.testandset():
return False
# Assign variables, if so requested.
for assignment in self.pre_assign:
assignment.assign(my_task, my_task)
# Run task-specific code.
result = self._on_ready_before_hook(my_task)
self.reached_event.emit(my_task.workflow, my_task)
if result:
result = self._on_ready_hook(my_task)
# Run user code, if any.
if result:
result = self.ready_event.emit(my_task.workflow, my_task)
if result:
# Assign variables, if so requested.
for assignment in self.post_assign:
assignment.assign(my_task, my_task)
# Release locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
mutex.unlock()
return result
def _on_ready_before_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_ready_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_cancel(self, my_task):
"""
May be called by another task to cancel the operation before it was
completed.
Return True on success, False otherwise.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
return True
def _on_trigger(self, my_task):
"""
May be called by another task to trigger a task-specific
event.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
raise NotImplementedError("Trigger not supported by this task.")
def _on_complete(self, my_task):
"""
Return True on success, False otherwise. Should not be overwritten,
overwrite _on_complete_hook() instead.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: boolean
@return: True on success, False otherwise.
"""
assert my_task is not None
assert not self.cancelled
if my_task.workflow.debug:
print "Executing task:", my_task.get_name()
if not self._on_complete_hook(my_task):
return False
# Notify the Workflow.
my_task.workflow._task_completed_notify(my_task)
if my_task.workflow.debug:
my_task.workflow.outer_job.task_tree.dump()
self.completed_event.emit(my_task.workflow, my_task)
return True
def _on_complete_hook(self, my_task):
"""
A hook into _on_complete() that does the task specific work.
@type my_task: Task
@param my_task: The associated task in the task tree.
@rtype: bool
@return: True on success, False otherwise.
"""
# If we have more than one output, implicitly split.
my_task._update_children(self.outputs)
return True
class Workflow(object):
"""
The engine that executes a workflow.
It is a essentially a facility for managing all branches.
A Workflow is also the place that holds the data of a running workflow.
"""
def __init__(self,
workflow_spec,
deserializing=False,
task_class=None,
**kwargs):
"""
Constructor.
:param deserializing: set to true when deserializing to avoid
generating tasks twice (and associated problems with multiple
hierarchies of tasks)
"""
assert workflow_spec is not None
LOG.debug("__init__ Workflow instance: %s" % self.__str__())
self.spec = workflow_spec
self.data = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
if task_class:
self.task_class = task_class
else:
if 'parent' in kwargs:
self.task_class = kwargs['parent'].task_class
else:
self.task_class = Task
if deserializing:
assert 'Root' in workflow_spec.task_specs
root = workflow_spec.task_specs['Root'] # Probably deserialized
else:
if 'Root' in workflow_spec.task_specs:
root = workflow_spec.task_specs['Root']
else:
root = specs.Simple(workflow_spec, 'Root')
self.task_tree = self.task_class(self, root)
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE)
self.spec.start._predict(start)
if 'parent' not in kwargs:
start.task_spec._update_state(start)
#start.dump()
def is_completed(self):
"""
Returns True if the entire Workflow is completed, False otherwise.
"""
mask = Task.NOT_FINISHED_MASK
iter = Task.Iterator(self.task_tree, mask)
try:
iter.next()
except:
# No waiting tasks found.
return True
return False
def has_failed(self):
mask = Task.FAILED
iter = Task.Iterator(self.task_tree, mask)
try:
iter.next()
return True
except:
return False
def _get_waiting_tasks(self):
waiting = Task.Iterator(self.task_tree, Task.WAITING)
return [w for w in waiting]
def _task_completed_notify(self, task):
# Commented cause Workflow and Tasks data are separated now
#if task.get_name() == 'End':
# self.data.update(task.data)
#self.data.update(task.data) # Update workflow data with each completed task's data
#print '_task_completed_notify. task: %s, n_subscribers: %s, is_completed: %s' % (
# task, self.completed_event.n_subscribers(), self.is_completed())
#LOG.debug('data(%s): %s', self.spec.name, self.data)
# Update the state of every WAITING task.
for thetask in self._get_waiting_tasks():
thetask.task_spec._update_state(thetask)
if self.completed_event.n_subscribers() == 0:
# Since is_completed() is expensive it makes sense to bail
# out if calling it is not necessary.
return
if self.is_completed():
self.completed_event(self)
def _get_mutex(self, name):
if name not in self.locks:
self.locks[name] = mutex()
return self.locks[name]
def get_data(self, name, default=None):
"""
Returns the value of the data field with the given name, or the given
default value if the data field does not exist.
:type name: string
:param name: A data field name.
:type default: obj
:param default: Return this value if the data field does not exist.
:rtype: obj
:returns: The value of the data field.
"""
return self.data.get(name, default)
def get_name(self):
return self.spec.name
def cancel(self, success=False):
"""
Cancels all open tasks in the workflow.
:type success: boolean
:param success: Whether the Workflow should be marked as successfully
completed.
"""
self.success = success
cancel = []
mask = Task.NOT_FINISHED_MASK
for task in Task.Iterator(self.task_tree, mask):
cancel.append(task)
for task in cancel:
task.cancel()
def get_task_spec_from_name(self, name):
"""
Returns the task spec with the given name.
:type name: string
:param name: The name of the task.
:rtype: TaskSpec
:returns: The task spec with the given name.
"""
return self.spec.get_task_spec_from_name(name)
def get_task(self, id):
"""
Returns the task with the given id.
:type id:integer
:param id: The id of a task.
:rtype: Task
:returns: The task with the given id.
"""
tasks = [task for task in self.get_tasks() if task.id == id]
return tasks[0] if len(tasks) == 1 else None
def get_tasks_from_spec_name(self, name):
"""
Returns all tasks whose spec has the given name.
@type name: str
@param name: The name of a task spec.
@rtype: Task
@return: The task that relates to the spec with the given name.
"""
return [
task for task in self.get_tasks() if task.task_spec.name == name
]
def get_tasks(self, state=Task.ANY_MASK):
"""
Returns a list of Task objects with the given state.
:type state: integer
:param state: A bitmask of states.
:rtype: list[Task]
:returns: A list of tasks.
"""
return [t for t in Task.Iterator(self.task_tree, state)]
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
:type task_id: integer
:param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def complete_next(self, pick_up=True):
"""
Runs the next task.
Returns True if completed, False otherwise.
:type pick_up: boolean
:param pick_up: When True, this method attempts to choose the next
task not by searching beginning at the root, but by
searching from the position at which the last call
of complete_next() left off.
:rtype: boolean
:returns: True if all tasks were completed, False otherwise.
"""
# Try to pick up where we left off.
blacklist = []
if pick_up and self.last_task is not None:
try:
iter = Task.Iterator(self.last_task, Task.READY)
next = iter.next()
except StopIteration:
next = None
self.last_task = None
if next is not None:
if next.complete():
self.last_task = next
return True
blacklist.append(next)
# Walk through all ready tasks.
for task in Task.Iterator(self.task_tree, Task.READY):
for blacklisted_task in blacklist:
if task._is_descendant_of(blacklisted_task):
continue
if task.complete():
self.last_task = task
return True
blacklist.append(task)
# Walk through all waiting tasks.
for task in Task.Iterator(self.task_tree, Task.WAITING):
task.task_spec._update_state(task)
if not task._has_state(Task.WAITING):
self.last_task = task
# XXX: without this children of WAITING celery task never predicted
task._sync_children(task.task_spec.outputs)
return True
return False
def complete_all(self, pick_up=True):
"""
Runs all branches until completion. This is a convenience wrapper
around complete_next(), and the pick_up argument is passed along.
:type pick_up: boolean
:param pick_up: Passed on to each call of complete_next().
"""
while self.complete_next(pick_up):
pass
def get_dump(self):
"""
Returns a complete dump of the current internal task tree for
debugging.
:rtype: string
:returns: The debug information.
"""
return self.task_tree.get_dump()
def dump(self):
"""
Like get_dump(), but prints the output to the terminal instead of
returning it.
"""
print self.task_tree.dump()
def serialize(self, serializer, **kwargs):
"""
Serializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: object
:returns: The serialized workflow.
"""
return serializer.serialize_workflow(self, **kwargs)
@classmethod
def deserialize(cls, serializer, s_state, **kwargs):
"""
Deserializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type s_state: object
:param s_state: The serialized workflow.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: Workflow
:returns: The workflow instance.
"""
return serializer.deserialize_workflow(s_state, **kwargs)
class Workflow(object):
"""
The engine that executes a workflow.
It is a essentially a facility for managing all branches.
A Workflow is also the place that holds the attributes of a running workflow.
"""
def __init__(self, workflow_spec, deserializing=False, **kwargs):
"""
Constructor.
:param deserializing: set to true when deserializing to avoid
generating tasks twice (and associated problems with multiple
hierarchies of tasks)
"""
assert workflow_spec is not None
LOG.debug("__init__ Workflow instance: %s" % self.__str__())
self.spec = workflow_spec
self.task_id_assigner = TaskIdAssigner()
self.attributes = {}
self.outer_workflow = kwargs.get('parent', self)
self.locks = {}
self.last_task = None
if deserializing:
assert 'Root' in workflow_spec.task_specs
root = workflow_spec.task_specs['Root'] # Probably deserialized
else:
if 'Root' in workflow_spec.task_specs:
root = workflow_spec.task_specs['Root']
else:
root = specs.Simple(workflow_spec, 'Root')
self.task_tree = Task(self, root)
self.success = True
self.debug = False
# Events.
self.completed_event = Event()
# Prevent the root task from being executed.
self.task_tree.state = Task.COMPLETED
start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE)
self.spec.start._predict(start)
if 'parent' not in kwargs:
start.task_spec._update_state(start)
#start.dump()
def is_completed(self):
"""
Returns True if the entire Workflow is completed, False otherwise.
"""
mask = Task.NOT_FINISHED_MASK
iter = Task.Iterator(self.task_tree, mask)
try:
iter.next()
except:
# No waiting tasks found.
return True
return False
def _get_waiting_tasks(self):
waiting = Task.Iterator(self.task_tree, Task.WAITING)
return [w for w in waiting]
def _task_completed_notify(self, task):
if task.get_name() == 'End':
self.attributes.update(task.get_attributes())
# Update the state of every WAITING task.
for thetask in self._get_waiting_tasks():
thetask.task_spec._update_state(thetask)
if self.completed_event.n_subscribers() == 0:
# Since is_completed() is expensive it makes sense to bail
# out if calling it is not necessary.
return
if self.is_completed():
self.completed_event(self)
def _get_mutex(self, name):
if name not in self.locks:
self.locks[name] = mutex()
return self.locks[name]
def get_attribute(self, name, default=None):
"""
Returns the value of the attribute with the given name, or the given
default value if the attribute does not exist.
:type name: string
:param name: An attribute name.
:type default: obj
:param default: Return this value if the attribute does not exist.
:rtype: obj
:returns: The value of the attribute.
"""
return self.attributes.get(name, default)
def cancel(self, success=False):
"""
Cancels all open tasks in the workflow.
:type success: boolean
:param success: Whether the Workflow should be marked as successfully
completed.
"""
self.success = success
cancel = []
mask = Task.NOT_FINISHED_MASK
for task in Task.Iterator(self.task_tree, mask):
cancel.append(task)
for task in cancel:
task.cancel()
def get_task_spec_from_name(self, name):
"""
Returns the task spec with the given name.
:type name: string
:param name: The name of the task.
:rtype: TaskSpec
:returns: The task spec with the given name.
"""
return self.spec.get_task_spec_from_name(name)
def get_task(self, id):
"""
Returns the task with the given id.
:type id:integer
:param id: The id of a task.
:rtype: Task
:returns: The task with the given id.
"""
tasks = [task for task in self.get_tasks() if task.id == id]
return tasks[0] if len(tasks) == 1 else None
def get_tasks(self, state=Task.ANY_MASK):
"""
Returns a list of Task objects with the given state.
:type state: integer
:param state: A bitmask of states.
:rtype: list[Task]
:returns: A list of tasks.
"""
return [t for t in Task.Iterator(self.task_tree, state)]
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
:type task_id: integer
:param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def complete_next(self, pick_up=True):
"""
Runs the next task.
Returns True if completed, False otherwise.
:type pick_up: boolean
:param pick_up: When True, this method attempts to choose the next
task not by searching beginning at the root, but by
searching from the position at which the last call
of complete_next() left off.
:rtype: boolean
:returns: True if all tasks were completed, False otherwise.
"""
# Try to pick up where we left off.
blacklist = []
if pick_up and self.last_task is not None:
try:
iter = Task.Iterator(self.last_task, Task.READY)
next = iter.next()
except:
next = None
self.last_task = None
if next is not None:
if next.complete():
self.last_task = next
return True
blacklist.append(next)
# Walk through all ready tasks.
for task in Task.Iterator(self.task_tree, Task.READY):
for blacklisted_task in blacklist:
if task._is_descendant_of(blacklisted_task):
continue
if task.complete():
self.last_task = task
return True
blacklist.append(task)
# Walk through all waiting tasks.
for task in Task.Iterator(self.task_tree, Task.WAITING):
task.task_spec._update_state(task)
if not task._has_state(Task.WAITING):
self.last_task = task
return True
return False
def complete_all(self, pick_up=True):
"""
Runs all branches until completion. This is a convenience wrapper
around complete_next(), and the pick_up argument is passed along.
:type pick_up: boolean
:param pick_up: Passed on to each call of complete_next().
"""
while self.complete_next(pick_up):
pass
def get_dump(self):
"""
Returns a complete dump of the current internal task tree for
debugging.
:rtype: string
:returns: The debug information.
"""
return self.task_tree.get_dump()
def dump(self):
"""
Like get_dump(), but prints the output to the terminal instead of
returning it.
"""
print self.task_tree.dump()
def serialize(self, serializer, **kwargs):
"""
Serializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: object
:returns: The serialized workflow.
"""
return serializer.serialize_workflow(self, **kwargs)
@classmethod
def deserialize(cls, serializer, s_state, **kwargs):
"""
Deserializes a Workflow instance using the provided serializer.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type s_state: object
:param s_state: The serialized workflow.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: Workflow
:returns: The workflow instance.
"""
return serializer.deserialize_workflow(s_state, **kwargs)
class TaskSpec(object):
"""
This class implements an abstract base type for all tasks.
Tasks provide the following signals:
- **entered**: called when the state changes to READY or WAITING, at a
time where spec data is not yet initialized.
- **reached**: called when the state changes to READY or WAITING, at a
time where spec data is already initialized using data_assign
and pre-assign.
- **ready**: called when the state changes to READY, at a time where
spec data is already initialized using data_assign and
pre-assign.
- **completed**: called when the state changes to COMPLETED, at a time
before the post-assign variables are assigned.
- **cancelled**: called when the state changes to CANCELLED, at a time
before the post-assign variables are assigned.
- **finished**: called when the state changes to COMPLETED or CANCELLED,
at the last possible time after the post-assign variables are
assigned and mutexes are released.
Event sequence is: entered -> reached -> ready -> completed -> finished
(cancelled may happen at any time)
The only events where implementing something other than state tracking
may be useful are the following:
- Reached: You could mess with the pre-assign variables here, for
example. Other then that, there is probably no need in a real
application.
- Ready: This is where a task could implement custom code, for example
for triggering an external system. This is also the only event where a
return value has a meaning (returning non-True will mean that the
post-assign procedure is skipped.)
"""
def __init__(self, parent, name, **kwargs):
"""
Constructor.
The difference between the assignment of a data value using
the data argument versus pre_assign and post_assign is that
changes made using data are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are spec data fields that, once defined, can
no longer be modified.
:type parent: L{SpiffWorkflow.specs.WorkflowSpec}
:param parent: A reference to the parent (usually a workflow).
:type name: string
:param name: A name for the task.
:type lock: list(str)
:param lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
:type data: dict((str, object))
:param data: name/value pairs
:type defines: dict((str, object))
:param defines: name/value pairs
:type pre_assign: list((str, object))
:param pre_assign: a list of name/value pairs
:type post_assign: list((str, object))
:param post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = name
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.data = kwargs.get('data', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self.cancelled_event = Event()
self.finished_event = Event()
# Error handling
self.error_handlers = []
self._parent._add_notify(self)
self.data.update(self.defines)
assert self.id is not None
def _connect_notify(self, taskspec):
"""
Called by the previous task to let us know that it exists.
:type taskspec: TaskSpec
:param taskspec: The task by which this method is executed.
"""
self.inputs.append(taskspec)
def ancestors(self):
"""Returns list of ancestor task specs based on inputs"""
results = []
def recursive_find_ancestors(task, stack):
for input in task.inputs:
if input not in stack:
stack.append(input)
recursive_find_ancestors(input, stack)
recursive_find_ancestors(self, results)
return results
def _get_activated_tasks(self, my_task, destination):
"""
Returns the list of tasks that were activated in the previous
call of execute(). Only returns tasks that point towards the
destination task, i.e. those which have destination as a
descendant.
:type my_task: Task
:param my_task: The associated task in the task tree.
:type destination: Task
:param destination: The destination task.
"""
return my_task.children
def _get_activated_threads(self, my_task):
"""
Returns the list of threads that were activated in the previous
call of execute().
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
return my_task.children
def set_data(self, **kwargs):
"""
Defines the given data field(s) using the given name/value pairs.
"""
for key in kwargs:
if key in self.defines:
msg = "Spec data %s can not be modified" % key
raise WorkflowException(self, msg)
self.data.update(kwargs)
def get_data(self, name, default=None):
"""
Returns the value of the data field with the given name, or the
given default value if the data was not defined.
:type name: string
:param name: The name of the data field.
:type default: string
:param default: Returned if the data field is not defined.
"""
return self.data.get(name, default)
def get_name_for(self, task):
return str(valueof(task, self.name))
def connect(self, taskspec):
"""
Connect the *following* task to this one. In other words, the
given task is added as an output task.
:type taskspec: TaskSpec
:param taskspec: The new output task.
"""
self.outputs.append(taskspec)
taskspec._connect_notify(self)
def connect_error_handler(self, errhdlr_taskspec):
self.error_handlers.append(errhdlr_taskspec)
errhdlr_taskspec._connect_notify(self)
def follow(self, taskspec):
"""
Make this task follow the provided one. In other words, this task is
added to the given task outputs.
This is an alias to connect, just easier to understand when reading
code - ex: my_task.follow(the_other_task)
Adding it after being confused by .connect one times too many!
:type taskspec: TaskSpec
:param taskspec: The task to follow.
"""
taskspec.connect(self)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise WorkflowException(self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise WorkflowException(self, 'No input task connected.')
def _predict(self, my_task, seen=None, looked_ahead=0):
"""
Updates the branch such that all possible future routes are added.
Should NOT be overwritten! Instead, overwrite _predict_hook().
:type my_task: Task
:param my_task: The associated task in the task tree.
:type seen: list[taskspec]
:param seen: A list of already visited tasks.
:type looked_ahead: integer
:param looked_ahead: The depth of the predicted path so far.
"""
if my_task._is_finished():
return
if seen is None:
seen = []
elif self in seen:
return
if not my_task._is_finished():
self._predict_hook(my_task)
if not my_task._is_definite():
if looked_ahead + 1 >= self.lookahead:
return
seen.append(self)
for child in my_task.children:
child.task_spec._predict(child, seen[:], looked_ahead + 1)
def _predict_hook(self, my_task):
# If the task's status is not predicted, we default to FUTURE
# for all it's outputs.
# Otherwise, copy my own state to the children.
if my_task._is_definite():
best_state = Task.FUTURE
else:
best_state = my_task.state
my_task._sync_children(self.outputs, best_state)
for child in my_task.children:
if not child._is_definite():
child._set_state(best_state)
def _update_state(self, my_task):
"""
Called whenever any event happens that may affect the
state of this task in the workflow. For example, if a predecessor
completes it makes sure to call this method so we can react.
This method called by Workflow.complete_next for READY and WAITING tasks
to complete and poll their progress
"""
if my_task.parent.workflow == my_task.workflow:
my_task._inherit_data()
try:
self._update_state_hook(my_task)
except TaskError:
self._on_error(my_task)
def _on_error(self, my_task):
exc_info = my_task.exc_info = sys.exc_info()
LOG.warn("'%s' error: %s", my_task.get_name(), exc_info[1])
my_task._set_state(Task.ERROR)
for eh in self.error_handlers:
if eh.match(my_task, exc_info[1]):
break
else:
eh = self._parent.default_error_handler
if not eh:
return
eh.select(my_task)
eh_task = my_task._add_child(eh)
# Move eh_task to the childs top
my_task.children.pop()
i = 0
for child in my_task.children:
if child._is_finished():
i += 1
else:
break
my_task.children.insert(i, eh_task)
eh._update_state(eh_task)
def _update_state_hook(self, my_task):
"""
Typically this method should perform the following actions::
- Update the state of the corresponding task.
- Update the predictions for its successors.
Returning non-False will cause the task to go into READY.
Returning any other value will cause no action.
"""
if my_task._is_predicted():
self._predict(my_task)
LOG.debug(
"'%s'._update_state_hook says parent (%s, state=%s) "
"is_finished=%s" %
(self.get_name_for(my_task), my_task.parent.get_name(),
my_task.parent.get_state_name(), my_task.parent._is_finished()))
if not my_task.parent._is_finished():
return
self.entered_event.emit(my_task.workflow, my_task)
my_task._ready()
def _on_ready(self, my_task):
"""
Return True on success, False otherwise.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
assert my_task is not None
self.test()
# Acquire locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
if not mutex.testandset():
return
# Assign variables, if so requested.
for assignment in self.pre_assign:
assignment.assign(my_task, my_task)
# Run task-specific code.
self._on_ready_before_hook(my_task)
self.reached_event.emit(my_task.workflow, my_task)
self._on_ready_hook(my_task)
# Run user code, if any.
if self.ready_event.emit(my_task.workflow, my_task):
# Assign variables, if so requested.
for assignment in self.post_assign:
assignment.assign(my_task, my_task)
# Release locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
mutex.unlock()
self.finished_event.emit(my_task.workflow, my_task)
def _on_ready_before_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
pass
def _on_ready_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
pass
def _on_cancel(self, my_task):
"""
May be called by another task to cancel the operation before it was
completed.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
self.cancelled_event.emit(my_task.workflow, my_task)
def _on_trigger(self, my_task):
"""
May be called by another task to trigger a task-specific
event.
:type my_task: Task
:param my_task: The associated task in the task tree.
:rtype: boolean
:returns: True on success, False otherwise.
"""
raise NotImplementedError("Trigger not supported by this task.")
def _on_complete(self, my_task):
"""
Return True on success, False otherwise. Should not be overwritten,
overwrite _on_complete_hook() instead.
:type my_task: Task
:param my_task: The associated task in the task tree.
:rtype: boolean
:returns: True on success, False otherwise.
"""
assert my_task is not None
if my_task.workflow.debug:
print "Executing task:", my_task.get_name()
self._on_complete_hook(my_task)
# Notify the Workflow.
my_task.workflow._task_completed_notify(my_task)
if my_task.workflow.debug:
if hasattr(my_task.workflow, "outer_workflow"):
my_task.workflow.outer_workflow.task_tree.dump()
self.completed_event.emit(my_task.workflow, my_task)
return True
def _on_complete_hook(self, my_task):
"""
A hook into _on_complete() that does the task specific work.
:type my_task: Task
:param my_task: The associated task in the task tree.
:rtype: bool
:returns: True on success, False otherwise.
"""
# If we have more than one output, implicitly split.
for child in my_task.children:
child.task_spec._update_state(child)
def serialize(self, serializer, **kwargs):
"""
Serializes the instance using the provided serializer.
.. note::
The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: object
:returns: The serialized object.
"""
return serializer._serialize_task_spec(self, **kwargs)
@classmethod
def deserialize(cls, serializer, wf_spec, s_state, **kwargs):
"""
Deserializes the instance using the provided serializer.
.. note::
The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type wf_spec: L{SpiffWorkflow.spec.WorkflowSpec}
:param wf_spec: An instance of the WorkflowSpec.
:type s_state: object
:param s_state: The serialized task specification object.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: TaskSpec
:returns: The task specification instance.
"""
instance = cls(wf_spec, s_state['name'])
return serializer._deserialize_task_spec(wf_spec, s_state, instance,
**kwargs)
class TaskSpec(object):
"""
This class implements an abstract base type for all tasks.
Tasks provide the following signals:
- **entered**: called when the state changes to READY or WAITING, at a
time where spec data is not yet initialized.
- **reached**: called when the state changes to READY or WAITING, at a
time where spec data is already initialized using data_assign
and pre-assign.
- **ready**: called when the state changes to READY, at a time where
spec data is already initialized using data_assign and
pre-assign.
- **completed**: called when the state changes to COMPLETED, at a time
before the post-assign variables are assigned.
- **cancelled**: called when the state changes to CANCELLED, at a time
before the post-assign variables are assigned.
- **finished**: called when the state changes to COMPLETED or CANCELLED,
at the last possible time after the post-assign variables are
assigned and mutexes are released.
Event sequence is: entered -> reached -> ready -> completed -> finished
(cancelled may happen at any time)
The only events where implementing something other than state tracking
may be useful are the following:
- Reached: You could mess with the pre-assign variables here, for
example. Other then that, there is probably no need in a real
application.
- Ready: This is where a task could implement custom code, for example
for triggering an external system. This is also the only event where a
return value has a meaning (returning non-True will mean that the
post-assign procedure is skipped.)
"""
def __init__(self, parent, name, **kwargs):
"""
Constructor.
The difference between the assignment of a data value using
the data argument versus pre_assign and post_assign is that
changes made using data are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are spec data fields that, once defined, can
no longer be modified.
:type parent: L{SpiffWorkflow.specs.WorkflowSpec}
:param parent: A reference to the parent (usually a workflow).
:type name: string
:param name: A name for the task.
:type lock: list(str)
:param lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
:type data: dict((str, object))
:param data: name/value pairs
:type defines: dict((str, object))
:param defines: name/value pairs
:type pre_assign: list((str, object))
:param pre_assign: a list of name/value pairs
:type post_assign: list((str, object))
:param post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = name
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.data = kwargs.get('data', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self.cancelled_event = Event()
self.finished_event = Event()
# Error handling
self.error_handlers = []
self._parent._add_notify(self)
self.data.update(self.defines)
assert self.id is not None
def _connect_notify(self, taskspec):
"""
Called by the previous task to let us know that it exists.
:type taskspec: TaskSpec
:param taskspec: The task by which this method is executed.
"""
self.inputs.append(taskspec)
def ancestors(self):
"""Returns list of ancestor task specs based on inputs"""
results = []
def recursive_find_ancestors(task, stack):
for input in task.inputs:
if input not in stack:
stack.append(input)
recursive_find_ancestors(input, stack)
recursive_find_ancestors(self, results)
return results
def _get_activated_tasks(self, my_task, destination):
"""
Returns the list of tasks that were activated in the previous
call of execute(). Only returns tasks that point towards the
destination task, i.e. those which have destination as a
descendant.
:type my_task: Task
:param my_task: The associated task in the task tree.
:type destination: Task
:param destination: The destination task.
"""
return my_task.children
def _get_activated_threads(self, my_task):
"""
Returns the list of threads that were activated in the previous
call of execute().
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
return my_task.children
def set_data(self, **kwargs):
"""
Defines the given data field(s) using the given name/value pairs.
"""
for key in kwargs:
if key in self.defines:
msg = "Spec data %s can not be modified" % key
raise WorkflowException(self, msg)
self.data.update(kwargs)
def get_data(self, name, default=None):
"""
Returns the value of the data field with the given name, or the
given default value if the data was not defined.
:type name: string
:param name: The name of the data field.
:type default: string
:param default: Returned if the data field is not defined.
"""
return self.data.get(name, default)
def get_name_for(self, task):
return str(valueof(task, self.name))
def connect(self, taskspec):
"""
Connect the *following* task to this one. In other words, the
given task is added as an output task.
:type taskspec: TaskSpec
:param taskspec: The new output task.
"""
self.outputs.append(taskspec)
taskspec._connect_notify(self)
def connect_error_handler(self, errhdlr_taskspec):
self.error_handlers.append(errhdlr_taskspec)
errhdlr_taskspec._connect_notify(self)
def follow(self, taskspec):
"""
Make this task follow the provided one. In other words, this task is
added to the given task outputs.
This is an alias to connect, just easier to understand when reading
code - ex: my_task.follow(the_other_task)
Adding it after being confused by .connect one times too many!
:type taskspec: TaskSpec
:param taskspec: The task to follow.
"""
taskspec.connect(self)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise WorkflowException(self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise WorkflowException(self, 'No input task connected.')
def _predict(self, my_task, seen=None, looked_ahead=0):
"""
Updates the branch such that all possible future routes are added.
Should NOT be overwritten! Instead, overwrite _predict_hook().
:type my_task: Task
:param my_task: The associated task in the task tree.
:type seen: list[taskspec]
:param seen: A list of already visited tasks.
:type looked_ahead: integer
:param looked_ahead: The depth of the predicted path so far.
"""
if my_task._is_finished():
return
if seen is None:
seen = []
elif self in seen:
return
if not my_task._is_finished():
self._predict_hook(my_task)
if not my_task._is_definite():
if looked_ahead + 1 >= self.lookahead:
return
seen.append(self)
for child in my_task.children:
child.task_spec._predict(child, seen[:], looked_ahead + 1)
def _predict_hook(self, my_task):
# If the task's status is not predicted, we default to FUTURE
# for all it's outputs.
# Otherwise, copy my own state to the children.
if my_task._is_definite():
best_state = Task.FUTURE
else:
best_state = my_task.state
my_task._sync_children(self.outputs, best_state)
for child in my_task.children:
if not child._is_definite():
child._set_state(best_state)
def _update_state(self, my_task):
"""
Called whenever any event happens that may affect the
state of this task in the workflow. For example, if a predecessor
completes it makes sure to call this method so we can react.
This method called by Workflow.complete_next for READY and WAITING tasks
to complete and poll their progress
"""
if my_task.parent.workflow == my_task.workflow:
my_task._inherit_data()
try:
self._update_state_hook(my_task)
except TaskError:
self._on_error(my_task)
def _on_error(self, my_task):
exc_info = my_task.exc_info = sys.exc_info()
LOG.warn("'%s' error: %s", my_task.get_name(), exc_info[1])
my_task._set_state(Task.ERROR)
for eh in self.error_handlers:
if eh.match(my_task, exc_info[1]):
break
else:
eh = self._parent.default_error_handler
if not eh:
return
eh.select(my_task)
eh_task = my_task._add_child(eh)
# Move eh_task to the childs top
my_task.children.pop()
i = 0
for child in my_task.children:
if child._is_finished():
i += 1
else:
break
my_task.children.insert(i, eh_task)
eh._update_state(eh_task)
def _update_state_hook(self, my_task):
"""
Typically this method should perform the following actions::
- Update the state of the corresponding task.
- Update the predictions for its successors.
Returning non-False will cause the task to go into READY.
Returning any other value will cause no action.
"""
if my_task._is_predicted():
self._predict(my_task)
LOG.debug("'%s'._update_state_hook says parent (%s, state=%s) "
"is_finished=%s" % (self.get_name_for(my_task), my_task.parent.get_name(),
my_task.parent.get_state_name(),
my_task.parent._is_finished()))
if not my_task.parent._is_finished():
return
self.entered_event.emit(my_task.workflow, my_task)
my_task._ready()
def _on_ready(self, my_task):
"""
Return True on success, False otherwise.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
assert my_task is not None
self.test()
# Acquire locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
if not mutex.testandset():
return
# Assign variables, if so requested.
for assignment in self.pre_assign:
assignment.assign(my_task, my_task)
# Run task-specific code.
self._on_ready_before_hook(my_task)
self.reached_event.emit(my_task.workflow, my_task)
self._on_ready_hook(my_task)
# Run user code, if any.
if self.ready_event.emit(my_task.workflow, my_task):
# Assign variables, if so requested.
for assignment in self.post_assign:
assignment.assign(my_task, my_task)
# Release locks, if any.
for lock in self.locks:
mutex = my_task.workflow._get_mutex(lock)
mutex.unlock()
self.finished_event.emit(my_task.workflow, my_task)
def _on_ready_before_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
pass
def _on_ready_hook(self, my_task):
"""
A hook into _on_ready() that does the task specific work.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
pass
def _on_cancel(self, my_task):
"""
May be called by another task to cancel the operation before it was
completed.
:type my_task: Task
:param my_task: The associated task in the task tree.
"""
self.cancelled_event.emit(my_task.workflow, my_task)
def _on_trigger(self, my_task):
"""
May be called by another task to trigger a task-specific
event.
:type my_task: Task
:param my_task: The associated task in the task tree.
:rtype: boolean
:returns: True on success, False otherwise.
"""
raise NotImplementedError("Trigger not supported by this task.")
def _on_complete(self, my_task):
"""
Return True on success, False otherwise. Should not be overwritten,
overwrite _on_complete_hook() instead.
:type my_task: Task
:param my_task: The associated task in the task tree.
:rtype: boolean
:returns: True on success, False otherwise.
"""
assert my_task is not None
if my_task.workflow.debug:
print "Executing task:", my_task.get_name()
self._on_complete_hook(my_task)
# Notify the Workflow.
my_task.workflow._task_completed_notify(my_task)
if my_task.workflow.debug:
if hasattr(my_task.workflow, "outer_workflow"):
my_task.workflow.outer_workflow.task_tree.dump()
self.completed_event.emit(my_task.workflow, my_task)
return True
def _on_complete_hook(self, my_task):
"""
A hook into _on_complete() that does the task specific work.
:type my_task: Task
:param my_task: The associated task in the task tree.
:rtype: bool
:returns: True on success, False otherwise.
"""
# If we have more than one output, implicitly split.
for child in my_task.children:
child.task_spec._update_state(child)
def serialize(self, serializer, **kwargs):
"""
Serializes the instance using the provided serializer.
.. note::
The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: object
:returns: The serialized object.
"""
return serializer._serialize_task_spec(self, **kwargs)
@classmethod
def deserialize(cls, serializer, wf_spec, s_state, **kwargs):
"""
Deserializes the instance using the provided serializer.
.. note::
The events of a TaskSpec are not serialized. If you
use them, make sure to re-connect them once the spec is
deserialized.
:type serializer: L{SpiffWorkflow.storage.Serializer}
:param serializer: The serializer to use.
:type wf_spec: L{SpiffWorkflow.spec.WorkflowSpec}
:param wf_spec: An instance of the WorkflowSpec.
:type s_state: object
:param s_state: The serialized task specification object.
:type kwargs: dict
:param kwargs: Passed to the serializer.
:rtype: TaskSpec
:returns: The task specification instance.
"""
instance = cls(wf_spec, s_state['name'])
return serializer._deserialize_task_spec(wf_spec,
s_state,
instance,
**kwargs)
def __init__(self, parent, name, **kwargs):
"""
Constructor.
The difference between the assignment of a data value using
the data argument versus pre_assign and post_assign is that
changes made using data are task-local, i.e. they are
not visible to other tasks.
Similarly, "defines" are spec data fields that, once defined, can
no longer be modified.
:type parent: L{SpiffWorkflow.specs.WorkflowSpec}
:param parent: A reference to the parent (usually a workflow).
:type name: string
:param name: A name for the task.
:type lock: list(str)
:param lock: A list of mutex names. The mutex is acquired
on entry of execute() and released on leave of
execute().
:type data: dict((str, object))
:param data: name/value pairs
:type defines: dict((str, object))
:param defines: name/value pairs
:type pre_assign: list((str, object))
:param pre_assign: a list of name/value pairs
:type post_assign: list((str, object))
:param post_assign: a list of name/value pairs
"""
assert parent is not None
assert name is not None
self._parent = parent
self.id = None
self.name = name
self.description = kwargs.get('description', '')
self.inputs = []
self.outputs = []
self.manual = False
self.internal = False # Only for easing debugging.
self.data = kwargs.get('data', {})
self.defines = kwargs.get('defines', {})
self.pre_assign = kwargs.get('pre_assign', [])
self.post_assign = kwargs.get('post_assign', [])
self.locks = kwargs.get('lock', [])
self.lookahead = 2 # Maximum number of MAYBE predictions.
# Events.
self.entered_event = Event()
self.reached_event = Event()
self.ready_event = Event()
self.completed_event = Event()
self.cancelled_event = Event()
self.finished_event = Event()
# Error handling
self.error_handlers = []
self._parent._add_notify(self)
self.data.update(self.defines)
assert self.id is not None
