class Workflow(object): """ The engine that executes a workflow. It is a essentially a facility for managing all branches. A Workflow is also the place that holds the data of a running workflow. """ def __init__(self, workflow_spec, deserializing=False, task_class=None, **kwargs): """ Constructor. :param deserializing: set to true when deserializing to avoid generating tasks twice (and associated problems with multiple hierarchies of tasks) """ assert workflow_spec is not None LOG.debug("__init__ Workflow instance: %s" % self.__str__()) self.spec = workflow_spec self.data = {} self.outer_workflow = kwargs.get('parent', self) self.locks = {} self.last_task = None if task_class: self.task_class = task_class else: if 'parent' in kwargs: self.task_class = kwargs['parent'].task_class else: self.task_class = Task if deserializing: assert 'Root' in workflow_spec.task_specs root = workflow_spec.task_specs['Root'] # Probably deserialized else: if 'Root' in workflow_spec.task_specs: root = workflow_spec.task_specs['Root'] else: root = specs.Simple(workflow_spec, 'Root') self.task_tree = self.task_class(self, root) self.success = True self.debug = False # Events. self.completed_event = Event() # Prevent the root task from being executed. self.task_tree.state = Task.COMPLETED start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE) self.spec.start._predict(start) if 'parent' not in kwargs: start.task_spec._update_state(start) #start.dump() def is_completed(self): """ Returns True if the entire Workflow is completed, False otherwise. """ mask = Task.NOT_FINISHED_MASK iter = Task.Iterator(self.task_tree, mask) try: iter.next() except: # No waiting tasks found. return True return False def has_failed(self): mask = Task.FAILED iter = Task.Iterator(self.task_tree, mask) try: iter.next() return True except: return False def _get_waiting_tasks(self): waiting = Task.Iterator(self.task_tree, Task.WAITING) return [w for w in waiting] def _task_completed_notify(self, task): # Commented cause Workflow and Tasks data are separated now #if task.get_name() == 'End': # self.data.update(task.data) #self.data.update(task.data) # Update workflow data with each completed task's data #print '_task_completed_notify. task: %s, n_subscribers: %s, is_completed: %s' % ( # task, self.completed_event.n_subscribers(), self.is_completed()) #LOG.debug('data(%s): %s', self.spec.name, self.data) # Update the state of every WAITING task. for thetask in self._get_waiting_tasks(): thetask.task_spec._update_state(thetask) if self.completed_event.n_subscribers() == 0: # Since is_completed() is expensive it makes sense to bail # out if calling it is not necessary. return if self.is_completed(): self.completed_event(self) def _get_mutex(self, name): if name not in self.locks: self.locks[name] = mutex() return self.locks[name] def get_data(self, name, default=None): """ Returns the value of the data field with the given name, or the given default value if the data field does not exist. :type name: string :param name: A data field name. :type default: obj :param default: Return this value if the data field does not exist. :rtype: obj :returns: The value of the data field. """ return self.data.get(name, default) def get_name(self): return self.spec.name def cancel(self, success=False): """ Cancels all open tasks in the workflow. :type success: boolean :param success: Whether the Workflow should be marked as successfully completed. """ self.success = success cancel = [] mask = Task.NOT_FINISHED_MASK for task in Task.Iterator(self.task_tree, mask): cancel.append(task) for task in cancel: task.cancel() def get_task_spec_from_name(self, name): """ Returns the task spec with the given name. :type name: string :param name: The name of the task. :rtype: TaskSpec :returns: The task spec with the given name. """ return self.spec.get_task_spec_from_name(name) def get_task(self, id): """ Returns the task with the given id. :type id:integer :param id: The id of a task. :rtype: Task :returns: The task with the given id. """ tasks = [task for task in self.get_tasks() if task.id == id] return tasks[0] if len(tasks) == 1 else None def get_tasks_from_spec_name(self, name): """ Returns all tasks whose spec has the given name. @type name: str @param name: The name of a task spec. @rtype: Task @return: The task that relates to the spec with the given name. """ return [ task for task in self.get_tasks() if task.task_spec.name == name ] def get_tasks(self, state=Task.ANY_MASK): """ Returns a list of Task objects with the given state. :type state: integer :param state: A bitmask of states. :rtype: list[Task] :returns: A list of tasks. """ return [t for t in Task.Iterator(self.task_tree, state)] def complete_task_from_id(self, task_id): """ Runs the task with the given id. :type task_id: integer :param task_id: The id of the Task object. """ if task_id is None: raise WorkflowException(self.spec, 'task_id is None') for task in self.task_tree: if task.id == task_id: return task.complete() msg = 'A task with the given task_id (%s) was not found' % task_id raise WorkflowException(self.spec, msg) def complete_next(self, pick_up=True): """ Runs the next task. Returns True if completed, False otherwise. :type pick_up: boolean :param pick_up: When True, this method attempts to choose the next task not by searching beginning at the root, but by searching from the position at which the last call of complete_next() left off. :rtype: boolean :returns: True if all tasks were completed, False otherwise. """ # Try to pick up where we left off. blacklist = [] if pick_up and self.last_task is not None: try: iter = Task.Iterator(self.last_task, Task.READY) next = iter.next() except StopIteration: next = None self.last_task = None if next is not None: if next.complete(): self.last_task = next return True blacklist.append(next) # Walk through all ready tasks. for task in Task.Iterator(self.task_tree, Task.READY): for blacklisted_task in blacklist: if task._is_descendant_of(blacklisted_task): continue if task.complete(): self.last_task = task return True blacklist.append(task) # Walk through all waiting tasks. for task in Task.Iterator(self.task_tree, Task.WAITING): task.task_spec._update_state(task) if not task._has_state(Task.WAITING): self.last_task = task # XXX: without this children of WAITING celery task never predicted task._sync_children(task.task_spec.outputs) return True return False def complete_all(self, pick_up=True): """ Runs all branches until completion. This is a convenience wrapper around complete_next(), and the pick_up argument is passed along. :type pick_up: boolean :param pick_up: Passed on to each call of complete_next(). """ while self.complete_next(pick_up): pass def get_dump(self): """ Returns a complete dump of the current internal task tree for debugging. :rtype: string :returns: The debug information. """ return self.task_tree.get_dump() def dump(self): """ Like get_dump(), but prints the output to the terminal instead of returning it. """ print self.task_tree.dump() def serialize(self, serializer, **kwargs): """ Serializes a Workflow instance using the provided serializer. :type serializer: L{SpiffWorkflow.storage.Serializer} :param serializer: The serializer to use. :type kwargs: dict :param kwargs: Passed to the serializer. :rtype: object :returns: The serialized workflow. """ return serializer.serialize_workflow(self, **kwargs) @classmethod def deserialize(cls, serializer, s_state, **kwargs): """ Deserializes a Workflow instance using the provided serializer. :type serializer: L{SpiffWorkflow.storage.Serializer} :param serializer: The serializer to use. :type s_state: object :param s_state: The serialized workflow. :type kwargs: dict :param kwargs: Passed to the serializer. :rtype: Workflow :returns: The workflow instance. """ return serializer.deserialize_workflow(s_state, **kwargs)
class Workflow(object): """ The engine that executes a workflow. It is a essentially a facility for managing all branches. A Workflow is also the place that holds the attributes of a running workflow. """ def __init__(self, workflow_spec, **kwargs): """ Constructor. """ assert workflow_spec is not None self.spec = workflow_spec self.task_id_assigner = TaskIdAssigner() self.attributes = {} self.outer_workflow = kwargs.get('parent', self) self.locks = {} self.last_task = None self.task_tree = Task(self, specs.Simple(workflow_spec, 'Root')) self.success = True self.debug = False # Events. self.completed_event = Event() # Prevent the root task from being executed. self.task_tree.state = Task.COMPLETED start = self.task_tree._add_child(self.spec.start) self.spec.start._predict(start) if not kwargs.has_key('parent'): start.task_spec._update_state(start) #start.dump() def is_completed(self): """ Returns True if the entire Workflow is completed, False otherwise. """ mask = Task.NOT_FINISHED_MASK iter = Task.Iterator(self.task_tree, mask) try: next = iter.next() except: # No waiting tasks found. return True return False def _get_waiting_tasks(self): waiting = Task.Iterator(self.task_tree, Task.WAITING) return [w for w in waiting] def _task_completed_notify(self, task): if task.get_name() == 'End': self.attributes.update(task.get_attributes()) # Update the state of every WAITING task. for thetask in self._get_waiting_tasks(): thetask.task_spec._update_state(thetask) if self.completed_event.n_subscribers() == 0: # Since is_completed() is expensive it makes sense to bail # out if calling it is not necessary. return if self.is_completed(): self.completed_event(self) def _get_mutex(self, name): if not self.locks.has_key(name): self.locks[name] = mutex() return self.locks[name] def get_attribute(self, name, default=None): """ Returns the value of the attribute with the given name, or the given default value if the attribute does not exist. @type name: string @param name: An attribute name. @type default: obj @param default: Return this value if the attribute does not exist. @rtype: obj @return: The value of the attribute. """ return self.attributes.get(name, default) def cancel(self, success=False): """ Cancels all open tasks in the workflow. @type success: boolean @param success: Whether the Workflow should be marked as successfully completed. """ self.success = success cancel = [] mask = Task.NOT_FINISHED_MASK for task in Task.Iterator(self.task_tree, mask): cancel.append(task) for task in cancel: task.cancel() def get_task_spec_from_name(self, name): """ Returns the task spec with the given name. @type name: string @param name: The name of the task. @rtype: TaskSpec @return: The task spec with the given name. """ return self.spec.get_task_spec_from_name(name) def get_task(self, id): """ Returns the task with the given id. @type id:integer @param id: The id of a state. @rtype: Task @return: The task with the given id. """ tasks = [task for task in self.get_tasks() if task.id == id] return tasks[0] if len(tasks) == 1 else None def get_tasks(self, state=Task.ANY_MASK): """ Returns a list of Task objects with the given state. @type state: integer @param state: A bitmask of states. @rtype: list[Task] @return: A list of tasks. """ return [t for t in Task.Iterator(self.task_tree, state)] def complete_task_from_id(self, task_id): """ Runs the task with the given id. @type task_id: integer @param task_id: The id of the Task object. """ if task_id is None: raise WorkflowException(self.spec, 'task_id is None') for task in self.task_tree: if task.id == task_id: return task.complete() msg = 'A task with the given task_id (%s) was not found' % task_id raise WorkflowException(self.spec, msg) def complete_next(self, pick_up=True): """ Runs the next task. Returns True if completed, False otherwise. @type pick_up: boolean @param pick_up: When True, this method attempts to choose the next task not by searching beginning at the root, but by searching from the position at which the last call of complete_next() left off. @rtype: boolean @return: True if all tasks were completed, False otherwise. """ # Try to pick up where we left off. blacklist = [] if pick_up and self.last_task is not None: try: iter = Task.Iterator(self.last_task, Task.READY) next = iter.next() except: next = None self.last_task = None if next is not None: if next.complete(): self.last_task = next return True blacklist.append(next) # Walk through all waiting tasks. for task in Task.Iterator(self.task_tree, Task.READY): for blacklisted_task in blacklist: if task._is_descendant_of(blacklisted_task): continue if task.complete(): self.last_task = task return True blacklist.append(task) return False def complete_all(self, pick_up=True): """ Runs all branches until completion. This is a convenience wrapper around complete_next(), and the pick_up argument is passed along. @type pick_up: boolean @param pick_up: Passed on to each call of complete_next(). """ while self.complete_next(pick_up): pass def get_dump(self): """ Returns a complete dump of the current internal task tree for debugging. @rtype: string @return: The debug information. """ return self.task_tree.get_dump() def dump(self): """ Like get_dump(), but prints the output to the terminal instead of returning it. """ return self.task_tree.dump() def serialize(self, serializer): """ Serializes a Workflow instance using the provided serializer. """ return serializer.serialize_workflow(self) @classmethod def deserialize(cls, serializer, s_state): """ Deserializes a Workflow instance using the provided serializer. """ return serializer.deserialize_workflow(s_state)
class Workflow(object): """ The engine that executes a workflow. It is a essentially a facility for managing all branches. A Workflow is also the place that holds the attributes of a running workflow. """ def __init__(self, workflow_spec, deserializing=False, **kwargs): """ Constructor. :param deserializing: set to true when deserializing to avoid generating tasks twice (and associated problems with multiple hierarchies of tasks) """ assert workflow_spec is not None LOG.debug("__init__ Workflow instance: %s" % self.__str__()) self.spec = workflow_spec self.task_id_assigner = TaskIdAssigner() self.attributes = {} self.outer_workflow = kwargs.get('parent', self) self.locks = {} self.last_task = None if deserializing: assert 'Root' in workflow_spec.task_specs root = workflow_spec.task_specs['Root'] # Probably deserialized else: if 'Root' in workflow_spec.task_specs: root = workflow_spec.task_specs['Root'] else: root = specs.Simple(workflow_spec, 'Root') self.task_tree = Task(self, root) self.success = True self.debug = False # Events. self.completed_event = Event() # Prevent the root task from being executed. self.task_tree.state = Task.COMPLETED start = self.task_tree._add_child(self.spec.start, state=Task.FUTURE) self.spec.start._predict(start) if 'parent' not in kwargs: start.task_spec._update_state(start) #start.dump() def is_completed(self): """ Returns True if the entire Workflow is completed, False otherwise. """ mask = Task.NOT_FINISHED_MASK iter = Task.Iterator(self.task_tree, mask) try: iter.next() except: # No waiting tasks found. return True return False def _get_waiting_tasks(self): waiting = Task.Iterator(self.task_tree, Task.WAITING) return [w for w in waiting] def _task_completed_notify(self, task): if task.get_name() == 'End': self.attributes.update(task.get_attributes()) # Update the state of every WAITING task. for thetask in self._get_waiting_tasks(): thetask.task_spec._update_state(thetask) if self.completed_event.n_subscribers() == 0: # Since is_completed() is expensive it makes sense to bail # out if calling it is not necessary. return if self.is_completed(): self.completed_event(self) def _get_mutex(self, name): if name not in self.locks: self.locks[name] = mutex() return self.locks[name] def get_attribute(self, name, default=None): """ Returns the value of the attribute with the given name, or the given default value if the attribute does not exist. :type name: string :param name: An attribute name. :type default: obj :param default: Return this value if the attribute does not exist. :rtype: obj :returns: The value of the attribute. """ return self.attributes.get(name, default) def cancel(self, success=False): """ Cancels all open tasks in the workflow. :type success: boolean :param success: Whether the Workflow should be marked as successfully completed. """ self.success = success cancel = [] mask = Task.NOT_FINISHED_MASK for task in Task.Iterator(self.task_tree, mask): cancel.append(task) for task in cancel: task.cancel() def get_task_spec_from_name(self, name): """ Returns the task spec with the given name. :type name: string :param name: The name of the task. :rtype: TaskSpec :returns: The task spec with the given name. """ return self.spec.get_task_spec_from_name(name) def get_task(self, id): """ Returns the task with the given id. :type id:integer :param id: The id of a task. :rtype: Task :returns: The task with the given id. """ tasks = [task for task in self.get_tasks() if task.id == id] return tasks[0] if len(tasks) == 1 else None def get_tasks(self, state=Task.ANY_MASK): """ Returns a list of Task objects with the given state. :type state: integer :param state: A bitmask of states. :rtype: list[Task] :returns: A list of tasks. """ return [t for t in Task.Iterator(self.task_tree, state)] def complete_task_from_id(self, task_id): """ Runs the task with the given id. :type task_id: integer :param task_id: The id of the Task object. """ if task_id is None: raise WorkflowException(self.spec, 'task_id is None') for task in self.task_tree: if task.id == task_id: return task.complete() msg = 'A task with the given task_id (%s) was not found' % task_id raise WorkflowException(self.spec, msg) def complete_next(self, pick_up=True): """ Runs the next task. Returns True if completed, False otherwise. :type pick_up: boolean :param pick_up: When True, this method attempts to choose the next task not by searching beginning at the root, but by searching from the position at which the last call of complete_next() left off. :rtype: boolean :returns: True if all tasks were completed, False otherwise. """ # Try to pick up where we left off. blacklist = [] if pick_up and self.last_task is not None: try: iter = Task.Iterator(self.last_task, Task.READY) next = iter.next() except: next = None self.last_task = None if next is not None: if next.complete(): self.last_task = next return True blacklist.append(next) # Walk through all ready tasks. for task in Task.Iterator(self.task_tree, Task.READY): for blacklisted_task in blacklist: if task._is_descendant_of(blacklisted_task): continue if task.complete(): self.last_task = task return True blacklist.append(task) # Walk through all waiting tasks. for task in Task.Iterator(self.task_tree, Task.WAITING): task.task_spec._update_state(task) if not task._has_state(Task.WAITING): self.last_task = task return True return False def complete_all(self, pick_up=True): """ Runs all branches until completion. This is a convenience wrapper around complete_next(), and the pick_up argument is passed along. :type pick_up: boolean :param pick_up: Passed on to each call of complete_next(). """ while self.complete_next(pick_up): pass def get_dump(self): """ Returns a complete dump of the current internal task tree for debugging. :rtype: string :returns: The debug information. """ return self.task_tree.get_dump() def dump(self): """ Like get_dump(), but prints the output to the terminal instead of returning it. """ print self.task_tree.dump() def serialize(self, serializer, **kwargs): """ Serializes a Workflow instance using the provided serializer. :type serializer: L{SpiffWorkflow.storage.Serializer} :param serializer: The serializer to use. :type kwargs: dict :param kwargs: Passed to the serializer. :rtype: object :returns: The serialized workflow. """ return serializer.serialize_workflow(self, **kwargs) @classmethod def deserialize(cls, serializer, s_state, **kwargs): """ Deserializes a Workflow instance using the provided serializer. :type serializer: L{SpiffWorkflow.storage.Serializer} :param serializer: The serializer to use. :type s_state: object :param s_state: The serialized workflow. :type kwargs: dict :param kwargs: Passed to the serializer. :rtype: Workflow :returns: The workflow instance. """ return serializer.deserialize_workflow(s_state, **kwargs)