def test(self): """ Checks whether all required attributes are set. Throws an exception if an error was detected. """ if len(self.inputs) != 0: raise WorkflowException(self, 'StartTask with an input.') elif len(self.outputs) < 1: raise WorkflowException(self, 'No output task connected.')
def test(self): """ Checks whether all required attributes are set. Throws an exception if an error was detected. """ if self.id is None: raise WorkflowException(self, 'TaskSpec is not yet instanciated.') if len(self.inputs) < 1: raise WorkflowException(self, 'No input task connected.')
def complete_task_from_id(self, task_id): """ Runs the task with the given id. :type task_id: integer :param task_id: The id of the Task object. """ if task_id is None: raise WorkflowException(self.spec, 'task_id is None') for task in self.task_tree: if task.id == task_id: return task.complete() msg = 'A task with the given task_id (%s) was not found' % task_id raise WorkflowException(self.spec, msg)
def _try_fire_structured(self, my_task, force=False): # Retrieve a list of all activated tasks from the associated # task that did the conditional parallel split. split_task = my_task._find_ancestor_from_name(self.split_task) if split_task is None: msg = 'Join with %s, which was not reached' % self.split_task raise WorkflowException(self, msg) tasks = split_task.task_spec._get_activated_tasks(split_task, my_task) # The default threshold is the number of branches that were started. threshold = valueof(my_task, self.threshold) if threshold is None: threshold = len(tasks) # Look up which tasks have already completed. waiting_tasks = [] completed = 0 for task in tasks: # Refresh path prediction. task.task_spec._predict(task) if not self._branch_may_merge_at(task): completed += 1 elif self._branch_is_complete(task): completed += 1 else: waiting_tasks.append(task) # If the threshold was reached, get ready to fire. return force or completed >= threshold, waiting_tasks
def _retry_fire(self, my_task): """ Abort celery task and retry it""" if not my_task._has_state(Task.WAITING): raise WorkflowException( my_task, "Cannot refire a task that is not" "in WAITING state") # Check state of existing call and abort it (save history) if my_task.internal_data.get('task_id') is not None: if not 'async_call' in my_task.internal_data: task_id = my_task.internal_data['task_id'] my_task.internal_data['async_call'] = default_app.AsyncResult( task_id) my_task.internal_data['deserialized'] = True my_task.internal_data['async_call'].state # manually refresh async_call = my_task.internal_data['async_call'] if async_call.state == 'FAILED': pass elif async_call.state in ['RETRY', 'PENDING', 'STARTED']: async_call.revoke() LOG.info("Celery task '%s' was in %s state and was revoked", async_call.state, async_call) elif async_call.state == 'SUCCESS': LOG.warning( "Celery task '%s' succeeded, but a refire was " "requested", async_call) self._clear_celery_task_data(my_task) # Retrigger return self._try_fire(my_task)
def test(self): """ Checks whether all required attributes are set. Throws an exception if an error was detected. """ TaskSpec.test(self) if len(self.outputs) > 0: raise WorkflowException(self, 'Cancel with an output.')
def test(self): """ Checks whether all required attributes are set. Throws an exception if an error was detected. """ TaskSpec.test(self) if len(self.cond_task_specs) < 1: raise WorkflowException(self, 'At least one output required.') for condition, name in self.cond_task_specs: if name is None: raise WorkflowException(self, 'Condition with no task spec.') task_spec = self._parent.get_task_spec_from_name(name) if task_spec is None: msg = 'Condition leads to non-existent task ' + repr(name) raise WorkflowException(self, msg) if condition is None: continue
def test(self): """ Checks whether all required attributes are set. Throws an exception if an error was detected. """ MultiChoice.test(self) if self.default_task_spec is None: raise WorkflowException(self, 'A default output is required.')
def set_data(self, **kwargs): """ Defines the given data field(s) using the given name/value pairs. """ for key in kwargs: if key in self.defines: msg = "Spec data %s can not be modified" % key raise WorkflowException(self, msg) self.data.update(kwargs)
def set_property(self, **kwargs): """ Defines the given property name/value pairs. """ for key in kwargs: if key in self.defines: msg = "Property %s can not be modified" % key raise WorkflowException(self, msg) self.properties.update(kwargs)
def test(self): """ Checks whether all required attributes are set. Throws an exception if an error was detected. """ #This has been overidden to allow a single default flow out (without a condition) - useful for #the converging type TaskSpec.test(self) # if len(self.cond_task_specs) < 1: # raise WorkflowException(self, 'At least one output required.') for condition, name in self.cond_task_specs: if name is None: raise WorkflowException(self, 'Condition with no task spec.') task_spec = self._parent.get_task_spec_from_name(name) if task_spec is None: msg = 'Condition leads to non-existent task ' + repr(name) raise WorkflowException(self, msg) if condition is None: continue if self.default_task_spec is None: raise WorkflowException(self, 'A default output is required.')
def _sync_children(self, task_specs, state=MAYBE): """ This method syncs up the task's children with the given list of task specs. In other words:: - Add one child for each given TaskSpec, unless that child already exists. - Remove all children for which there is no spec in the given list, unless it is a "triggered" task. .. note:: It is an error if the task has a non-predicted child that is not given in the TaskSpecs. :type task_specs: list(TaskSpec) :param task_specs: The list of task specs that may become children. :type state: integer :param state: The bitmask of states for the new children. """ LOG.debug("Updating children for %s" % self.get_name()) if task_specs is None: raise ValueError('"task_specs" argument is None') add = task_specs[:] # Create a list of all children that are no longer needed. remove = [] for child in self.children: # Triggered tasks are never removed. if child.triggered: continue # Check whether the task needs to be removed. if child.task_spec in add: add.remove(child.task_spec) continue # Non-predicted tasks must not be removed, so they HAVE to be in # the given task spec list. if child._is_definite(): raise WorkflowException( self.task_spec, 'removal of non-predicted child %s' % repr(child)) remove.append(child) # Remove and add the children accordingly. for child in remove: self.children.remove(child) for task_spec in add: self._add_child(task_spec, state)
def _try_fire(self, my_task): # If the threshold was already reached, there is nothing else to do. if my_task._has_state(Task.COMPLETED): return False if my_task._has_state(Task.READY): return True # Retrieve a list of all activated tasks from the associated # task that did the conditional parallel split. split_task = my_task._find_ancestor_from_name(self.split_task) if split_task is None: msg = 'Join with %s, which was not reached' % self.split_task raise WorkflowException(self, msg) tasks = split_task.task_spec._get_activated_threads(split_task) # The default threshold is the number of threads that were started. threshold = valueof(my_task, self.threshold) if threshold is None: threshold = len(tasks) # Look up which tasks have already completed. waiting_tasks = [] completed = 0 for task in tasks: # Refresh path prediction. task.task_spec._predict(task) if self._branch_is_complete(task): completed += 1 else: waiting_tasks.append(task) # If the threshold was reached, get ready to fire. if completed >= threshold: # If this is a cancelling join, cancel all incoming branches, # except for the one that just completed. if self.cancel_remaining: for task in waiting_tasks: task.cancel() return True # We do NOT set the task state to COMPLETED, because in # case all other incoming tasks get cancelled (or never reach # the ThreadMerge for other reasons, such as reaching a stub branch), # we need to revisit it. return False
def _add_child(self, task_spec, state=MAYBE): """ Adds a new child and assigns the given TaskSpec to it. :type task_spec: TaskSpec :param task_spec: The task spec that is assigned to the new child. :type state: integer :param state: The bitmask of states for the new child. :rtype: Task :returns: The new child task. """ if task_spec is None: raise ValueError(self, '_add_child() requires a TaskSpec') if self._is_predicted() and state & self.PREDICTED_MASK == 0: msg = 'Attempt to add non-predicted child to predicted task' raise WorkflowException(self.task_spec, msg) task = Task(self.workflow, task_spec, self, state=state) task.thread_id = self.thread_id if state == self.READY: task._ready() return task
def _setstate(self, value, force=False): """ Setting force to True allows for changing a state after it COMPLETED. This would otherwise be invalid. """ if self._state == value: return if value < self._state and not force: raise WorkflowException(self.task_spec, 'state went from %s to %s!' % ( self.get_state_name(), self.state_names[value])) if __debug__: old = self.get_state_name() self._state = value if __debug__: self.log.append("Moving '%s' from %s to %s" % (self.get_name(), old, self.get_state_name())) self.state_history.append(value) LOG.debug("Moving '%s' (spec=%s) from %s to %s" % (self.get_name(), self.task_spec.name, old, self.get_state_name()))
def test(self): TaskSpec.test(self) if self.file is not None and not os.path.exists(self.file): raise WorkflowException(self, 'File does not exist: %s' % self.file)
def _update_children(self, task_specs, state=None): """ This method adds one child for each given TaskSpec, unless that child already exists. The state of COMPLETED tasks is never changed. If this method is passed a state: - The state of TRIGGERED tasks is not changed. - The state for all children is set to the given value. If this method is not passed a state: - The state for all children is updated by calling the child's _update_state() method. If the task currently has a child that is not given in the TaskSpec, the child is removed. It is an error if the task has a non-LIKELY child that is not given in the TaskSpecs. @type task_specs: list(TaskSpec) @param task_specs: The list of task specs that may become children. @type state: integer @param state: The bitmask of states for newly added children. """ LOG.debug("Updating children for %s" % self.get_name()) if task_specs is None: raise ValueError('"task_specs" argument is None') if type(task_specs) != type([]): task_specs = [task_specs] # Create a list of all children that are no longer needed, and # set the state of all others. add = task_specs[:] remove = [] for child in self.children: # Must not be TRIGGERED or COMPLETED. if child._has_state(Task.TRIGGERED): if state is None: child.task_spec._update_state(child) continue if child._is_finished(): add.remove(child.task_spec) continue # Check whether the item needs to be added or removed. if child.task_spec not in add: if not self._is_definite(): msg = 'Attempt to remove non-predicted %s' % child.get_name() raise WorkflowException(self, msg) remove.append(child) continue add.remove(child.task_spec) # Update the state. if state is not None: child.state = state else: child.task_spec._update_state(child) # Remove all children that are no longer specified. for child in remove: self.children.remove(child) # Add a new child for each of the remaining tasks. for task_spec in add: if state is not None: self._add_child(task_spec, state) else: child = self._add_child(task_spec, self.LIKELY) task_spec._update_state(child)
def _connect_notify(self, task_spec): """ Called by the previous task to let us know that it exists. """ raise WorkflowException(self, 'StartTask can not have any inputs.')