def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if len(self.inputs) != 0:
raise WorkflowException(self, 'StartTask with an input.')
elif len(self.outputs) < 1:
raise WorkflowException(self, 'No output task connected.')
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
if self.id is None:
raise WorkflowException(self, 'TaskSpec is not yet instanciated.')
if len(self.inputs) < 1:
raise WorkflowException(self, 'No input task connected.')
def complete_task_from_id(self, task_id):
"""
Runs the task with the given id.
:type task_id: integer
:param task_id: The id of the Task object.
"""
if task_id is None:
raise WorkflowException(self.spec, 'task_id is None')
for task in self.task_tree:
if task.id == task_id:
return task.complete()
msg = 'A task with the given task_id (%s) was not found' % task_id
raise WorkflowException(self.spec, msg)
def _try_fire_structured(self, my_task, force=False):
# Retrieve a list of all activated tasks from the associated
# task that did the conditional parallel split.
split_task = my_task._find_ancestor_from_name(self.split_task)
if split_task is None:
msg = 'Join with %s, which was not reached' % self.split_task
raise WorkflowException(self, msg)
tasks = split_task.task_spec._get_activated_tasks(split_task, my_task)
# The default threshold is the number of branches that were started.
threshold = valueof(my_task, self.threshold)
if threshold is None:
threshold = len(tasks)
# Look up which tasks have already completed.
waiting_tasks = []
completed = 0
for task in tasks:
# Refresh path prediction.
task.task_spec._predict(task)
if not self._branch_may_merge_at(task):
completed += 1
elif self._branch_is_complete(task):
completed += 1
else:
waiting_tasks.append(task)
# If the threshold was reached, get ready to fire.
return force or completed >= threshold, waiting_tasks
def _retry_fire(self, my_task):
""" Abort celery task and retry it"""
if not my_task._has_state(Task.WAITING):
raise WorkflowException(
my_task, "Cannot refire a task that is not"
"in WAITING state")
# Check state of existing call and abort it (save history)
if my_task.internal_data.get('task_id') is not None:
if not 'async_call' in my_task.internal_data:
task_id = my_task.internal_data['task_id']
my_task.internal_data['async_call'] = default_app.AsyncResult(
task_id)
my_task.internal_data['deserialized'] = True
my_task.internal_data['async_call'].state # manually refresh
async_call = my_task.internal_data['async_call']
if async_call.state == 'FAILED':
pass
elif async_call.state in ['RETRY', 'PENDING', 'STARTED']:
async_call.revoke()
LOG.info("Celery task '%s' was in %s state and was revoked",
async_call.state, async_call)
elif async_call.state == 'SUCCESS':
LOG.warning(
"Celery task '%s' succeeded, but a refire was "
"requested", async_call)
self._clear_celery_task_data(my_task)
# Retrigger
return self._try_fire(my_task)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
TaskSpec.test(self)
if len(self.outputs) > 0:
raise WorkflowException(self, 'Cancel with an output.')
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
TaskSpec.test(self)
if len(self.cond_task_specs) < 1:
raise WorkflowException(self, 'At least one output required.')
for condition, name in self.cond_task_specs:
if name is None:
raise WorkflowException(self, 'Condition with no task spec.')
task_spec = self._parent.get_task_spec_from_name(name)
if task_spec is None:
msg = 'Condition leads to non-existent task ' + repr(name)
raise WorkflowException(self, msg)
if condition is None:
continue
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
MultiChoice.test(self)
if self.default_task_spec is None:
raise WorkflowException(self, 'A default output is required.')
def set_data(self, **kwargs):
"""
Defines the given data field(s) using the given name/value pairs.
"""
for key in kwargs:
if key in self.defines:
msg = "Spec data %s can not be modified" % key
raise WorkflowException(self, msg)
self.data.update(kwargs)
def set_property(self, **kwargs):
"""
Defines the given property name/value pairs.
"""
for key in kwargs:
if key in self.defines:
msg = "Property %s can not be modified" % key
raise WorkflowException(self, msg)
self.properties.update(kwargs)
def test(self):
"""
Checks whether all required attributes are set. Throws an exception
if an error was detected.
"""
#This has been overidden to allow a single default flow out (without a condition) - useful for
#the converging type
TaskSpec.test(self)
# if len(self.cond_task_specs) < 1:
# raise WorkflowException(self, 'At least one output required.')
for condition, name in self.cond_task_specs:
if name is None:
raise WorkflowException(self, 'Condition with no task spec.')
task_spec = self._parent.get_task_spec_from_name(name)
if task_spec is None:
msg = 'Condition leads to non-existent task ' + repr(name)
raise WorkflowException(self, msg)
if condition is None:
continue
if self.default_task_spec is None:
raise WorkflowException(self, 'A default output is required.')
def _sync_children(self, task_specs, state=MAYBE):
"""
This method syncs up the task's children with the given list of task
specs. In other words::
- Add one child for each given TaskSpec, unless that child already
exists.
- Remove all children for which there is no spec in the given list,
unless it is a "triggered" task.
.. note::
It is an error if the task has a non-predicted child that is
not given in the TaskSpecs.
:type task_specs: list(TaskSpec)
:param task_specs: The list of task specs that may become children.
:type state: integer
:param state: The bitmask of states for the new children.
"""
LOG.debug("Updating children for %s" % self.get_name())
if task_specs is None:
raise ValueError('"task_specs" argument is None')
add = task_specs[:]
# Create a list of all children that are no longer needed.
remove = []
for child in self.children:
# Triggered tasks are never removed.
if child.triggered:
continue
# Check whether the task needs to be removed.
if child.task_spec in add:
add.remove(child.task_spec)
continue
# Non-predicted tasks must not be removed, so they HAVE to be in
# the given task spec list.
if child._is_definite():
raise WorkflowException(
self.task_spec,
'removal of non-predicted child %s' % repr(child))
remove.append(child)
# Remove and add the children accordingly.
for child in remove:
self.children.remove(child)
for task_spec in add:
self._add_child(task_spec, state)
def _try_fire(self, my_task):
# If the threshold was already reached, there is nothing else to do.
if my_task._has_state(Task.COMPLETED):
return False
if my_task._has_state(Task.READY):
return True
# Retrieve a list of all activated tasks from the associated
# task that did the conditional parallel split.
split_task = my_task._find_ancestor_from_name(self.split_task)
if split_task is None:
msg = 'Join with %s, which was not reached' % self.split_task
raise WorkflowException(self, msg)
tasks = split_task.task_spec._get_activated_threads(split_task)
# The default threshold is the number of threads that were started.
threshold = valueof(my_task, self.threshold)
if threshold is None:
threshold = len(tasks)
# Look up which tasks have already completed.
waiting_tasks = []
completed = 0
for task in tasks:
# Refresh path prediction.
task.task_spec._predict(task)
if self._branch_is_complete(task):
completed += 1
else:
waiting_tasks.append(task)
# If the threshold was reached, get ready to fire.
if completed >= threshold:
# If this is a cancelling join, cancel all incoming branches,
# except for the one that just completed.
if self.cancel_remaining:
for task in waiting_tasks:
task.cancel()
return True
# We do NOT set the task state to COMPLETED, because in
# case all other incoming tasks get cancelled (or never reach
# the ThreadMerge for other reasons, such as reaching a stub branch),
# we need to revisit it.
return False
def _add_child(self, task_spec, state=MAYBE):
"""
Adds a new child and assigns the given TaskSpec to it.
:type task_spec: TaskSpec
:param task_spec: The task spec that is assigned to the new child.
:type state: integer
:param state: The bitmask of states for the new child.
:rtype: Task
:returns: The new child task.
"""
if task_spec is None:
raise ValueError(self, '_add_child() requires a TaskSpec')
if self._is_predicted() and state & self.PREDICTED_MASK == 0:
msg = 'Attempt to add non-predicted child to predicted task'
raise WorkflowException(self.task_spec, msg)
task = Task(self.workflow, task_spec, self, state=state)
task.thread_id = self.thread_id
if state == self.READY:
task._ready()
return task
def _setstate(self, value, force=False):
"""
Setting force to True allows for changing a state after it
COMPLETED. This would otherwise be invalid.
"""
if self._state == value:
return
if value < self._state and not force:
raise WorkflowException(self.task_spec,
'state went from %s to %s!' % (
self.get_state_name(),
self.state_names[value]))
if __debug__:
old = self.get_state_name()
self._state = value
if __debug__:
self.log.append("Moving '%s' from %s to %s" % (self.get_name(),
old, self.get_state_name()))
self.state_history.append(value)
LOG.debug("Moving '%s' (spec=%s) from %s to %s" % (self.get_name(),
self.task_spec.name, old, self.get_state_name()))
def test(self):
TaskSpec.test(self)
if self.file is not None and not os.path.exists(self.file):
raise WorkflowException(self,
'File does not exist: %s' % self.file)
def _update_children(self, task_specs, state=None):
"""
This method adds one child for each given TaskSpec, unless that
child already exists.
The state of COMPLETED tasks is never changed.
If this method is passed a state:
- The state of TRIGGERED tasks is not changed.
- The state for all children is set to the given value.
If this method is not passed a state:
- The state for all children is updated by calling the child's
_update_state() method.
If the task currently has a child that is not given in the TaskSpec,
the child is removed.
It is an error if the task has a non-LIKELY child that is
not given in the TaskSpecs.
@type task_specs: list(TaskSpec)
@param task_specs: The list of task specs that may become children.
@type state: integer
@param state: The bitmask of states for newly added children.
"""
LOG.debug("Updating children for %s" % self.get_name())
if task_specs is None:
raise ValueError('"task_specs" argument is None')
if type(task_specs) != type([]):
task_specs = [task_specs]
# Create a list of all children that are no longer needed, and
# set the state of all others.
add = task_specs[:]
remove = []
for child in self.children:
# Must not be TRIGGERED or COMPLETED.
if child._has_state(Task.TRIGGERED):
if state is None:
child.task_spec._update_state(child)
continue
if child._is_finished():
add.remove(child.task_spec)
continue
# Check whether the item needs to be added or removed.
if child.task_spec not in add:
if not self._is_definite():
msg = 'Attempt to remove non-predicted %s' % child.get_name()
raise WorkflowException(self, msg)
remove.append(child)
continue
add.remove(child.task_spec)
# Update the state.
if state is not None:
child.state = state
else:
child.task_spec._update_state(child)
# Remove all children that are no longer specified.
for child in remove:
self.children.remove(child)
# Add a new child for each of the remaining tasks.
for task_spec in add:
if state is not None:
self._add_child(task_spec, state)
else:
child = self._add_child(task_spec, self.LIKELY)
task_spec._update_state(child)
def _connect_notify(self, task_spec):
"""
Called by the previous task to let us know that it exists.
"""
raise WorkflowException(self, 'StartTask can not have any inputs.')