def _do_requeue(self, job_id, old_status, new_status) -> str:
"""Re-queue all tasks of the job, and the job itself.
:returns: the new job status, if this status transition should be
followed by another one.
"""
if old_status == 'under-construction':
# Nothing to do, the job compiler has just finished its work; the tasks have
# already been set to 'queued' status.
self._log.debug('Ignoring job status change %r -> %r', old_status, new_status)
return ''
if old_status == 'completed':
# Re-queue all tasks except cancel-requested; those should remain
# untouched; changing their status is only allowed by managers, to avoid
# race conditions.
query = {'status': {'$ne': 'cancel-requested'}} # type: typing.Dict[str, typing.Any]
else:
# Re-queue any non-completed task. Cancel-requested tasks should also be
# untouched; changing their status is only allowed by managers, to avoid
# race conditions.
query = {'status': {'$nin': ['completed', 'cancel-requested']}}
# Update the tasks.
query['job'] = job_id
current_flamenco.update_status_q('tasks', query, 'queued',
extra_unset={'failed_by_workers'})
return 'queued'
def _do_requeue(self, job_id, old_status, new_status) -> str:
"""Re-queue all tasks of the job, and the job itself.
:returns: the new job status, if this status transition should be
followed by another one.
"""
if old_status == 'under-construction':
# Nothing to do, the job compiler has just finished its work; the tasks have
# already been set to 'queued' status.
self._log.debug('Ignoring job status change %r -> %r', old_status,
new_status)
return ''
if old_status == 'completed':
# Re-queue all tasks except cancel-requested; those should remain
# untouched; changing their status is only allowed by managers, to avoid
# race conditions.
query = {
'status': {
'$ne': 'cancel-requested'
}
} # type: typing.Dict[str, typing.Any]
else:
# Re-queue any non-completed task. Cancel-requested tasks should also be
# untouched; changing their status is only allowed by managers, to avoid
# race conditions.
query = {'status': {'$nin': ['completed', 'cancel-requested']}}
# Update the tasks.
query['job'] = job_id
current_flamenco.update_status_q('tasks',
query,
'queued',
extra_unset={'failed_by_workers'})
return 'queued'
def api_set_task_status_for_job(self, job_id: bson.ObjectId, from_status: str, to_status: str,
*, now: datetime.datetime = None):
"""Updates the task status for all tasks of a job that have a particular status."""
self._log.info('Flipping all tasks of job %s from status %r to %r',
job_id, from_status, to_status)
from flamenco import current_flamenco
current_flamenco.update_status_q('tasks',
{'job': job_id, 'status': from_status},
to_status,
now=now)
def _do_cancel_tasks(self, job_id, old_status, new_status) -> str:
"""Directly cancel any task that might run in the future.
Only cancels tasks that haven't been touched by a manager yet;
otherwise it requests the Manager to cancel the tasks.
:returns: the next job status, if a status change is required.
"""
current_flamenco.update_status_q('tasks', {
'job': job_id,
'status': 'queued'
}, 'canceled')
# Request cancel of any task that might run on the manager.
cancelreq_result = current_flamenco.update_status_q(
'tasks', {
'job': job_id,
'status': {
'$in': ['active', 'claimed-by-manager', 'soft-failed']
}
}, 'cancel-requested')
# Update the activity of all the tasks we just cancelled (or requested cancellation),
# so that users can tell why they were cancelled.
current_flamenco.task_manager.api_set_activity(
{
'job': job_id,
'status': {
'$in': ['cancel-requested', 'canceled']
},
'activity': {
'$exists': False
}
}, 'Server cancelled this task because the job got status %r.' %
new_status)
# If the new status is xxx-requested, and no tasks were marked as cancel-requested,
# we can directly transition the job to 'xxx', without waiting for more task
# updates.
if new_status.endswith(
'-requested') and cancelreq_result.modified_count == 0:
goto_status = new_status.replace('-requested', 'ed')
self._log.info(
'handle_job_status_change(%s, %s, %s): no cancel-requested tasks, '
'so transitioning directly to %s', job_id, old_status,
new_status, goto_status)
return goto_status
return ''
def _do_cancel_tasks(self, job_id, old_status, new_status) -> str:
"""Directly cancel any task that might run in the future.
Only cancels tasks that haven't been touched by a manager yet;
otherwise it requests the Manager to cancel the tasks.
:returns: the next job status, if a status change is required.
"""
current_flamenco.update_status_q(
'tasks',
{'job': job_id, 'status': 'queued'},
'canceled')
# Request cancel of any task that might run on the manager.
cancelreq_result = current_flamenco.update_status_q(
'tasks',
{'job': job_id, 'status': {'$in': ['active', 'claimed-by-manager', 'soft-failed']}},
'cancel-requested')
# Update the activity of all the tasks we just cancelled (or requested cancellation),
# so that users can tell why they were cancelled.
current_flamenco.task_manager.api_set_activity(
{'job': job_id,
'status': {'$in': ['cancel-requested', 'canceled']},
'activity': {'$exists': False}},
'Server cancelled this task because the job got status %r.' % new_status
)
# If the new status is xxx-requested, and no tasks were marked as cancel-requested,
# we can directly transition the job to 'xxx', without waiting for more task
# updates.
if new_status.endswith('-requested') and cancelreq_result.modified_count == 0:
goto_status = new_status.replace('-requested', 'ed')
self._log.info('handle_job_status_change(%s, %s, %s): no cancel-requested tasks, '
'so transitioning directly to %s',
job_id, old_status, new_status, goto_status)
return goto_status
return ''
def handle_job_status_change(self, job_id, old_status, new_status):
"""Updates task statuses based on this job status transition."""
query = None
to_status = None
if new_status in {'completed', 'canceled'}:
# Nothing to do; this will happen as a response to all tasks receiving this status.
return
elif new_status == 'active':
# Nothing to do; this happens when a task gets started, which has nothing to
# do with other tasks in the job.
return
elif new_status in {'cancel-requested', 'failed'}:
# Directly cancel any task that might run in the future, but is not touched by
# a manager yet.
current_flamenco.update_status_q(
'tasks',
{'job': job_id, 'status': 'queued'},
'canceled')
# Request cancel of any task that might run on the manager.
cancelreq_result = current_flamenco.update_status_q(
'tasks',
{'job': job_id, 'status': {'$in': ['active', 'claimed-by-manager']}},
'cancel-requested')
# Update the activity of all the tasks we just cancelled (or requested cancellation),
# so that users can tell why they were cancelled.
current_flamenco.task_manager.api_set_activity(
{'job': job_id,
'status': {'$in': ['cancel-requested', 'canceled']},
'activity': {'$exists': False}},
'Server cancelled this task because the job got status %r.' % new_status
)
# If the new status is cancel-requested, and no tasks were marked as cancel-requested,
# we can directly transition the job to 'canceled', without waiting for more task
# updates.
if new_status == 'cancel-requested' and cancelreq_result.modified_count == 0:
self._log.info('handle_job_status_change(%s, %s, %s): no cancel-requested tasks, '
'so transitioning directly to canceled',
job_id, old_status, new_status)
self.api_set_job_status(job_id, 'canceled')
return
elif new_status == 'queued':
if old_status == 'under-construction':
# Nothing to do, the job compiler has just finished its work; the tasks have
# already been set to 'queued' status.
self._log.debug('Ignoring job status change %r -> %r', old_status, new_status)
return
if old_status == 'completed':
# Re-queue all tasks except cancel-requested; those should remain
# untouched; changing their status is only allowed by managers, to avoid
# race conditions.
query = {'status': {'$ne': 'cancel-requested'}}
else:
# Re-queue any non-completed task. Cancel-requested tasks should also be
# untouched; changing their status is only allowed by managers, to avoid
# race conditions.
query = {'status': {'$nin': ['completed', 'cancel-requested']}}
to_status = 'queued'
if query is None:
self._log.debug('Job %s status change from %s to %s has no effect on tasks.',
job_id, old_status, new_status)
return
if to_status is None:
self._log.error('Job %s status change from %s to %s has to_status=None, aborting.',
job_id, old_status, new_status)
return
# Update the tasks.
query['job'] = job_id
current_flamenco.update_status_q('tasks', query, to_status)