def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
with transaction.atomic():
jobs_to_pending = []
# Retrieve locked job models
for job_model in Job.objects.get_locked_jobs(self._job_ids):
if job_model.can_be_uncanceled():
jobs_to_pending.append(job_model)
# Uncancel jobs by setting them to PENDING
if jobs_to_pending:
job_ids = Job.objects.update_jobs_to_pending(jobs_to_pending, self.when)
logger.info('Set %d job(s) to PENDING status', len(job_ids))
# Create messages to update recipes
from recipe.messages.update_recipes import create_update_recipes_messages_from_jobs
self.new_messages.extend(create_update_recipes_messages_from_jobs(self._job_ids))
# Send messages to update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
self.new_messages.extend(create_update_recipe_metrics_messages_from_jobs(self._job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
with transaction.atomic():
jobs_to_canceled = []
# Retrieve locked job models
for job_model in Job.objects.get_locked_jobs(self._job_ids):
if job_model.can_be_canceled():
jobs_to_canceled.append(job_model)
# Update jobs that need status set to CANCELED
if jobs_to_canceled:
job_ids = Job.objects.update_jobs_to_canceled(jobs_to_canceled, self.when)
from queue.models import Queue
Queue.objects.cancel_queued_jobs(job_ids)
logger.info('Set %d job(s) to CANCELED status', len(job_ids))
# Need to update recipes of canceled jobs so that dependent jobs are BLOCKED and update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
from recipe.messages.update_recipes import create_update_recipes_messages_from_jobs
self.new_messages.extend(create_update_recipe_metrics_messages_from_jobs(self._job_ids))
self.new_messages.extend(create_update_recipes_messages_from_jobs(self._job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
with transaction.atomic():
jobs_to_blocked = []
# Retrieve locked job models
for job_model in Job.objects.get_locked_jobs(
self._blocked_job_ids):
if not job_model.last_status_change or job_model.last_status_change < self.status_change:
# Status update is not old, so perform the update
jobs_to_blocked.append(job_model)
# Update jobs that need status set to BLOCKED
if jobs_to_blocked:
job_ids = Job.objects.update_jobs_to_blocked(
jobs_to_blocked, self.status_change)
logger.info('Set %d job(s) to BLOCKED status', len(job_ids))
# Send messages to update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
self.new_messages.extend(
create_update_recipe_metrics_messages_from_jobs(
self._blocked_job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
job_ids = []
for job_list in self._running_jobs.values():
for job_tuple in job_list:
job_ids.append(job_tuple[0])
with transaction.atomic():
# Retrieve locked job models
job_models = {}
for job in Job.objects.get_locked_jobs(job_ids):
job_models[job.id] = job
jobs_to_running = []
for node_id, job_list in self._running_jobs.items():
job_ids_for_node_update = []
for job_tuple in job_list:
job_id = job_tuple[0]
exe_num = job_tuple[1]
job_model = job_models[job_id]
if job_model.num_exes != exe_num:
continue # Execution number does not match so this update is out of date, ignore job
# Execution numbers match, so this job needs to have its node_id set
job_ids_for_node_update.append(job_id)
# Job will later be set to RUNNING
jobs_to_running.append(job_model)
# Update jobs for this node
if job_ids_for_node_update:
Job.objects.update_jobs_node(job_ids_for_node_update,
node_id, self._started)
# Update jobs that need status set to RUNNING
if jobs_to_running:
running_job_ids = Job.objects.update_jobs_to_running(
jobs_to_running, self._started)
logger.info('Set %d job(s) to RUNNING status',
len(running_job_ids))
# Send messages to update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
self.new_messages.extend(
create_update_recipe_metrics_messages_from_jobs(job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
when = now()
job_ids = [job.job_id for job in self._completed_jobs]
with transaction.atomic():
# Retrieve locked job models
job_models = {}
for job_model in Job.objects.get_locked_jobs(job_ids):
job_models[job_model.id] = job_model
jobs_to_complete = []
for completed_job in self._completed_jobs:
job_model = job_models[completed_job.job_id]
# If execution number does not match, then this update is obsolete
if job_model.num_exes != completed_job.exe_num:
# Ignore this job
continue
jobs_to_complete.append(job_model)
job_ids_to_complete = [job.id for job in jobs_to_complete]
# Update jobs to completed
completed_job_ids = []
if jobs_to_complete:
completed_job_ids = Job.objects.update_jobs_to_completed(
jobs_to_complete, self.ended)
logger.info('Set %d job(s) to COMPLETED status',
len(completed_job_ids))
# Create messages for jobs that are both COMPLETED and have output
if job_ids_to_complete:
msgs = process_completed_jobs_with_output(
job_ids_to_complete, when)
self.new_messages.extend(msgs)
# Send messages to update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
self.new_messages.extend(
create_update_recipe_metrics_messages_from_jobs(job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
job_ids = []
for queued_job in self._queued_jobs:
job_ids.append(queued_job.job_id)
with transaction.atomic():
# Retrieve locked job models
job_models = {}
for job in Job.objects.get_locked_jobs(job_ids):
job_models[job.id] = job
jobs_to_queue = []
for queued_job in self._queued_jobs:
job_model = job_models[queued_job.job_id]
# If execution number does not match, then this update is obsolete
if job_model.num_exes != queued_job.exe_num:
# Ignore this job
continue
jobs_to_queue.append(job_model)
# Queue jobs
if jobs_to_queue:
queued_job_ids = Queue.objects.queue_jobs(
jobs_to_queue,
requeue=self.requeue,
priority=self.priority)
logger.info('Queued %d job(s)', len(queued_job_ids))
# Send messages to update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
self.new_messages.extend(
create_update_recipe_metrics_messages_from_jobs(job_ids))
return True
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
from queue.messages.queued_jobs import create_queued_jobs_messages, QueuedJob
job_ids = []
for job_list in self._failed_jobs.values():
for failed_job in job_list:
job_ids.append(failed_job.job_id)
root_recipe_ids = set()
with transaction.atomic():
# Retrieve locked job models
job_models = {}
for job in Job.objects.get_locked_jobs(job_ids):
job_models[job.id] = job
if job.root_recipe_id:
root_recipe_ids.add(job.root_recipe_id)
# Get job models with related fields
# TODO: once long running job types are gone, the related fields are not needed
for job in Job.objects.get_jobs_with_related(job_ids):
job_models[job.id] = job
jobs_to_retry = []
all_failed_job_ids = []
for error_id, job_list in self._failed_jobs.items():
error = get_error(error_id)
jobs_to_fail = []
for failed_job in job_list:
job_model = job_models[failed_job.job_id]
# If job cannot be failed or execution number does not match, then this update is obsolete
if not job_model.can_be_failed() or job_model.num_exes != failed_job.exe_num:
# Ignore this job
continue
# Re-try job if error supports re-try and there are more tries left
retry = error.should_be_retried and job_model.num_exes < job_model.max_tries
# Also re-try long running jobs
retry = retry or job_model.job_type.is_long_running
# Do not re-try superseded jobs
retry = retry and not job_model.is_superseded
if retry:
jobs_to_retry.append(QueuedJob(job_model.id, job_model.num_exes))
else:
jobs_to_fail.append(job_model)
# Update jobs that failed with this error
if jobs_to_fail:
failed_job_ids = Job.objects.update_jobs_to_failed(jobs_to_fail, error_id, self.ended)
logger.info('Set %d job(s) to FAILED status with error %s', len(failed_job_ids), error.name)
all_failed_job_ids.extend(failed_job_ids)
# Need to update recipes of failed jobs so that dependent jobs are BLOCKED
if root_recipe_ids:
from recipe.messages.update_recipe import create_update_recipe_messages_from_node
self.new_messages.extend(create_update_recipe_messages_from_node(root_recipe_ids))
# Place jobs to retry back onto the queue
if jobs_to_retry:
self.new_messages.extend(create_queued_jobs_messages(jobs_to_retry, requeue=True))
# Send messages to update recipe metrics
from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs
self.new_messages.extend(create_update_recipe_metrics_messages_from_jobs(job_ids))
return True
