def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ with transaction.atomic(): jobs_to_pending = [] # Retrieve locked job models for job_model in Job.objects.get_locked_jobs(self._job_ids): if job_model.can_be_uncanceled(): jobs_to_pending.append(job_model) # Uncancel jobs by setting them to PENDING if jobs_to_pending: job_ids = Job.objects.update_jobs_to_pending(jobs_to_pending, self.when) logger.info('Set %d job(s) to PENDING status', len(job_ids)) # Create messages to update recipes from recipe.messages.update_recipes import create_update_recipes_messages_from_jobs self.new_messages.extend(create_update_recipes_messages_from_jobs(self._job_ids)) # Send messages to update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs self.new_messages.extend(create_update_recipe_metrics_messages_from_jobs(self._job_ids)) return True
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ with transaction.atomic(): jobs_to_canceled = [] # Retrieve locked job models for job_model in Job.objects.get_locked_jobs(self._job_ids): if job_model.can_be_canceled(): jobs_to_canceled.append(job_model) # Update jobs that need status set to CANCELED if jobs_to_canceled: job_ids = Job.objects.update_jobs_to_canceled(jobs_to_canceled, self.when) from queue.models import Queue Queue.objects.cancel_queued_jobs(job_ids) logger.info('Set %d job(s) to CANCELED status', len(job_ids)) # Need to update recipes of canceled jobs so that dependent jobs are BLOCKED and update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs from recipe.messages.update_recipes import create_update_recipes_messages_from_jobs self.new_messages.extend(create_update_recipe_metrics_messages_from_jobs(self._job_ids)) self.new_messages.extend(create_update_recipes_messages_from_jobs(self._job_ids)) return True
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ with transaction.atomic(): jobs_to_blocked = [] # Retrieve locked job models for job_model in Job.objects.get_locked_jobs( self._blocked_job_ids): if not job_model.last_status_change or job_model.last_status_change < self.status_change: # Status update is not old, so perform the update jobs_to_blocked.append(job_model) # Update jobs that need status set to BLOCKED if jobs_to_blocked: job_ids = Job.objects.update_jobs_to_blocked( jobs_to_blocked, self.status_change) logger.info('Set %d job(s) to BLOCKED status', len(job_ids)) # Send messages to update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs self.new_messages.extend( create_update_recipe_metrics_messages_from_jobs( self._blocked_job_ids)) return True
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ job_ids = [] for job_list in self._running_jobs.values(): for job_tuple in job_list: job_ids.append(job_tuple[0]) with transaction.atomic(): # Retrieve locked job models job_models = {} for job in Job.objects.get_locked_jobs(job_ids): job_models[job.id] = job jobs_to_running = [] for node_id, job_list in self._running_jobs.items(): job_ids_for_node_update = [] for job_tuple in job_list: job_id = job_tuple[0] exe_num = job_tuple[1] job_model = job_models[job_id] if job_model.num_exes != exe_num: continue # Execution number does not match so this update is out of date, ignore job # Execution numbers match, so this job needs to have its node_id set job_ids_for_node_update.append(job_id) # Job will later be set to RUNNING jobs_to_running.append(job_model) # Update jobs for this node if job_ids_for_node_update: Job.objects.update_jobs_node(job_ids_for_node_update, node_id, self._started) # Update jobs that need status set to RUNNING if jobs_to_running: running_job_ids = Job.objects.update_jobs_to_running( jobs_to_running, self._started) logger.info('Set %d job(s) to RUNNING status', len(running_job_ids)) # Send messages to update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs self.new_messages.extend( create_update_recipe_metrics_messages_from_jobs(job_ids)) return True
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ when = now() job_ids = [job.job_id for job in self._completed_jobs] with transaction.atomic(): # Retrieve locked job models job_models = {} for job_model in Job.objects.get_locked_jobs(job_ids): job_models[job_model.id] = job_model jobs_to_complete = [] for completed_job in self._completed_jobs: job_model = job_models[completed_job.job_id] # If execution number does not match, then this update is obsolete if job_model.num_exes != completed_job.exe_num: # Ignore this job continue jobs_to_complete.append(job_model) job_ids_to_complete = [job.id for job in jobs_to_complete] # Update jobs to completed completed_job_ids = [] if jobs_to_complete: completed_job_ids = Job.objects.update_jobs_to_completed( jobs_to_complete, self.ended) logger.info('Set %d job(s) to COMPLETED status', len(completed_job_ids)) # Create messages for jobs that are both COMPLETED and have output if job_ids_to_complete: msgs = process_completed_jobs_with_output( job_ids_to_complete, when) self.new_messages.extend(msgs) # Send messages to update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs self.new_messages.extend( create_update_recipe_metrics_messages_from_jobs(job_ids)) return True
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ job_ids = [] for queued_job in self._queued_jobs: job_ids.append(queued_job.job_id) with transaction.atomic(): # Retrieve locked job models job_models = {} for job in Job.objects.get_locked_jobs(job_ids): job_models[job.id] = job jobs_to_queue = [] for queued_job in self._queued_jobs: job_model = job_models[queued_job.job_id] # If execution number does not match, then this update is obsolete if job_model.num_exes != queued_job.exe_num: # Ignore this job continue jobs_to_queue.append(job_model) # Queue jobs if jobs_to_queue: queued_job_ids = Queue.objects.queue_jobs( jobs_to_queue, requeue=self.requeue, priority=self.priority) logger.info('Queued %d job(s)', len(queued_job_ids)) # Send messages to update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs self.new_messages.extend( create_update_recipe_metrics_messages_from_jobs(job_ids)) return True
def execute(self): """See :meth:`messaging.messages.message.CommandMessage.execute` """ from queue.messages.queued_jobs import create_queued_jobs_messages, QueuedJob job_ids = [] for job_list in self._failed_jobs.values(): for failed_job in job_list: job_ids.append(failed_job.job_id) root_recipe_ids = set() with transaction.atomic(): # Retrieve locked job models job_models = {} for job in Job.objects.get_locked_jobs(job_ids): job_models[job.id] = job if job.root_recipe_id: root_recipe_ids.add(job.root_recipe_id) # Get job models with related fields # TODO: once long running job types are gone, the related fields are not needed for job in Job.objects.get_jobs_with_related(job_ids): job_models[job.id] = job jobs_to_retry = [] all_failed_job_ids = [] for error_id, job_list in self._failed_jobs.items(): error = get_error(error_id) jobs_to_fail = [] for failed_job in job_list: job_model = job_models[failed_job.job_id] # If job cannot be failed or execution number does not match, then this update is obsolete if not job_model.can_be_failed() or job_model.num_exes != failed_job.exe_num: # Ignore this job continue # Re-try job if error supports re-try and there are more tries left retry = error.should_be_retried and job_model.num_exes < job_model.max_tries # Also re-try long running jobs retry = retry or job_model.job_type.is_long_running # Do not re-try superseded jobs retry = retry and not job_model.is_superseded if retry: jobs_to_retry.append(QueuedJob(job_model.id, job_model.num_exes)) else: jobs_to_fail.append(job_model) # Update jobs that failed with this error if jobs_to_fail: failed_job_ids = Job.objects.update_jobs_to_failed(jobs_to_fail, error_id, self.ended) logger.info('Set %d job(s) to FAILED status with error %s', len(failed_job_ids), error.name) all_failed_job_ids.extend(failed_job_ids) # Need to update recipes of failed jobs so that dependent jobs are BLOCKED if root_recipe_ids: from recipe.messages.update_recipe import create_update_recipe_messages_from_node self.new_messages.extend(create_update_recipe_messages_from_node(root_recipe_ids)) # Place jobs to retry back onto the queue if jobs_to_retry: self.new_messages.extend(create_queued_jobs_messages(jobs_to_retry, requeue=True)) # Send messages to update recipe metrics from recipe.messages.update_recipe_metrics import create_update_recipe_metrics_messages_from_jobs self.new_messages.extend(create_update_recipe_metrics_messages_from_jobs(job_ids)) return True