def _reconcile_running_jobs(self): """Looks up all currently running jobs in the database and sets them up to be reconciled with Mesos""" # List of task IDs to reconcile task_ids = [] # Query for job executions that are running job_exes = JobExecution.objects.get_running_job_exes() # Find current task IDs for running executions for job_exe in job_exes: running_job_exe = running_job_mgr.get_job_exe(job_exe.id) if running_job_exe: task = running_job_exe.current_task if task: task_ids.append(task.id) else: # Fail any executions that the scheduler has lost Queue.objects.handle_job_failure(job_exe.id, now(), [], Error.objects.get_builtin_error('scheduler-lost')) # Send task IDs to reconciliation thread recon_mgr.add_task_ids(task_ids)
def statusUpdate(self, driver, status): """ Invoked when the status of a task has changed (e.g., a slave is lost and so the task is lost, a task finishes and an executor sends a status update saying so, etc.) Note that returning from this callback acknowledges receipt of this status update. If for whatever reason the scheduler aborts during this callback (or the process exits) another status update will be delivered. Note, however, that this is currently not true if the slave sending the status update is lost or fails during that time. See documentation for :meth:`mesos_api.mesos.Scheduler.statusUpdate`. """ started = now() model = utils.create_task_update_model(status) mesos_status = model.status task_update = TaskStatusUpdate(model, utils.get_status_agent_id(status)) task_id = task_update.task_id if mesos_status == 'TASK_LOST': logger.warning('Status update for task %s: %s', task_id, mesos_status) else: logger.info('Status update for task %s: %s', task_id, mesos_status) # Since we have a status update for this task, remove it from reconciliation set recon_mgr.remove_task_id(task_id) # Hand off task update to be saved in the database task_update_mgr.add_task_update(model) if task_id.startswith(CLEANUP_TASK_ID_PREFIX): cleanup_mgr.handle_task_update(task_update) else: job_exe_id = JobExecution.get_job_exe_id(task_id) try: running_job_exe = running_job_mgr.get_job_exe(job_exe_id) if running_job_exe: running_job_exe.task_update(task_update) # Remove finished job execution if running_job_exe.is_finished(): running_job_mgr.remove_job_exe(job_exe_id) cleanup_mgr.add_job_execution(running_job_exe) else: # Scheduler doesn't have any knowledge of this job execution Queue.objects.handle_job_failure(job_exe_id, now(), [], Error.objects.get_builtin_error('scheduler-lost')) except Exception: logger.exception('Error handling status update for job execution: %s', job_exe_id) # Error handling status update, add task so it can be reconciled recon_mgr.add_task_ids([task_id]) duration = now() - started msg = 'Scheduler statusUpdate() took %.3f seconds' if duration > ScaleScheduler.DATABASE_WARN_THRESHOLD: logger.warning(msg, duration.total_seconds()) else: logger.debug(msg, duration.total_seconds())