def test_master_success_influences_other_experiment_workers_status(self): with patch('runner.tasks.experiments.start_experiment.delay' ) as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiment = ExperimentFactory() assert ExperimentLifeCycle.is_done(experiment.last_status) is False # Add jobs master = ExperimentJobFactory(experiment=experiment, role=TaskType.MASTER) assert JobLifeCycle.is_done(master.last_status) is False workers = [ ExperimentJobFactory(experiment=experiment, role=TaskType.WORKER) for _ in range(2) ] for worker in workers: worker.refresh_from_db() assert JobLifeCycle.is_done(worker.last_status) is False # Set master to succeeded ExperimentJobStatusFactory(job=master, status=JobLifeCycle.SUCCEEDED) # All worker should have a success status for worker in workers: worker.refresh_from_db() assert worker.last_status == JobLifeCycle.SUCCEEDED # Experiment last status should be success experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.SUCCEEDED
def update_job_containers(event, status, job_container_name): if JobLifeCycle.is_done(status): # Remove the job monitoring job_uuid = event['metadata']['labels']['job_uuid'] logger.info('Stop monitoring job_uuid: %s', job_uuid) RedisJobContainers.remove_job(job_uuid) if event['status']['container_statuses'] is None: return def get_container_id(container_id): if not container_id: return None if container_id.startswith('docker://'): return container_id[len('docker://'):] return container_id for container_status in event['status']['container_statuses']: if container_status['name'] != job_container_name: continue container_id = get_container_id(container_status['container_id']) if container_id: job_uuid = event['metadata']['labels']['job_uuid'] if container_status['state']['running'] is not None: logger.info('Monitoring (container_id, job_uuid): (%s, %s)', container_id, job_uuid) RedisJobContainers.monitor(container_id=container_id, job_uuid=job_uuid) else: RedisJobContainers.remove_container(container_id=container_id)
def calculated_status(self): master_status = self.jobs.filter(role=TaskType.MASTER)[0].last_status calculated_status = master_status if JobLifeCycle.is_done( master_status) else None if calculated_status is None: calculated_status = ExperimentLifeCycle.jobs_status( self.last_job_statuses) if calculated_status is None: return self.last_status return calculated_status
def _set_status(self, status_model, logger, status, message=None, details=None): current_status = self.last_status # We should not update statuses anymore if JobLifeCycle.is_done(current_status): logger.info('Received a new status `{}` for job `{}`. ' 'But the job is already done with status `{}`'.format( status, self.unique_name, current_status)) return False if status != current_status: # Add new status to the job status_model.objects.create(job=self, status=status, message=message, details=details) return True return False
def jobs_starting(job_statuses): return any([ True if JobLifeCycle.is_starting(job_status) else False for job_status in job_statuses ])