def test_master_success_influences_other_experiment_workers_status(self): with patch('scheduler.tasks.experiments.experiments_build.apply_async' ) as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiment = ExperimentFactory() assert ExperimentLifeCycle.is_done(experiment.last_status) is False # Add jobs master = ExperimentJobFactory(experiment=experiment, role=TaskType.MASTER) assert JobLifeCycle.is_done(master.last_status) is False workers = [ ExperimentJobFactory(experiment=experiment, role=TaskType.WORKER) for _ in range(2) ] for worker in workers: worker.refresh_from_db() assert JobLifeCycle.is_done(worker.last_status) is False # Set master to succeeded ExperimentJobStatusFactory(job=master, status=JobLifeCycle.SUCCEEDED) # All worker should have a success status for worker in workers: worker.refresh_from_db() assert worker.last_status == JobLifeCycle.SUCCEEDED # Experiment last status should be success experiment.refresh_from_db() assert experiment.last_status == ExperimentLifeCycle.SUCCEEDED
def test_sync_experiments_and_jobs_statuses(self): with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa with patch.object(Experiment, 'set_status') as _: # noqa experiments = [ExperimentFactory() for _ in range(3)] done_xp, no_jobs_xp, xp_with_jobs = experiments # Set done status with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa ExperimentStatusFactory(experiment=done_xp, status=JobLifeCycle.FAILED) # Create jobs for xp_with_jobs and update status, and do not update the xp status with patch.object(Experiment, 'set_status') as _: # noqa job = ExperimentJobFactory(experiment=xp_with_jobs) ExperimentJobStatusFactory(job=job, status=JobLifeCycle.RUNNING) xp_with_jobs.refresh_from_db() assert xp_with_jobs.last_status is None # Mock sync experiments and jobs constants with patch('scheduler.tasks.experiments.' 'experiments_check_status.apply_async') as check_status_mock: sync_experiments_and_jobs_statuses() assert check_status_mock.call_count == 1 # Call sync experiments and jobs constants sync_experiments_and_jobs_statuses() done_xp.refresh_from_db() no_jobs_xp.refresh_from_db() xp_with_jobs.refresh_from_db() assert done_xp.last_status == ExperimentLifeCycle.FAILED assert no_jobs_xp.last_status is None assert xp_with_jobs.last_status == ExperimentLifeCycle.RUNNING