def test_master_success_influences_other_experiment_workers_status(self):
with patch('scheduler.tasks.experiments.experiments_build.apply_async'
) as _: # noqa
with patch.object(Experiment, 'set_status') as _: # noqa
experiment = ExperimentFactory()
assert ExperimentLifeCycle.is_done(experiment.last_status) is False
# Add jobs
master = ExperimentJobFactory(experiment=experiment,
role=TaskType.MASTER)
assert JobLifeCycle.is_done(master.last_status) is False
workers = [
ExperimentJobFactory(experiment=experiment, role=TaskType.WORKER)
for _ in range(2)
]
for worker in workers:
worker.refresh_from_db()
assert JobLifeCycle.is_done(worker.last_status) is False
# Set master to succeeded
ExperimentJobStatusFactory(job=master, status=JobLifeCycle.SUCCEEDED)
# All worker should have a success status
for worker in workers:
worker.refresh_from_db()
assert worker.last_status == JobLifeCycle.SUCCEEDED
# Experiment last status should be success
experiment.refresh_from_db()
assert experiment.last_status == ExperimentLifeCycle.SUCCEEDED
def test_sync_experiments_and_jobs_statuses(self):
with patch('scheduler.tasks.experiments.experiments_build.apply_async') as _: # noqa
with patch.object(Experiment, 'set_status') as _: # noqa
experiments = [ExperimentFactory() for _ in range(3)]
done_xp, no_jobs_xp, xp_with_jobs = experiments
# Set done status
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
ExperimentStatusFactory(experiment=done_xp, status=JobLifeCycle.FAILED)
# Create jobs for xp_with_jobs and update status, and do not update the xp status
with patch.object(Experiment, 'set_status') as _: # noqa
job = ExperimentJobFactory(experiment=xp_with_jobs)
ExperimentJobStatusFactory(job=job, status=JobLifeCycle.RUNNING)
xp_with_jobs.refresh_from_db()
assert xp_with_jobs.last_status is None
# Mock sync experiments and jobs constants
with patch('scheduler.tasks.experiments.'
'experiments_check_status.apply_async') as check_status_mock:
sync_experiments_and_jobs_statuses()
assert check_status_mock.call_count == 1
# Call sync experiments and jobs constants
sync_experiments_and_jobs_statuses()
done_xp.refresh_from_db()
no_jobs_xp.refresh_from_db()
xp_with_jobs.refresh_from_db()
assert done_xp.last_status == ExperimentLifeCycle.FAILED
assert no_jobs_xp.last_status is None
assert xp_with_jobs.last_status == ExperimentLifeCycle.RUNNING