def test_stopping_group_stops_iteration(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
self.assertEqual(
mock_fct.call_count,
math.ceil(experiment_group.experiments.count() /
conf.get('GROUP_CHUNKS')) + 1)
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp,
status=ExperimentLifeCycle.SUCCEEDED)
# Mark group as stopped
ExperimentGroupStatusFactory(experiment_group=experiment_group,
status=ExperimentGroupLifeCycle.STOPPED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async'
) as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 0
def test_stopping_group_stops_iteration(self):
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp,
status=ExperimentLifeCycle.SUCCEEDED)
# Mark group as stopped
ExperimentGroupStatusFactory(experiment_group=experiment_group,
status=ExperimentGroupLifeCycle.STOPPED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async'
) as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 0
def test_status_update_results_in_new_updated_at_datetime(self):
experiment_group = ExperimentGroupFactory()
updated_at = experiment_group.updated_at
# Create new status
ExperimentGroupStatusFactory(experiment_group=experiment_group,
status=ExperimentGroupLifeCycle.RUNNING)
experiment_group.refresh_from_db()
assert updated_at < experiment_group.updated_at
updated_at = experiment_group.updated_at
# Create status Using set_status
experiment_group.set_status(ExperimentLifeCycle.FAILED)
experiment_group.refresh_from_db()
assert updated_at < experiment_group.updated_at