def test_stopping_group_stops_iteration(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
self.assertEqual(
mock_fct.call_count,
math.ceil(experiment_group.experiments.count() /
conf.get('GROUP_CHUNKS')) + 1)
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp,
status=ExperimentLifeCycle.SUCCEEDED)
# Mark group as stopped
ExperimentGroupStatusFactory(experiment_group=experiment_group,
status=ExperimentGroupLifeCycle.STOPPED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async'
) as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 0
def test_stopping_group_stops_iteration(self):
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp,
status=ExperimentLifeCycle.SUCCEEDED)
# Mark group as stopped
ExperimentGroupStatusFactory(experiment_group=experiment_group,
status=ExperimentGroupLifeCycle.STOPPED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async'
) as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 0
def test_hyperband_rescheduling(self):
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
ExperimentGroupFactory(content=experiment_group_spec_content_hyperband)
assert mock_fct.call_count == 1
with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async') as mock_fct1:
with patch('scheduler.tasks.experiments.experiments_build.apply_async') as mock_fct2:
ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct1.call_count == 1
assert mock_fct2.call_count == 9
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 1
# Fake reduce
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
assert experiment_group.non_done_experiments.count() == 9
# Mark experiment as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct2:
with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3:
hp_hyperband_start(experiment_group.id)
assert mock_fct2.call_count == 1
assert mock_fct3.call_count == 1
def test_hyperband_rescheduling(self):
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
ExperimentGroupFactory(content=experiment_group_spec_content_hyperband)
assert mock_fct.call_count == 1
with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async') as mock_fct1:
with patch('scheduler.tasks.experiments.experiments_build.apply_async') as mock_fct2:
ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct1.call_count == 1
assert mock_fct2.call_count == 9
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 1
# Fake reduce
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
assert experiment_group.non_done_experiments.count() == 9
# Mark experiment as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct2:
with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3:
hp_hyperband_start(experiment_group.id)
assert mock_fct2.call_count == 1
assert mock_fct3.call_count == 1
def test_stopping_group_stops_iteration(self):
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
# Mark group as stopped
ExperimentGroupStatusFactory(experiment_group=experiment_group,
status=ExperimentGroupLifeCycle.STOPPED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 0
def test_hyperband_rescheduling(self, create_build_job):
build = BuildJobFactory()
BuildJobStatus.objects.create(status=JobLifeCycle.SUCCEEDED, job=build)
create_build_job.return_value = build, True, True
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
assert mock_fct.call_count == 2
with patch.object(GroupChecks, 'is_checked') as mock_is_check:
with patch(
'hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async'
) as mock_fct1:
with patch('scheduler.tasks.experiments.'
'experiments_build.apply_async') as mock_fct2:
mock_is_check.return_value = False
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule
)
assert experiment_group.iteration_config.num_suggestions == 9
assert mock_fct1.call_count == 2
# 9 experiments, but since we are mocking the scheduling function, it's ~ 3 x calls,
# every call to start tries to schedule again, but in reality it's just 9 calls
assert mock_fct2.call_count >= 9 * 2
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
self.assertEqual(
mock_fct.call_count,
math.ceil(experiment_group.experiments.count() /
conf.get('GROUP_CHUNKS')) + 1)
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21,
'num_suggestions': 9
})
experiment_group.iteration.experiments.set(
experiment_group.experiments.values_list('id', flat=True))
# Mark experiments as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
with patch('hpsearch.tasks.hyperband.'
'hp_hyperband_start.apply_async') as xp_trigger_start:
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(
experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
assert xp_trigger_start.call_count == experiment_group.experiments.count(
)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async'
) as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 1
# Fake reduce
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
self.assertEqual(
mock_fct.call_count,
math.ceil(experiment_group.experiments.count() /
conf.get('GROUP_CHUNKS')) + 1)
assert experiment_group.non_done_experiments.count() == 9
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
with patch('hpsearch.tasks.hyperband.'
'hp_hyperband_start.apply_async') as xp_trigger_start:
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(
experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
assert xp_trigger_start.call_count == experiment_group.experiments.count(
)
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct2:
with patch.object(HyperbandIterationManager,
'reduce_configs') as mock_fct3:
hp_hyperband_start(experiment_group.id)
assert mock_fct2.call_count == 1
assert mock_fct3.call_count == 1
