def test_managers(self, _):
experiment_group = ExperimentGroupFactory(content=None, hptuning=None)
assert experiment_group.search_manager is None
# Adding hptuning
experiment_group.hptuning = {
'concurrency': 2,
'grid_search': {'n_experiments': 10},
'matrix': {'lr': {'values': [1, 2, 3]}}
}
experiment_group.save()
experiment_group = ExperimentGroup.objects.get(id=experiment_group.id)
assert isinstance(experiment_group.search_manager, GridSearchManager)
assert experiment_group.iteration_manager is None
# Adding hptuning
experiment_group.hptuning = {
'concurrency': 2,
'random_search': {'n_experiments': 10},
'matrix': {'lr': {'values': [1, 2, 3]}}
}
experiment_group.save()
experiment_group = ExperimentGroup.objects.get(id=experiment_group.id)
assert isinstance(experiment_group.search_manager, RandomSearchManager)
assert experiment_group.iteration_manager is None
# Adding hptuning
experiment_group.hptuning = {
'concurrency': 2,
'hyperband': {
'max_iter': 10,
'eta': 3,
'resource': {'name': 'steps', 'type': 'int'},
'resume': False,
'metric': {'name': 'loss', 'optimization': 'minimize'}
},
'matrix': {'lr': {'values': [1, 2, 3]}}
}
experiment_group.save()
experiment_group = ExperimentGroup.objects.get(id=experiment_group.id)
assert isinstance(experiment_group.search_manager, HyperbandSearchManager)
assert isinstance(experiment_group.iteration_manager, HyperbandIterationManager)
# Adding hptuning
experiment_group.hptuning = {
'concurrency': 2,
'bo': {
'n_iterations': 4,
'n_initial_trials': 4,
'metric': {
'name': 'loss',
'optimization': 'minimize'
},
'utility_function': {
'acquisition_function': 'ei',
'eps': 1.2,
'gaussian_process': {
'kernel': 'matern',
'length_scale': 1.0,
'nu': 1.9,
'n_restarts_optimizer': 0
}
}
},
'matrix': {'lr': {'values': [1, 2, 3]}}
}
experiment_group.save()
experiment_group = ExperimentGroup.objects.get(id=experiment_group.id)
assert isinstance(experiment_group.search_manager, BOSearchManager)
assert isinstance(experiment_group.iteration_manager, BOIterationManager)
def test_spec_creation_triggers_experiments_planning(self, mock_fct):
experiment_group = ExperimentGroupFactory()
assert Experiment.objects.filter(experiment_group=experiment_group).count() == 0
assert mock_fct.call_count == 1
def create_experiment_group(self):
with patch('hpsearch.tasks.grid.hp_grid_search_start.apply_async') as _: # noqa
return ExperimentGroupFactory(project=self.project)
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory()
self.project = self.experiment_group.project
def test_hyperband_rescheduling(self):
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
assert mock_fct.call_count == 1
with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async'
) as mock_fct1:
with patch(
'scheduler.tasks.experiments.experiments_build.apply_async'
) as mock_fct2:
ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct1.call_count == 1
assert mock_fct2.call_count == 9
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21
})
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp,
status=ExperimentLifeCycle.SUCCEEDED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async'
) as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 1
# Fake reduce
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct:
experiment_group = ExperimentGroupFactory(
content=
experiment_group_spec_content_hyperband_trigger_reschedule)
assert mock_fct.call_count == 1
assert experiment_group.non_done_experiments.count() == 9
# Mark experiment as done
with patch(
'scheduler.experiment_scheduler.stop_experiment') as _: # noqa
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp,
status=ExperimentLifeCycle.SUCCEEDED)
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async'
) as mock_fct2:
with patch.object(HyperbandIterationManager,
'reduce_configs') as mock_fct3:
hp_hyperband_start(experiment_group.id)
assert mock_fct2.call_count == 1
assert mock_fct3.call_count == 1
def test_experiment_group_deletion_deletes_old_data(self, delete_path, _):
experiment_group = ExperimentGroupFactory()
assert delete_path.call_count == 2 # outputs + logs
experiment_group.delete()
assert delete_path.call_count == 2 + 2 # outputs + logs
def test_spec_creation_triggers_experiments_planning(self):
with patch('experiment_groups.tasks.create_group_experiments.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory()
assert Experiment.objects.filter(experiment_group=experiment_group).count() == 0
assert mock_fct.call_count == 1
def test_get_ordered_experiments_by_metric(self, _):
experiment_group = ExperimentGroupFactory()
self.assertEqual(
len(
experiment_group.get_ordered_experiments_by_metric(
experiment_ids=[],
metric='precision',
optimization='maximize')), 0)
experiments = []
experiment_ids = []
for _ in range(5):
experiment = ExperimentFactory(experiment_group=experiment_group)
experiments.append(experiment)
experiment_ids.append(experiment.id)
# Create metrics for 2 experiments
for experiment in experiments[:2]:
ExperimentMetric.objects.create(
experiment=experiment, values={'accuracy': random.random()})
# Test metric values for accuracy (2 values)
experiment_metrics = experiment_group.get_ordered_experiments_by_metric(
experiment_ids=experiment_ids,
metric='accuracy',
optimization='maximize')
assert len(experiment_metrics) == 5
metrics = [
m.accuracy for m in experiment_metrics if m.accuracy is not None
]
assert len(metrics) == 2
assert sorted(metrics, reverse=True) == metrics
# Add more metrics (precision for all, and loss for 3)
for experiment in experiments:
ExperimentMetric.objects.create(
experiment=experiment, values={'precision': random.random()})
for experiment in experiments[:3]:
ExperimentMetric.objects.create(experiment=experiment,
values={'loss': random.random()})
# Testing again for accuracy should be propagated to all experiments but only 2 with values
experiment_metrics = experiment_group.get_ordered_experiments_by_metric(
experiment_ids=experiment_ids,
metric='accuracy',
optimization='maximize')
assert len(experiment_metrics) == 5
metrics = [
m.accuracy for m in experiment_metrics if m.accuracy is not None
]
assert len(metrics) == 2
assert sorted(metrics, reverse=True) == metrics
# Test for precisions, even after an updated metrics, the values were passed to last_metric
experiment_metrics = experiment_group.get_ordered_experiments_by_metric(
experiment_ids=experiment_ids,
metric='precision',
optimization='maximize')
assert len(experiment_metrics) == 5
metrics = [
m.precision for m in experiment_metrics if m.precision is not None
]
assert len(metrics) == 5
assert sorted(metrics, reverse=True) == metrics
# Loss is only visible on 3 experiments
experiment_metrics = experiment_group.get_ordered_experiments_by_metric(
experiment_ids=experiment_ids,
metric='loss',
optimization='minimize')
assert len(experiment_metrics) == 5
metrics = [m.loss for m in experiment_metrics if m.loss is not None]
assert len(metrics) == 3
assert sorted(metrics) == metrics
# Check non existing metric
experiment_metrics = experiment_group.get_ordered_experiments_by_metric(
experiment_ids=experiment_ids,
metric='metric_dummy',
optimization='maximize')
self.assertEqual(len(experiment_metrics), 5)
self.assertEqual(
len([m for m in experiment_metrics if m.metric_dummy is not None]),
0)
def setUp(self):
super().setUp()
self.experiment = ExperimentFactory()
self.experiment_group = ExperimentGroupFactory()
def test_get_suggestions(self):
# Manager1
experiment_group = ExperimentGroupFactory(
params=self.manager1.params_config.to_dict())
# Fake iteration
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 2
})
suggestions = self.manager1.get_suggestions(
iteration_config=experiment_group.iteration_config)
assert len(suggestions) == 9
for suggestion in suggestions:
assert 'steps' in suggestion
self.almost_equal(suggestion['steps'], 9.99)
assert 'feature1' in suggestion
assert 'feature2' in suggestion
assert 'feature3' in suggestion
# Fake iteration
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 1,
'bracket_iteration': 0
})
suggestions = self.manager1.get_suggestions(
iteration_config=experiment_group.iteration_config)
assert len(suggestions) == 5
for suggestion in suggestions:
assert 'steps' in suggestion
self.almost_equal(suggestion['steps'], 3.33)
assert 'feature1' in suggestion
assert 'feature2' in suggestion
assert 'feature3' in suggestion
# Fake iteration
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 2,
'bracket_iteration': 0
})
suggestions = self.manager1.get_suggestions(
iteration_config=experiment_group.iteration_config)
assert len(suggestions) == 3
for suggestion in suggestions:
assert 'steps' in suggestion
self.almost_equal(suggestion['steps'], 9.99)
assert 'feature1' in suggestion
assert 'feature2' in suggestion
assert 'feature3' in suggestion
# Manager2
experiment_group = ExperimentGroupFactory(
params=self.manager2.params_config.to_dict())
# Fake iteration
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 2,
'bracket_iteration': 0
})
suggestions = self.manager2.get_suggestions(
iteration_config=experiment_group.iteration_config)
assert len(suggestions) == 15
for suggestion in suggestions:
assert 'size' in suggestion
self.almost_equal(suggestion['size'], 9)
assert 'feature1' in suggestion
assert 'feature2' in suggestion
assert 'feature3' in suggestion
assert 'feature4' in suggestion
# Fake iteration
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 4,
'bracket_iteration': 0
})
suggestions = self.manager2.get_suggestions(
iteration_config=experiment_group.iteration_config)
assert len(suggestions) == 5
for suggestion in suggestions:
assert 'size' in suggestion
self.almost_equal(suggestion['size'], 81)
assert 'feature1' in suggestion
assert 'feature2' in suggestion
assert 'feature3' in suggestion
assert 'feature4' in suggestion
def test_hyperband_rescheduling(self):
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
ExperimentGroupFactory(content=experiment_group_spec_content_hyperband)
assert mock_fct.call_count == 2
with patch.object(GroupChecks, 'is_checked') as mock_is_check:
with patch('hpsearch.tasks.hyperband.hp_hyperband_iterate.apply_async') as mock_fct1:
with patch('scheduler.tasks.experiments.'
'experiments_build.apply_async') as mock_fct2:
mock_is_check.return_value = False
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
assert experiment_group.iteration_config.num_suggestions == 9
assert mock_fct1.call_count == 2
# 9 experiments, but since we are mocking the scheduling function, it's ~ 3 x calls,
# every call to start tries to schedule again, but in reality it's just 9 calls
assert mock_fct2.call_count >= 9 * 2
# Fake reschedule
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
self.assertEqual(
mock_fct.call_count,
math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1
)
ExperimentGroupIteration.objects.create(
experiment_group=experiment_group,
data={
'iteration': 0,
'bracket_iteration': 21,
'num_suggestions': 9
})
experiment_group.iteration.experiments.set(
experiment_group.experiments.values_list('id', flat=True))
# Mark experiments as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
with patch('hpsearch.tasks.hyperband.'
'hp_hyperband_start.apply_async') as xp_trigger_start:
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
assert xp_trigger_start.call_count == experiment_group.experiments.count()
with patch('hpsearch.tasks.hyperband.hp_hyperband_create.apply_async') as mock_fct1:
hp_hyperband_start(experiment_group.id)
assert mock_fct1.call_count == 1
# Fake reduce
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct:
experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband_trigger_reschedule)
self.assertEqual(
mock_fct.call_count,
math.ceil(experiment_group.experiments.count() / conf.get('GROUP_CHUNKS')) + 1
)
assert experiment_group.non_done_experiments.count() == 9
# Mark experiment as done
with patch('scheduler.experiment_scheduler.stop_experiment') as _: # noqa
with patch('hpsearch.tasks.hyperband.'
'hp_hyperband_start.apply_async') as xp_trigger_start:
for xp in experiment_group.experiments.all():
ExperimentStatusFactory(experiment=xp, status=ExperimentLifeCycle.SUCCEEDED)
assert xp_trigger_start.call_count == experiment_group.experiments.count()
with patch('hpsearch.tasks.hyperband.hp_hyperband_start.apply_async') as mock_fct2:
with patch.object(HyperbandIterationManager, 'reduce_configs') as mock_fct3:
hp_hyperband_start(experiment_group.id)
assert mock_fct2.call_count == 1
assert mock_fct3.call_count == 1
