def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
for _ in range(3):
ExperimentFactory(experiment_group=self.experiment_group)
self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
for _ in range(3):
ExperimentFactory(experiment_group=self.experiment_group)
self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)
class TestHyperbandIterationManagers(BaseTest):
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
for _ in range(3):
ExperimentFactory(experiment_group=self.experiment_group)
self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)
@flaky(max_runs=3)
def test_create_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
assert isinstance(iteration, ExperimentGroupIteration)
assert ExperimentGroupIteration.objects.count() == 1
assert iteration.experiment_group == self.experiment_group
assert iteration.data == {
'iteration': 0,
'num_suggestions': 0,
'bracket_iteration': 0,
'experiment_ids': [],
'experiments_metrics': None,
}
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert self.experiment_group.iteration_data == {
'iteration': 0,
'bracket_iteration': 0,
'num_suggestions': 0,
'experiment_ids': experiment_ids,
'experiments_metrics': None,
}
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert self.experiment_group.iteration_data == {
'iteration': 0,
'bracket_iteration': 0,
'num_suggestions': 0,
'experiment_ids': experiment_ids,
'experiments_metrics': None,
}
self.iteration_manager.update_iteration_num_suggestions(num_suggestions=3)
assert self.experiment_group.iteration_data == {
'iteration': 0,
'bracket_iteration': 0,
'num_suggestions': 3,
'experiment_ids': experiment_ids,
'experiments_metrics': None,
}
experiment_ids = list(self.experiment_group.experiments.values_list('id', flat=True))
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
self.assertEqual(sorted(self.experiment_group.iteration_data['experiment_ids']),
sorted(experiment_ids))
def test_update_iteration_raises_if_not_iteration_is_created(self):
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 0
def test_update_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert iteration.data['experiments_metrics'] is None
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 1
iteration.refresh_from_db()
assert iteration.data['experiments_metrics'] == []
ExperimentMetric.objects.create(
experiment_id=experiment_ids[0],
values={self.experiment_group.hptuning_config.hyperband.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
assert iteration.data['experiments_metrics'] == [[experiment_ids[0], 0.9]]
def test_get_reduced_configs(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert iteration.data['experiments_metrics'] is None
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 1
assert self.iteration_manager.get_reduced_configs() == []
class TestHyperbandIterationManagers(BaseTest):
DISABLE_RUNNER = True
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
for _ in range(3):
ExperimentFactory(experiment_group=self.experiment_group)
self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)
def test_create_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
assert isinstance(iteration, ExperimentGroupIteration)
assert ExperimentGroupIteration.objects.count() == 1
assert iteration.experiment_group == self.experiment_group
assert iteration.data == {
'iteration': 0,
'bracket_iteration': 0,
'experiment_ids': None,
'experiments_metrics': None,
}
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
iteration.refresh_from_db()
assert iteration.data == {
'iteration': 0,
'bracket_iteration': 0,
'experiment_ids': experiment_ids,
'experiments_metrics': None,
}
def test_update_iteration_raises_if_not_iteration_is_created(self):
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 0
def test_update_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert iteration.data['experiments_metrics'] is None
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 1
iteration.refresh_from_db()
assert iteration.data['experiments_metrics'] == []
ExperimentMetric.objects.create(
experiment_id=experiment_ids[0],
values={self.experiment_group.hptuning_config.hyperband.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
assert iteration.data['experiments_metrics'] == [[experiment_ids[0], 0.9]]
def test_get_reduced_configs(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert iteration.data['experiments_metrics'] is None
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 1
assert self.iteration_manager.get_reduced_configs() == []
class TestHyperbandIterationManagers(BaseTest):
DISABLE_RUNNER = True
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_hyperband)
for _ in range(3):
ExperimentFactory(experiment_group=self.experiment_group)
self.iteration_manager = HyperbandIterationManager(experiment_group=self.experiment_group)
def test_create_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
assert isinstance(iteration, ExperimentGroupIteration)
assert ExperimentGroupIteration.objects.count() == 1
assert iteration.experiment_group == self.experiment_group
assert iteration.data == {
'iteration': 0,
'bracket_iteration': 0,
'experiment_ids': None,
'experiments_metrics': None,
}
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
iteration.refresh_from_db()
assert iteration.data == {
'iteration': 0,
'bracket_iteration': 0,
'experiment_ids': experiment_ids,
'experiments_metrics': None,
}
def test_update_iteration_raises_if_not_iteration_is_created(self):
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 0
def test_update_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert iteration.data['experiments_metrics'] is None
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 1
iteration.refresh_from_db()
assert iteration.data['experiments_metrics'] == []
ExperimentMetric.objects.create(
experiment_id=experiment_ids[0],
values={self.experiment_group.params_config.hyperband.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
assert iteration.data['experiments_metrics'] == [[experiment_ids[0], 0.9]]
def test_get_reduced_configs(self):
assert ExperimentGroupIteration.objects.count() == 0
experiment_ids = [self.experiment_group.experiments.first().id]
iteration = self.iteration_manager.create_iteration()
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_ids)
assert iteration.data['experiments_metrics'] is None
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 1
assert self.iteration_manager.get_reduced_configs() == []
