class TestBOIterationManagers(BaseTest):
DISABLE_RUNNER = True
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_bo)
self.experiments_iter1 = [
ExperimentFactory(experiment_group=self.experiment_group,
declarations={'i': i}) for i in range(2)
]
self.experiments_iter2 = [
ExperimentFactory(experiment_group=self.experiment_group,
declarations={'i': i}) for i in range(2)
]
self.experiments_iter3 = [
ExperimentFactory(experiment_group=self.experiment_group,
declarations={'i': i}) for i in range(2)
]
self.iteration_manager = BOIterationManager(
experiment_group=self.experiment_group)
def test_create_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
assert self.experiment_group.current_iteration == 0
experiment_iter1_ids = [
experiment.id for experiment in self.experiments_iter1
]
experiments_iter1_configs = [[experiment.id, experiment.declarations]
for experiment in self.experiments_iter1]
iteration = self.iteration_manager.create_iteration(
experiment_ids=experiment_iter1_ids,
experiments_configs=experiments_iter1_configs)
assert isinstance(iteration, ExperimentGroupIteration)
assert ExperimentGroupIteration.objects.count() == 1
assert self.experiment_group.current_iteration == 1
assert iteration.experiment_group == self.experiment_group
assert iteration.data == {
'iteration': 0,
'old_experiment_ids': None,
'old_experiments_configs': None,
'old_experiments_metrics': None,
'experiment_ids': experiment_iter1_ids,
'experiments_configs': experiments_iter1_configs,
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter1_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={
self.experiment_group.hptuning_config.bo.metric.name: 0.8
})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter1_metrics = [[
experiment_id, 0.8
] for experiment_id in reversed(experiment_iter1_ids)]
assert iteration.data[
'experiments_metrics'] == experiment_iter1_metrics
# Creating a new iteration uses data from previous iteration
experiment_iter2_ids = [
experiment.id for experiment in self.experiments_iter2
]
experiments_iter2_configs = [[experiment.id, experiment.declarations]
for experiment in self.experiments_iter2]
iteration = self.iteration_manager.create_iteration(
experiment_ids=experiment_iter2_ids,
experiments_configs=experiments_iter2_configs)
assert self.experiment_group.current_iteration == 2
assert iteration.data == {
'iteration': 1,
'old_experiment_ids': experiment_iter1_ids,
'old_experiments_configs': experiments_iter1_configs,
'old_experiments_metrics': experiment_iter1_metrics,
'experiment_ids': experiment_iter2_ids,
'experiments_configs': experiments_iter2_configs,
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter2_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={
self.experiment_group.hptuning_config.bo.metric.name: 0.9
})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter2_metrics = [[
experiment_id, 0.9
] for experiment_id in reversed(experiment_iter2_ids)]
assert iteration.data[
'experiments_metrics'] == experiment_iter2_metrics
# Creating a new iteration uses data from previous iteration
experiment_iter3_ids = [
experiment.id for experiment in self.experiments_iter3
]
experiments_iter3_configs = [[experiment.id, experiment.declarations]
for experiment in self.experiments_iter3]
iteration = self.iteration_manager.create_iteration(
experiment_ids=experiment_iter3_ids,
experiments_configs=experiments_iter3_configs)
assert self.experiment_group.current_iteration == 3
assert iteration.data == {
'iteration': 2,
'old_experiment_ids': experiment_iter1_ids + experiment_iter2_ids,
'old_experiments_configs':
experiments_iter1_configs + experiments_iter2_configs,
'old_experiments_metrics':
experiment_iter1_metrics + experiment_iter2_metrics,
'experiment_ids': experiment_iter3_ids,
'experiments_configs': experiments_iter3_configs,
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter3_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={
self.experiment_group.hptuning_config.bo.metric.name: 0.9
})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter3_metrics = [[
experiment_id, 0.9
] for experiment_id in reversed(experiment_iter3_ids)]
assert iteration.data[
'experiments_metrics'] == experiment_iter3_metrics
def test_update_iteration_raises_if_not_iteration_is_created(self):
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 0
class TestBOIterationManagers(BaseTest):
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_bo)
self.experiments_iter1 = [
ExperimentFactory(experiment_group=self.experiment_group, declarations={'i': i})
for i in range(2)]
self.experiments_iter2 = [
ExperimentFactory(experiment_group=self.experiment_group, declarations={'i': i})
for i in range(2)]
self.experiments_iter3 = [
ExperimentFactory(experiment_group=self.experiment_group, declarations={'i': i})
for i in range(2)]
self.iteration_manager = BOIterationManager(experiment_group=self.experiment_group)
# @staticmethod
# def assert_equal_configs(config1, config2):
# assert config1['iteration'] == config2['iteration']
# assert config1['num_suggestions'] == config2['num_suggestions']
# assert sorted(config1['old_experiment_ids']) == sorted(config2['old_experiment_ids'])
# for v in config2['old_experiments_configs']:
# assert v in config1['old_experiments_configs']
# for v in config2['old_experiments_metrics']:
# assert v in config1['old_experiments_metrics']
# assert sorted(config1['experiment_ids']) == sorted(config2['experiment_ids'])
# for v in config2['experiments_configs']:
# assert v in config1['experiments_configs']
# for v in config2['experiments_metrics']:
# assert v in config1['experiments_metrics']
@flaky(max_runs=3)
def test_create_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
assert self.experiment_group.current_iteration == 0
experiment_iter1_ids = [experiment.id for experiment in self.experiments_iter1]
experiments_iter1_configs = [[experiment.id, experiment.declarations]
for experiment in reversed(self.experiments_iter1)]
iteration = self.iteration_manager.create_iteration(num_suggestions=2)
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_iter1_ids)
assert isinstance(iteration, ExperimentGroupIteration)
assert ExperimentGroupIteration.objects.count() == 1
assert self.experiment_group.current_iteration == 1
assert iteration.experiment_group == self.experiment_group
assert self.experiment_group.iteration_data == {
'iteration': 0,
'num_suggestions': 2,
'old_experiment_ids': None,
'old_experiments_configs': None,
'old_experiments_metrics': None,
'experiment_ids': experiment_iter1_ids,
'experiments_configs': [],
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter1_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={self.experiment_group.hptuning_config.bo.metric.name: 0.8})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter1_metrics = [
[experiment_id, 0.8] for experiment_id in reversed(experiment_iter1_ids)
]
assert iteration.data['experiments_metrics'] == experiment_iter1_metrics
# Creating a new iteration uses data from previous iteration
experiment_iter2_ids = [experiment.id for experiment in self.experiments_iter2]
experiments_iter2_configs = [[experiment.id, experiment.declarations]
for experiment in reversed(self.experiments_iter2)]
iteration = self.iteration_manager.create_iteration(num_suggestions=2)
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_iter2_ids)
self.iteration_manager.update_iteration()
assert self.experiment_group.current_iteration == 2
assert self.experiment_group.iteration_data == {
'iteration': 1,
'num_suggestions': 2,
'old_experiment_ids': experiment_iter1_ids,
'old_experiments_configs': experiments_iter1_configs,
'old_experiments_metrics': experiment_iter1_metrics,
'experiment_ids': experiment_iter2_ids,
'experiments_configs': experiments_iter2_configs,
'experiments_metrics': []
}
# Update iteration
for experiment_id in experiment_iter2_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={self.experiment_group.hptuning_config.bo.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter2_metrics = [
[experiment_id, 0.9] for experiment_id in reversed(experiment_iter2_ids)
]
assert iteration.data['experiments_metrics'] == experiment_iter2_metrics
# Creating a new iteration uses data from previous iteration
experiment_iter3_ids = [experiment.id for experiment in self.experiments_iter3]
experiments_iter3_configs = [[experiment.id, experiment.declarations]
for experiment in reversed(self.experiments_iter3)]
iteration = self.iteration_manager.create_iteration(num_suggestions=2)
self.iteration_manager.add_iteration_experiments(experiment_ids=experiment_iter3_ids)
self.iteration_manager.update_iteration()
assert self.experiment_group.current_iteration == 3
assert self.experiment_group.iteration_data == {
'iteration': 2,
'num_suggestions': 2,
'old_experiment_ids': experiment_iter1_ids + experiment_iter2_ids,
'old_experiments_configs': experiments_iter1_configs + experiments_iter2_configs,
'old_experiments_metrics': experiment_iter1_metrics + experiment_iter2_metrics,
'experiment_ids': experiment_iter3_ids,
'experiments_configs': experiments_iter3_configs,
'experiments_metrics': []
}
# Update iteration
for experiment_id in experiment_iter3_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={self.experiment_group.hptuning_config.bo.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter3_metrics = [
[experiment_id, 0.9] for experiment_id in reversed(experiment_iter3_ids)
]
assert iteration.data['experiments_metrics'] == experiment_iter3_metrics
def test_update_iteration_raises_if_not_iteration_is_created(self):
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 0
class TestBOIterationManagers(BaseTest):
DISABLE_RUNNER = True
def setUp(self):
super().setUp()
self.experiment_group = ExperimentGroupFactory(
content=experiment_group_spec_content_bo)
self.experiments_iter1 = [
ExperimentFactory(experiment_group=self.experiment_group, declarations={'i': i})
for i in range(2)]
self.experiments_iter2 = [
ExperimentFactory(experiment_group=self.experiment_group, declarations={'i': i})
for i in range(2)]
self.experiments_iter3 = [
ExperimentFactory(experiment_group=self.experiment_group, declarations={'i': i})
for i in range(2)]
self.iteration_manager = BOIterationManager(experiment_group=self.experiment_group)
def test_create_iteration(self):
assert ExperimentGroupIteration.objects.count() == 0
assert self.experiment_group.current_iteration == 0
experiment_iter1_ids = [experiment.id for experiment in self.experiments_iter1]
experiments_iter1_configs = [[experiment.id, experiment.declarations]
for experiment in self.experiments_iter1]
iteration = self.iteration_manager.create_iteration(
experiment_ids=experiment_iter1_ids,
experiments_configs=experiments_iter1_configs
)
assert isinstance(iteration, ExperimentGroupIteration)
assert ExperimentGroupIteration.objects.count() == 1
assert self.experiment_group.current_iteration == 1
assert iteration.experiment_group == self.experiment_group
assert iteration.data == {
'iteration': 0,
'old_experiment_ids': None,
'old_experiments_configs': None,
'old_experiments_metrics': None,
'experiment_ids': experiment_iter1_ids,
'experiments_configs': experiments_iter1_configs,
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter1_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={self.experiment_group.hptuning_config.bo.metric.name: 0.8})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter1_metrics = [
[experiment_id, 0.8] for experiment_id in experiment_iter1_ids
]
assert iteration.data['experiments_metrics'] == experiment_iter1_metrics
# Creating a new iteration uses data from previous iteration
experiment_iter2_ids = [experiment.id for experiment in self.experiments_iter2]
experiments_iter2_configs = [[experiment.id, experiment.declarations]
for experiment in self.experiments_iter2]
iteration = self.iteration_manager.create_iteration(
experiment_ids=experiment_iter2_ids,
experiments_configs=experiments_iter2_configs
)
assert self.experiment_group.current_iteration == 2
assert iteration.data == {
'iteration': 1,
'old_experiment_ids': experiment_iter1_ids,
'old_experiments_configs': experiments_iter1_configs,
'old_experiments_metrics': experiment_iter1_metrics,
'experiment_ids': experiment_iter2_ids,
'experiments_configs': experiments_iter2_configs,
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter2_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={self.experiment_group.hptuning_config.bo.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter2_metrics = [
[experiment_id, 0.9] for experiment_id in experiment_iter2_ids
]
assert iteration.data['experiments_metrics'] == experiment_iter2_metrics
# Creating a new iteration uses data from previous iteration
experiment_iter3_ids = [experiment.id for experiment in self.experiments_iter3]
experiments_iter3_configs = [[experiment.id, experiment.declarations]
for experiment in self.experiments_iter3]
iteration = self.iteration_manager.create_iteration(
experiment_ids=experiment_iter3_ids,
experiments_configs=experiments_iter3_configs
)
assert self.experiment_group.current_iteration == 3
assert iteration.data == {
'iteration': 2,
'old_experiment_ids': experiment_iter1_ids + experiment_iter2_ids,
'old_experiments_configs': experiments_iter1_configs + experiments_iter2_configs,
'old_experiments_metrics': experiment_iter1_metrics + experiment_iter2_metrics,
'experiment_ids': experiment_iter3_ids,
'experiments_configs': experiments_iter3_configs,
'experiments_metrics': None
}
# Update iteration
for experiment_id in experiment_iter3_ids:
ExperimentMetric.objects.create(
experiment_id=experiment_id,
values={self.experiment_group.hptuning_config.bo.metric.name: 0.9})
self.iteration_manager.update_iteration()
iteration.refresh_from_db()
experiment_iter3_metrics = [
[experiment_id, 0.9] for experiment_id in experiment_iter3_ids
]
assert iteration.data['experiments_metrics'] == experiment_iter3_metrics
def test_update_iteration_raises_if_not_iteration_is_created(self):
self.iteration_manager.update_iteration()
assert ExperimentGroupIteration.objects.count() == 0