def test_sobol_GPEI_strategy(self, mock_GPEI_gen, mock_GPEI_update, mock_GPEI_init): exp = get_branin_experiment() sobol_GPEI_generation_strategy = GenerationStrategy( name="Sobol+GPEI", steps=[ GenerationStep(model=Models.SOBOL, num_arms=5), GenerationStep(model=Models.GPEI, num_arms=2), ], ) self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI") self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5]) exp.new_trial( generator_run=sobol_GPEI_generation_strategy.gen(exp)).run() for i in range(1, 8): if i == 7: # Check completeness error message. with self.assertRaisesRegex(ValueError, "Generation strategy"): g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1)) else: g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1)) exp.new_trial(generator_run=g).run() if i > 4: self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge) else: self.assertEqual(g._model_key, "Sobol") self.assertEqual( g._model_kwargs, { "seed": None, "deduplicate": False, "init_position": 0, "scramble": True, }, ) self.assertEqual( g._bridge_kwargs, { "optimization_config": None, "status_quo_features": None, "status_quo_name": None, "transform_configs": None, "transforms": Cont_X_trans, }, ) # Check for "seen data" error message. with self.assertRaisesRegex(ValueError, "Data for arm"): sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data())
def test_restore_from_generator_run(self): gs = GenerationStrategy( steps=[ GenerationStep(model=Models.SOBOL, num_arms=5), GenerationStep(model=Models.GPEI, num_arms=-1), ] ) with self.assertRaises(ValueError): gs._restore_model_from_generator_run() gs.gen(experiment=get_branin_experiment()) model = gs.model gs._restore_model_from_generator_run() # Model should be reset. self.assertIsNot(model, gs.model)
def testUpdateGenerationStrategy(self): generation_strategy = get_generation_strategy() save_generation_strategy(generation_strategy=generation_strategy) # Add data, save, reload generation_strategy._data = Data( df=pd.DataFrame.from_records([{ "metric_name": "foo", "mean": 1, "arm_name": "bar" }])) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_id( gs_id=generation_strategy._db_id) self.assertEqual(generation_strategy, loaded_generation_strategy) experiment = get_branin_experiment() generation_strategy = get_generation_strategy() save_experiment(experiment) # add generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen(experiment)) save_generation_strategy(generation_strategy=generation_strategy) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # add another generator run, save, reload experiment.new_trial(generator_run=generation_strategy.gen( experiment, new_data=get_branin_data())) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) # make sure that we can update the experiment too experiment.description = "foobar" save_experiment(experiment) loaded_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, loaded_generation_strategy) self.assertEqual(generation_strategy._experiment.description, experiment.description) self.assertEqual( generation_strategy._experiment.description, loaded_generation_strategy._experiment.description, )
def test_MTGP(self): """Tests MTGP instantiation.""" # Test Multi-type MTGP exp = get_multi_type_experiment(add_trials=True) mtgp = get_MTGP(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(mtgp, TorchModelBridge) # Test Single-type MTGP exp = get_branin_experiment() # Check that factory generates a valid sobol modelbridge. sobol = get_sobol(search_space=exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) for _ in range(5): sobol_run = sobol.gen(n=1) exp.new_batch_trial().add_generator_run(sobol_run).run() mtgp = get_MTGP(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(mtgp, TorchModelBridge)
def testEncodeDecodeGenerationStrategy(self): # Cannot load generation strategy before it has been saved with self.assertRaises(ValueError): load_generation_strategy_by_id(gs_id=0) # Check that we can encode and decode the generation strategy *before* # it has generated some trials and been updated with some data. generation_strategy = get_generation_strategy() # Check that we can save a generation strategy without an experiment # attached. save_generation_strategy(generation_strategy=generation_strategy) # Also try restoring this generation strategy by its ID in the DB. new_generation_strategy = load_generation_strategy_by_id( gs_id=generation_strategy._db_id) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsNone(generation_strategy._experiment) self.assertEqual(len(generation_strategy._generated), 0) self.assertEqual(len(generation_strategy._observed), 0) # Cannot load generation strategy before it has been saved experiment = get_branin_experiment() save_experiment(experiment) with self.assertRaises(ValueError): load_generation_strategy_by_experiment_name( experiment_name=experiment.name) # Check that we can encode and decode the generation strategy *after* # it has generated some trials and been updated with some data. generation_strategy = new_generation_strategy experiment.new_trial(generator_run=generation_strategy.gen(experiment)) experiment.new_trial( generation_strategy.gen(experiment, new_data=get_branin_data())) self.assertGreater(len(generation_strategy._generated), 0) self.assertGreater(len(generation_strategy._observed), 0) save_generation_strategy(generation_strategy=generation_strategy) save_experiment(experiment) # Try restoring the generation strategy using the experiment its # attached to. new_generation_strategy = load_generation_strategy_by_experiment_name( experiment_name=experiment.name) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) self.assertIsInstance(new_generation_strategy.model, ModelBridge) self.assertEqual(len(new_generation_strategy._generator_runs), 2) self.assertEqual(new_generation_strategy._experiment._name, experiment._name)
def test_sobol_GPEI(self): """Tests sobol + GPEI instantiation.""" exp = get_branin_experiment() # Check that factory generates a valid sobol modelbridge. sobol = get_sobol(search_space=exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) for _ in range(5): sobol_run = sobol.gen(n=1) exp.new_batch_trial().add_generator_run(sobol_run).run() # Check that factory generates a valid GP+EI modelbridge. exp.optimization_config = get_branin_optimization_config() gpei = get_GPEI(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpei, TorchModelBridge) gpei = get_GPEI(experiment=exp, data=exp.fetch_data(), search_space=exp.search_space) self.assertIsInstance(gpei, TorchModelBridge)
def test_enum_sobol_GPEI(self): """Tests Sobol instantiation through the Models enum.""" exp = get_branin_experiment() # Check that factory generates a valid sobol modelbridge. sobol = Models.SOBOL(search_space=exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) for _ in range(5): sobol_run = sobol.gen(n=1) self.assertEqual(sobol_run._model_key, "Sobol") exp.new_batch_trial().add_generator_run(sobol_run).run() # Check that factory generates a valid GP+EI modelbridge. exp.optimization_config = get_branin_optimization_config() gpei = Models.GPEI(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(gpei, TorchModelBridge) gpei = Models.GPEI(experiment=exp, data=exp.fetch_data(), search_space=exp.search_space) self.assertIsInstance(gpei, TorchModelBridge)
def test_with_factory_function(self): """Checks that generation strategy works with custom factory functions. No information about the model should be saved on generator run.""" def get_sobol(search_space: SearchSpace) -> RandomModelBridge: return RandomModelBridge( search_space=search_space, model=SobolGenerator(), transforms=Cont_X_trans, ) exp = get_branin_experiment() sobol_generation_strategy = GenerationStrategy( steps=[GenerationStep(model=get_sobol, num_arms=5)]) g = sobol_generation_strategy.gen(exp) self.assertIsInstance(sobol_generation_strategy.model, RandomModelBridge) self.assertIsNone(g._model_key) self.assertIsNone(g._model_kwargs) self.assertIsNone(g._bridge_kwargs)
def test_do_not_enforce_min_observations(self): # We should be able to move on to the next model if there is not # enough data observed if `enforce_num_arms` setting is False, in which # case the previous model should be used until there is enough data. exp = get_branin_experiment() gs = GenerationStrategy(steps=[ GenerationStep( model=Models.SOBOL, num_arms=5, min_arms_observed=5, enforce_num_arms=False, ), GenerationStep(model=Models.GPEI, num_arms=1), ]) for _ in range(5): gs.gen(exp) sobol = gs._model gs.gen(exp) # Make sure the same model is used to generate the 6th point. self.assertIs(gs._model, sobol)
def test_transform_callback_unitx(self, _): exp = get_branin_experiment() parameters = [ RangeParameter(name="x1", parameter_type=ParameterType.FLOAT, lower=0, upper=10), RangeParameter(name="x2", parameter_type=ParameterType.FLOAT, lower=0, upper=100), ] gpei = TorchModelBridge( experiment=exp, data=exp.fetch_data(), search_space=SearchSpace(parameters=parameters), model=BotorchModel(), transforms=[UnitX], ) transformed = gpei._transform_callback([0.75, 0.35]) self.assertTrue(np.allclose(transformed, [0.75, 0.35]))
def testDecodeGenerationStrategy(self): generation_strategy = get_generation_strategy() experiment = get_branin_experiment() gs_json = object_to_json(generation_strategy) new_generation_strategy = generation_strategy_from_json( experiment, gs_json) self.assertEqual(generation_strategy, new_generation_strategy) self.assertGreater(len(new_generation_strategy._steps), 0) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) # Model has not yet been initialized on this GS since it hasn't generated # anything yet. self.assertIsNone(new_generation_strategy.model) # Check that we can encode and decode the generation strategy after # it has generated some trials. generation_strategy = new_generation_strategy experiment.new_trial(generator_run=generation_strategy.gen(experiment)) gs_json = object_to_json(generation_strategy) new_generation_strategy = generation_strategy_from_json( experiment, gs_json) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) # Since this GS has now generated one generator run, model should have # been initialized and restored when decoding from JSON. self.assertIsInstance(new_generation_strategy.model, ModelBridge) # Check that we can encode and decode the generation strategy after # it has generated some trials and been updated with some data. generation_strategy = new_generation_strategy experiment.new_trial( generation_strategy.gen(experiment, new_data=get_branin_data())) self.assertGreater(len(generation_strategy._generated), 0) self.assertGreater(len(generation_strategy._observed), 0) gs_json = object_to_json(generation_strategy) new_generation_strategy = generation_strategy_from_json( experiment, gs_json) self.assertEqual(generation_strategy, new_generation_strategy) self.assertIsInstance(new_generation_strategy._steps[0].model, Models) self.assertIsInstance(new_generation_strategy.model, ModelBridge)
def test_transform_callback_int(self, _): exp = get_branin_experiment() parameters = [ RangeParameter(name="x1", parameter_type=ParameterType.INT, lower=1, upper=10), RangeParameter(name="x2", parameter_type=ParameterType.INT, lower=5, upper=15), ] gpei = TorchModelBridge( experiment=exp, data=exp.fetch_data(), search_space=SearchSpace(parameters=parameters), model=BotorchModel(), transforms=[IntToFloat], torch_dtype=torch.double, ) transformed = gpei._transform_callback([5.4, 7.6]) self.assertTrue(np.allclose(transformed, [5, 8]))
def test_sobol_GPEI_strategy_batches( self, mock_GPEI_gen, mock_GPEI_update, mock_GPEI_init ): exp = get_branin_experiment() sobol_GPEI_generation_strategy = GenerationStrategy( name="Sobol+GPEI", steps=[ GenerationStep(model=Models.SOBOL, num_arms=5), GenerationStep(model=Models.GPEI, num_arms=8), ], ) self.assertEqual(sobol_GPEI_generation_strategy.name, "Sobol+GPEI") self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5]) exp.new_batch_trial( generator_run=sobol_GPEI_generation_strategy.gen(exp, n=2) ).run() for i in range(1, 8): if i == 2: with self.assertRaisesRegex(ValueError, "Cannot generate 2 new"): g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1), n=2 ) g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1) ) elif i == 7: # Check completeness error message. with self.assertRaisesRegex(ValueError, "Generation strategy"): g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1), n=2 ) else: g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1), n=2 ) exp.new_batch_trial(generator_run=g).run() with self.assertRaises(ValueError): sobol_GPEI_generation_strategy.gen(exp, exp.fetch_data()) self.assertIsInstance(sobol_GPEI_generation_strategy.model, TorchModelBridge)
def test_MTGP(self): """Tests MTGP instantiation.""" # Test Multi-type MTGP exp = get_multi_type_experiment(add_trials=True) mtgp = get_MTGP(experiment=exp, data=exp.fetch_data()) self.assertIsInstance(mtgp, TorchModelBridge) # Test Single-type MTGP exp = get_branin_experiment() # Check that factory generates a valid sobol modelbridge. sobol = get_sobol(search_space=exp.search_space) self.assertIsInstance(sobol, RandomModelBridge) for _ in range(5): sobol_run = sobol.gen(n=1) exp.new_batch_trial().add_generator_run(sobol_run).run() mtgp = get_MTGP(experiment=exp, is_multi_type=False, data=exp.fetch_data()) self.assertIsInstance(mtgp, TorchModelBridge) # Test wrong call of Multi-type MTGP. The type of the input experiment # should be MultiTypeExperiment. with self.assertRaises(ValueError): get_MTGP(experiment=exp, is_multi_type=True, data=exp.fetch_data())
def test_sobol_GPEI_strategy_keep_generating(self, mock_GPEI_gen, mock_GPEI_update, mock_GPEI_init): exp = get_branin_experiment() sobol_GPEI_generation_strategy = GenerationStrategy(steps=[ GenerationStep(model=Models.SOBOL, num_arms=5), GenerationStep(model=Models.GPEI, num_arms=-1), ]) self.assertEqual(sobol_GPEI_generation_strategy.name, "sobol+GPEI") self.assertEqual(sobol_GPEI_generation_strategy.generator_changes, [5]) exp.new_trial( generator_run=sobol_GPEI_generation_strategy.gen(exp)).run() for i in range(1, 15): # Passing in all experiment data should cause an error as only # new data should be passed into `gen`. if i > 1: with self.assertRaisesRegex(ValueError, "Data for arm"): g = sobol_GPEI_generation_strategy.gen( exp, exp.fetch_data()) g = sobol_GPEI_generation_strategy.gen( exp, exp._fetch_trial_data(trial_index=i - 1)) exp.new_trial(generator_run=g).run() if i > 4: mock_GPEI_init.assert_called()
def test_uniform(self): exp = get_branin_experiment() uniform = get_uniform(exp.search_space) self.assertIsInstance(uniform, RandomModelBridge) uniform_run = uniform.gen(n=5) self.assertEqual(len(uniform_run.arms), 5)