def test_fixed_features(self): ms = ModelSpec(model_enum=Models.GPEI) self.assertIsNone(ms.fixed_features) new_features = ObservationFeatures(parameters={"a": 1.0}) ms.fixed_features = new_features self.assertEqual(ms.fixed_features, new_features) self.assertEqual(ms.model_gen_kwargs["fixed_features"], new_features)
def test_construct(self): ms = ModelSpec(model_enum=Models.GPEI) with self.assertRaises(UserInputError): ms.gen(n=1) ms.fit(experiment=self.experiment, data=self.data) ms.gen(n=1) with self.assertRaises(NotImplementedError): ms.update(experiment=self.experiment, new_data=self.data)
def setUp(self): self.branin_experiment = get_branin_experiment_with_multi_objective() sobol = Models.SOBOL(search_space=self.branin_experiment.search_space) sobol_run = sobol.gen(n=20) self.branin_experiment.new_batch_trial().add_generator_run( sobol_run ).run().mark_completed() data = self.branin_experiment.fetch_data() ms_gpei = ModelSpec(model_enum=Models.GPEI) ms_gpei.fit(experiment=self.branin_experiment, data=data) ms_gpkg = ModelSpec(model_enum=Models.GPKG) ms_gpkg.fit(experiment=self.branin_experiment, data=data) self.fitted_model_specs = [ms_gpei, ms_gpkg] self.model_selection_node = GenerationNode( model_specs=self.fitted_model_specs, best_model_selector=SingleDiagnosticBestModelSelector( diagnostic="Fisher exact test p", criterion=MetricAggregation.MEAN, metric_aggregation=DiagnosticCriterion.MIN, ), )
def test_properties(self): node = GenerationNode( model_specs=[ ModelSpec( model_enum=Models.GPEI, model_kwargs={}, model_gen_kwargs={ "n": 1, "fixed_features": ObservationFeatures( parameters={}, trial_index=0 ), }, ), ], ) dat = self.branin_experiment.lookup_data() node.fit( experiment=self.branin_experiment, data=dat, ) self.assertEqual(node.model_enum, node.model_specs[0].model_enum) self.assertEqual(node.model_kwargs, node.model_specs[0].model_kwargs) self.assertEqual(node.model_gen_kwargs, node.model_specs[0].model_gen_kwargs) self.assertEqual(node.model_cv_kwargs, node.model_specs[0].model_cv_kwargs) self.assertEqual(node.fixed_features, node.model_specs[0].fixed_features) self.assertEqual(node.cv_results, node.model_specs[0].cv_results) self.assertEqual(node.diagnostics, node.model_specs[0].diagnostics)
def setUp(self): self.sobol_model_spec = ModelSpec( model_enum=Models.SOBOL, model_kwargs={"init_position": 3}, model_gen_kwargs={"some_gen_kwarg": "some_value"}, ) self.sobol_generation_node = GenerationNode(model_specs=[self.sobol_model_spec]) self.branin_experiment = get_branin_experiment(with_completed_trial=True)
def test_cross_validate_with_non_GP_model( self, mock_cv: Mock, mock_diagnostics: Mock ): mock_enum = Mock() mock_enum.return_value = "fake-modelbridge" ms = ModelSpec(model_enum=mock_enum, model_cv_kwargs={"test_key": "test-value"}) ms.fit( experiment=self.experiment, data=self.experiment.trials[0].fetch_data(), ) with warnings.catch_warnings(record=True) as w: cv_results, cv_diagnostics = ms.cross_validate() self.assertEqual(len(w), 1) self.assertIn("cannot be cross validated", str(w[0].message)) self.assertIsNone(cv_results) self.assertIsNone(cv_diagnostics) mock_cv.assert_called_with(model="fake-modelbridge", test_key="test-value") mock_diagnostics.assert_not_called()
def setUp(self): self.model_kwargs = ({"init_position": 5},) self.sobol_generation_step = GenerationStep( model=Models.SOBOL, num_trials=5, model_kwargs=self.model_kwargs, ) self.model_spec = ModelSpec( model_enum=self.sobol_generation_step.model, model_kwargs=self.model_kwargs, model_gen_kwargs=None, )
def test_cross_validate_with_GP_model(self, mock_cv: Mock, mock_diagnostics: Mock): mock_enum = Mock() mock_enum.return_value = "fake-modelbridge" ms = ModelSpec(model_enum=mock_enum, model_cv_kwargs={"test_key": "test-value"}) ms.fit( experiment=self.experiment, data=self.experiment.trials[0].fetch_data(), ) cv_results, cv_diagnostics = ms.cross_validate() mock_cv.assert_called_with(model="fake-modelbridge", test_key="test-value") mock_diagnostics.assert_called_with(["fake-cv-result"]) self.assertIsNotNone(cv_results) self.assertIsNotNone(cv_diagnostics) with self.subTest("it caches CV results"): mock_cv.reset_mock() mock_diagnostics.reset_mock() cv_results, cv_diagnostics = ms.cross_validate() self.assertIsNotNone(cv_results) self.assertIsNotNone(cv_diagnostics) mock_cv.assert_not_called() mock_diagnostics.assert_not_called() with self.subTest("fit clears the CV cache"): mock_cv.reset_mock() mock_diagnostics.reset_mock() ms.fit( experiment=self.experiment, data=self.experiment.trials[0].fetch_data(), ) cv_results, cv_diagnostics = ms.cross_validate() self.assertIsNotNone(cv_results) self.assertIsNotNone(cv_diagnostics) mock_cv.assert_called_with(model="fake-modelbridge", test_key="test-value") mock_diagnostics.assert_called_with(["fake-cv-result"])
def test_multiple_same_fixed_features(self) -> None: node = GenerationNode( model_specs=[ ModelSpec( model_enum=Models.GPEI, model_kwargs={}, model_gen_kwargs={ "n": 2, "fixed_features": ObservationFeatures(parameters={"x": 0}), }, ), ModelSpec( model_enum=Models.GPEI, model_kwargs={}, model_gen_kwargs={ "n": 3, "fixed_features": ObservationFeatures(parameters={"x": 0}), }, ), ], ) self.assertEqual(node.fixed_features, ObservationFeatures(parameters={"x": 0}))
def __post_init__(self) -> None: if not isinstance(self.model, ModelRegistryBase): if not callable(self.model): raise UserInputError( "`model` in generation step must be either a `ModelRegistryBase` " "enum subclass entry or a callable factory function returning a " "model bridge instance.") model_spec = FactoryFunctionModelSpec( factory_function=self.model, model_kwargs=self.model_kwargs, model_gen_kwargs=self.model_gen_kwargs, ) else: model_spec = ModelSpec( model_enum=self.model, model_kwargs=self.model_kwargs, model_gen_kwargs=self.model_gen_kwargs, ) super().__init__(model_specs=[model_spec])
def test_model_key(self): ms = ModelSpec(model_enum=Models.GPEI) self.assertEqual(ms.model_key, "GPEI")