def test_fixed_features(self):
ms = ModelSpec(model_enum=Models.GPEI)
self.assertIsNone(ms.fixed_features)
new_features = ObservationFeatures(parameters={"a": 1.0})
ms.fixed_features = new_features
self.assertEqual(ms.fixed_features, new_features)
self.assertEqual(ms.model_gen_kwargs["fixed_features"], new_features)
def test_construct(self):
ms = ModelSpec(model_enum=Models.GPEI)
with self.assertRaises(UserInputError):
ms.gen(n=1)
ms.fit(experiment=self.experiment, data=self.data)
ms.gen(n=1)
with self.assertRaises(NotImplementedError):
ms.update(experiment=self.experiment, new_data=self.data)
def setUp(self):
self.branin_experiment = get_branin_experiment_with_multi_objective()
sobol = Models.SOBOL(search_space=self.branin_experiment.search_space)
sobol_run = sobol.gen(n=20)
self.branin_experiment.new_batch_trial().add_generator_run(
sobol_run
).run().mark_completed()
data = self.branin_experiment.fetch_data()
ms_gpei = ModelSpec(model_enum=Models.GPEI)
ms_gpei.fit(experiment=self.branin_experiment, data=data)
ms_gpkg = ModelSpec(model_enum=Models.GPKG)
ms_gpkg.fit(experiment=self.branin_experiment, data=data)
self.fitted_model_specs = [ms_gpei, ms_gpkg]
self.model_selection_node = GenerationNode(
model_specs=self.fitted_model_specs,
best_model_selector=SingleDiagnosticBestModelSelector(
diagnostic="Fisher exact test p",
criterion=MetricAggregation.MEAN,
metric_aggregation=DiagnosticCriterion.MIN,
),
)
def test_properties(self):
node = GenerationNode(
model_specs=[
ModelSpec(
model_enum=Models.GPEI,
model_kwargs={},
model_gen_kwargs={
"n": 1,
"fixed_features": ObservationFeatures(
parameters={}, trial_index=0
),
},
),
],
)
dat = self.branin_experiment.lookup_data()
node.fit(
experiment=self.branin_experiment,
data=dat,
)
self.assertEqual(node.model_enum, node.model_specs[0].model_enum)
self.assertEqual(node.model_kwargs, node.model_specs[0].model_kwargs)
self.assertEqual(node.model_gen_kwargs, node.model_specs[0].model_gen_kwargs)
self.assertEqual(node.model_cv_kwargs, node.model_specs[0].model_cv_kwargs)
self.assertEqual(node.fixed_features, node.model_specs[0].fixed_features)
self.assertEqual(node.cv_results, node.model_specs[0].cv_results)
self.assertEqual(node.diagnostics, node.model_specs[0].diagnostics)
def setUp(self):
self.sobol_model_spec = ModelSpec(
model_enum=Models.SOBOL,
model_kwargs={"init_position": 3},
model_gen_kwargs={"some_gen_kwarg": "some_value"},
)
self.sobol_generation_node = GenerationNode(model_specs=[self.sobol_model_spec])
self.branin_experiment = get_branin_experiment(with_completed_trial=True)
def test_cross_validate_with_non_GP_model(
self, mock_cv: Mock, mock_diagnostics: Mock
):
mock_enum = Mock()
mock_enum.return_value = "fake-modelbridge"
ms = ModelSpec(model_enum=mock_enum, model_cv_kwargs={"test_key": "test-value"})
ms.fit(
experiment=self.experiment,
data=self.experiment.trials[0].fetch_data(),
)
with warnings.catch_warnings(record=True) as w:
cv_results, cv_diagnostics = ms.cross_validate()
self.assertEqual(len(w), 1)
self.assertIn("cannot be cross validated", str(w[0].message))
self.assertIsNone(cv_results)
self.assertIsNone(cv_diagnostics)
mock_cv.assert_called_with(model="fake-modelbridge", test_key="test-value")
mock_diagnostics.assert_not_called()
def setUp(self):
self.model_kwargs = ({"init_position": 5},)
self.sobol_generation_step = GenerationStep(
model=Models.SOBOL,
num_trials=5,
model_kwargs=self.model_kwargs,
)
self.model_spec = ModelSpec(
model_enum=self.sobol_generation_step.model,
model_kwargs=self.model_kwargs,
model_gen_kwargs=None,
)
def test_cross_validate_with_GP_model(self, mock_cv: Mock,
mock_diagnostics: Mock):
mock_enum = Mock()
mock_enum.return_value = "fake-modelbridge"
ms = ModelSpec(model_enum=mock_enum,
model_cv_kwargs={"test_key": "test-value"})
ms.fit(
experiment=self.experiment,
data=self.experiment.trials[0].fetch_data(),
)
cv_results, cv_diagnostics = ms.cross_validate()
mock_cv.assert_called_with(model="fake-modelbridge",
test_key="test-value")
mock_diagnostics.assert_called_with(["fake-cv-result"])
self.assertIsNotNone(cv_results)
self.assertIsNotNone(cv_diagnostics)
with self.subTest("it caches CV results"):
mock_cv.reset_mock()
mock_diagnostics.reset_mock()
cv_results, cv_diagnostics = ms.cross_validate()
self.assertIsNotNone(cv_results)
self.assertIsNotNone(cv_diagnostics)
mock_cv.assert_not_called()
mock_diagnostics.assert_not_called()
with self.subTest("fit clears the CV cache"):
mock_cv.reset_mock()
mock_diagnostics.reset_mock()
ms.fit(
experiment=self.experiment,
data=self.experiment.trials[0].fetch_data(),
)
cv_results, cv_diagnostics = ms.cross_validate()
self.assertIsNotNone(cv_results)
self.assertIsNotNone(cv_diagnostics)
mock_cv.assert_called_with(model="fake-modelbridge",
test_key="test-value")
mock_diagnostics.assert_called_with(["fake-cv-result"])
def test_multiple_same_fixed_features(self) -> None:
node = GenerationNode(
model_specs=[
ModelSpec(
model_enum=Models.GPEI,
model_kwargs={},
model_gen_kwargs={
"n": 2,
"fixed_features": ObservationFeatures(parameters={"x": 0}),
},
),
ModelSpec(
model_enum=Models.GPEI,
model_kwargs={},
model_gen_kwargs={
"n": 3,
"fixed_features": ObservationFeatures(parameters={"x": 0}),
},
),
],
)
self.assertEqual(node.fixed_features, ObservationFeatures(parameters={"x": 0}))
def __post_init__(self) -> None:
if not isinstance(self.model, ModelRegistryBase):
if not callable(self.model):
raise UserInputError(
"`model` in generation step must be either a `ModelRegistryBase` "
"enum subclass entry or a callable factory function returning a "
"model bridge instance.")
model_spec = FactoryFunctionModelSpec(
factory_function=self.model,
model_kwargs=self.model_kwargs,
model_gen_kwargs=self.model_gen_kwargs,
)
else:
model_spec = ModelSpec(
model_enum=self.model,
model_kwargs=self.model_kwargs,
model_gen_kwargs=self.model_gen_kwargs,
)
super().__init__(model_specs=[model_spec])
def test_model_key(self):
ms = ModelSpec(model_enum=Models.GPEI)
self.assertEqual(ms.model_key, "GPEI")
