def setUp(self) -> None:
self.experiment = get_experiment()
self.arm = Arm({"x": 1, "y": "foo", "z": True, "w": 4})
self.trial = self.experiment.new_trial(GeneratorRun([self.arm]))
self.experiment_2 = get_experiment()
self.batch_trial = self.experiment_2.new_batch_trial(GeneratorRun([self.arm]))
self.batch_trial.set_status_quo_with_weight(self.experiment_2.status_quo, 1)
self.obs_feat = ObservationFeatures.from_arm(
arm=self.trial.arm, trial_index=np.int64(self.trial.index)
)
def test_setting_random_seed(self):
sobol = choose_generation_strategy(
search_space=get_factorial_search_space(), random_seed=9
)
sobol.gen(experiment=get_experiment())
# First model is actually a bridge, second is the Sobol engine.
self.assertEqual(sobol.model.model.seed, 9)
def get_random_experiment(self) -> Experiment:
"""Get an Experiment instance with random name."""
experiment = get_experiment()
experiment_name = "".join(random.choice(string.ascii_letters) for i in range(8))
experiment.name = experiment_name
return experiment
def get_generation_strategy(
with_experiment: bool = False) -> GenerationStrategy:
gs = choose_generation_strategy(search_space=get_search_space())
if with_experiment:
gs._experiment = get_experiment()
fake_func = get_experiment
# pyre-ignore[16]: testing hack to test serialization of callable kwargs
# in generation steps.
gs._steps[0].model_kwargs["model_constructor"] = fake_func
return gs
def setUp(self):
self.experiment = get_experiment()
self.experiment.status_quo = None
self.batch = self.experiment.new_batch_trial()
arms = get_arms()
weights = get_weights()
self.status_quo = arms[0]
self.sq_weight = weights[0]
self.arms = arms[1:]
self.weights = weights[1:]
self.batch.add_arms_and_weights(arms=self.arms, weights=self.weights)
def testBasicBatchCreation(self):
batch = self.experiment.new_batch_trial()
self.assertEqual(len(self.experiment.trials), 1)
self.assertEqual(self.experiment.trials[0], batch)
# Try (and fail) to re-attach batch
with self.assertRaises(ValueError):
self.experiment._attach_trial(batch)
# Try (and fail) to attach batch to another experiment
with self.assertRaises(ValueError):
new_exp = get_experiment()
new_exp._attach_trial(batch)
def setUp(self) -> None:
self.experiment = get_experiment()
self.arm = Arm({"x": 1, "y": "foo", "z": True, "w": 4})
self.trial = self.experiment.new_trial(GeneratorRun([self.arm]))
self.experiment_2 = get_experiment()
self.batch_trial = self.experiment_2.new_batch_trial(
GeneratorRun([self.arm]))
self.batch_trial.set_status_quo_with_weight(
self.experiment_2.status_quo, 1)
self.obs_feat = ObservationFeatures.from_arm(arm=self.trial.arm,
trial_index=np.int64(
self.trial.index))
self.hss_exp = get_hierarchical_search_space_experiment()
self.hss_sobol = Models.SOBOL(search_space=self.hss_exp.search_space)
self.hss_gr = self.hss_sobol.gen(n=1)
self.hss_trial = self.hss_exp.new_trial(self.hss_gr)
self.hss_arm = not_none(self.hss_trial.arm)
self.hss_cand_metadata = self.hss_trial._get_candidate_metadata(
arm_name=self.hss_arm.name)
self.hss_full_parameterization = self.hss_cand_metadata.get(
Keys.FULL_PARAMETERIZATION).copy()
self.assertTrue(
all(p_name in self.hss_full_parameterization
for p_name in self.hss_exp.search_space.parameters))
self.hss_obs_feat = ObservationFeatures.from_arm(
arm=self.hss_arm,
trial_index=np.int64(self.hss_trial.index),
metadata=self.hss_cand_metadata,
)
self.hss_obs_feat_all_params = ObservationFeatures.from_arm(
arm=Arm(self.hss_full_parameterization),
trial_index=np.int64(self.hss_trial.index),
metadata={
Keys.FULL_PARAMETERIZATION: self.hss_full_parameterization
},
)
def testClampObservationFeaturesNearBounds(self):
cases = [
(
ObservationFeatures(
parameters={"w": 1.0, "x": 2, "y": "foo", "z": True}
),
ObservationFeatures(
parameters={"w": 1.0, "x": 2, "y": "foo", "z": True}
),
),
(
ObservationFeatures(
parameters={"w": 0.0, "x": 2, "y": "foo", "z": True}
),
ObservationFeatures(
parameters={"w": 0.5, "x": 2, "y": "foo", "z": True}
),
),
(
ObservationFeatures(
parameters={"w": 100.0, "x": 2, "y": "foo", "z": True}
),
ObservationFeatures(
parameters={"w": 5.5, "x": 2, "y": "foo", "z": True}
),
),
(
ObservationFeatures(
parameters={"w": 1.0, "x": 0, "y": "foo", "z": True}
),
ObservationFeatures(
parameters={"w": 1.0, "x": 1, "y": "foo", "z": True}
),
),
(
ObservationFeatures(
parameters={"w": 1.0, "x": 11, "y": "foo", "z": True}
),
ObservationFeatures(
parameters={"w": 1.0, "x": 10, "y": "foo", "z": True}
),
),
]
search_space = get_experiment().search_space
for obs_ft, expected_obs_ft in cases:
actual_obs_ft = clamp_observation_features([obs_ft], search_space)
self.assertEqual(actual_obs_ft[0], expected_obs_ft)
def get_modelbridge(mock_gen_arms,
mock_observations_from_data,
status_quo_name: Optional[str] = None) -> ModelBridge:
exp = get_experiment()
modelbridge = ModelBridge(
search_space=get_search_space(),
model=FullFactorialGenerator(),
experiment=exp,
data=get_data(),
status_quo_name=status_quo_name,
)
modelbridge._predict = mock.MagicMock(
"ax.modelbridge.base.ModelBridge._predict",
autospec=True,
return_value=[get_observation().data],
)
return modelbridge
def setUp(self):
self.experiment = get_experiment()
def setUp(self):
self.experiment = get_experiment()
self.trial = self.experiment.new_trial()
self.arm = get_arms()[0]
self.trial.add_arm(self.arm)
def get_generation_strategy(
with_experiment: bool = False) -> GenerationStrategy:
gs = choose_generation_strategy(search_space=get_search_space())
if with_experiment:
gs._experiment = get_experiment()
return gs
def setUp(self):
self.exp = get_experiment()
init_test_engine_and_session_factory(force=True)
self.db_settings = DBSettings(url="sqlite://")
save_experiment(self.exp, self.db_settings)
def test_setting_experiment_attribute(self):
exp = get_experiment()
gs = choose_generation_strategy(search_space=exp.search_space,
experiment=exp)
self.assertEqual(gs._experiment, exp)
