def test_factorial_thompson_strategy(self, mock_update, mock_gen, mock_discrete): exp = get_branin_experiment() factorial_thompson_generation_strategy = GenerationStrategy(steps=[ GenerationStep(model=Models.FACTORIAL, num_arms=1), GenerationStep(model=Models.THOMPSON, num_arms=-1), ]) self.assertEqual(factorial_thompson_generation_strategy.name, "factorial+thompson") self.assertEqual( factorial_thompson_generation_strategy.generator_changes, [1]) for i in range(2): data = get_data() if i > 0 else None factorial_thompson_generation_strategy.gen(experiment=exp, new_data=data) exp.new_batch_trial().add_arm(Arm(parameters={"x1": i, "x2": i})) if i < 1: mock_discrete.assert_called() args, kwargs = mock_discrete.call_args self.assertIsInstance(kwargs.get("model"), FullFactorialGenerator) exp.new_batch_trial() else: mock_discrete.assert_called() args, kwargs = mock_discrete.call_args self.assertIsInstance( kwargs.get("model"), (ThompsonSampler, EmpiricalBayesThompsonSampler), )
def test_equality(self): gs1 = GenerationStrategy(steps=[ GenerationStep(model=Models.SOBOL, num_arms=5), GenerationStep(model=Models.GPEI, num_arms=-1), ]) gs2 = gs1.clone_reset() self.assertEqual(gs1, gs2) gs1._data = get_data() self.assertNotEqual(gs1, gs2) gs2._data = get_data() self.assertEqual(gs1, gs2) gs1 = gs1.clone_reset() gs2 = gs2.clone_reset() gs1.gen(experiment=get_branin_experiment()) self.assertNotEqual(gs1, gs2) gs2.gen(experiment=get_branin_experiment()) # Each generation strategy generated a different arm, so they are not # equal even though they have the same setup and generated the same # number of arms. self.assertNotEqual(gs1, gs2)
def testEmptyMetrics(self): empty_experiment = Experiment(name="test_experiment", search_space=get_search_space()) self.assertEqual(empty_experiment.num_trials, 0) with self.assertRaises(ValueError): empty_experiment.fetch_data() batch = empty_experiment.new_batch_trial() self.assertEqual(empty_experiment.num_trials, 1) with self.assertRaises(ValueError): batch.fetch_data() empty_experiment.add_tracking_metric(Metric(name="some_metric")) empty_experiment.attach_data(get_data()) self.assertFalse(empty_experiment.fetch_data().df.empty)
def get_modelbridge(mock_gen_arms, mock_observations_from_data, status_quo_name: Optional[str] = None) -> ModelBridge: exp = get_experiment() modelbridge = ModelBridge( search_space=get_search_space(), model=FullFactorialGenerator(), experiment=exp, data=get_data(), status_quo_name=status_quo_name, ) modelbridge._predict = mock.MagicMock( "ax.modelbridge.base.ModelBridge._predict", autospec=True, return_value=[get_observation().data], ) return modelbridge