def test_choose_next_2(self): def side_effect(X, derivative): return np.mean(X, axis=1).reshape((-1, 1)) smbo = SMAC(self.scenario, rng=1).solver smbo.incumbent = self.scenario.cs.sample_configuration() smbo.runhistory = RunHistory(aggregate_func=average_cost) smbo.model = mock.Mock(spec=RandomForestWithInstances) smbo.acquisition_func._compute = mock.Mock( spec=RandomForestWithInstances) smbo.acquisition_func._compute.side_effect = side_effect X = smbo.rng.rand(10, 2) Y = smbo.rng.rand(10, 1) x = smbo.choose_next(X, Y) self.assertEqual(smbo.model.train.call_count, 1) self.assertEqual(len(x), 2020) num_random_search = 0 num_local_search = 0 for i in range(0, 2020, 2): #print(x[i].origin) self.assertIsInstance(x[i], Configuration) if 'Random Search (sorted)' in x[i].origin: num_random_search += 1 elif 'Local Search' in x[i].origin: num_local_search += 1 # number of local search configs has to be least 10 # since x can have duplicates # which can be associated with the local search self.assertGreaterEqual(num_local_search, 10) for i in range(1, 2020, 2): self.assertIsInstance(x[i], Configuration) self.assertEqual(x[i].origin, 'Random Search')
def test_get_next_by_local_search(self, patch): # Without known incumbent class SideEffect(object): def __init__(self): self.call_number = 0 def __call__(self, *args, **kwargs): rval = 9 - self.call_number self.call_number += 1 return (ConfigurationMock(rval), [rval]) patch.side_effect = SideEffect() smbo = SMAC(self.scenario, rng=1).solver rand_confs = smbo.config_space.sample_configuration(size=9) rval = smbo._get_next_by_local_search(init_points=rand_confs) self.assertEqual(len(rval), 9) self.assertEqual(patch.call_count, 9) for i in range(9): self.assertIsInstance(rval[i][1], ConfigurationMock) self.assertEqual(rval[i][1].value, 9 - i) self.assertEqual(rval[i][0], 9 - i) self.assertEqual(rval[i][1].origin, 'Local Search') # With known incumbent patch.side_effect = SideEffect() smbo.incumbent = 'Incumbent' rval = smbo._get_next_by_local_search(init_points=[smbo.incumbent] + rand_confs) self.assertEqual(len(rval), 10) self.assertEqual(patch.call_count, 19) # Only the first local search in each iteration starts from the # incumbent self.assertEqual(patch.call_args_list[9][0][0], 'Incumbent') for i in range(10): self.assertEqual(rval[i][1].origin, 'Local Search')
def test_choose_next(self): seed = 42 smbo = SMAC(self.scenario, rng=seed).solver smbo.runhistory = RunHistory(aggregate_func=average_cost) X = self.scenario.cs.sample_configuration().get_array()[None, :] smbo.incumbent = self.scenario.cs.sample_configuration() Y = self.branin(X) x = smbo.choose_next(X, Y)[0].get_array() assert x.shape == (2, )
def test_choose_next_3(self): # Test with ten configurations in the runhistory def side_effect(X): return np.mean(X, axis=1).reshape((-1, 1)) def side_effect_predict(X): m, v = np.ones((X.shape[0], 1)), None return m, v smbo = SMAC(self.scenario, rng=1).solver smbo.incumbent = self.scenario.cs.sample_configuration() previous_configs = [smbo.incumbent] + [ self.scenario.cs.sample_configuration() for i in range(0, 20) ] smbo.runhistory = RunHistory(aggregate_func=average_cost) for i in range(0, len(previous_configs)): smbo.runhistory.add(previous_configs[i], i, 10, 1) smbo.model = mock.Mock(spec=RandomForestWithInstances) smbo.model.predict_marginalized_over_instances.side_effect = side_effect_predict smbo.acquisition_func._compute = mock.Mock( spec=RandomForestWithInstances) smbo.acquisition_func._compute.side_effect = side_effect X = smbo.rng.rand(10, 2) Y = smbo.rng.rand(10, 1) challengers = smbo.choose_next(X, Y) # Convert challenger list (a generator) to a real list challengers = [c for c in challengers] self.assertEqual(smbo.model.train.call_count, 1) # For each configuration it is randomly sampled whether to take it from the list of challengers or to sample it # completely at random. Therefore, it is not guaranteed to obtain twice the number of configurations selected # by EI. self.assertEqual(len(challengers), 9913) num_random_search_sorted = 0 num_random_search = 0 num_local_search = 0 for c in challengers: self.assertIsInstance(c, Configuration) if 'Random Search (sorted)' == c.origin: num_random_search_sorted += 1 elif 'Random Search' == c.origin: num_random_search += 1 elif 'Local Search' == c.origin: num_local_search += 1 else: raise ValueError(c.origin) self.assertEqual(num_local_search, 10) self.assertEqual(num_random_search_sorted, 4990) self.assertEqual(num_random_search, 4913)