def test_choose_next_2(self): def side_effect(X, derivative): return np.mean(X, axis=1).reshape((-1, 1)) smbo = SMAC(self.scenario, rng=1).solver smbo.incumbent = self.scenario.cs.sample_configuration() smbo.runhistory = RunHistory(aggregate_func=average_cost) smbo.model = mock.Mock(spec=RandomForestWithInstances) smbo.acquisition_func._compute = mock.Mock( spec=RandomForestWithInstances) smbo.acquisition_func._compute.side_effect = side_effect X = smbo.rng.rand(10, 2) Y = smbo.rng.rand(10, 1) x = smbo.choose_next(X, Y) self.assertEqual(smbo.model.train.call_count, 1) self.assertEqual(len(x), 2020) num_random_search = 0 num_local_search = 0 for i in range(0, 2020, 2): #print(x[i].origin) self.assertIsInstance(x[i], Configuration) if 'Random Search (sorted)' in x[i].origin: num_random_search += 1 elif 'Local Search' in x[i].origin: num_local_search += 1 # number of local search configs has to be least 10 # since x can have duplicates # which can be associated with the local search self.assertGreaterEqual(num_local_search, 10) for i in range(1, 2020, 2): self.assertIsInstance(x[i], Configuration) self.assertEqual(x[i].origin, 'Random Search')
def test_choose_next_3(self): # Test with ten configurations in the runhistory def side_effect(X): return np.mean(X, axis=1).reshape((-1, 1)) def side_effect_predict(X): m, v = np.ones((X.shape[0], 1)), None return m, v smbo = SMAC(self.scenario, rng=1).solver smbo.incumbent = self.scenario.cs.sample_configuration() previous_configs = [smbo.incumbent] + [ self.scenario.cs.sample_configuration() for i in range(0, 20) ] smbo.runhistory = RunHistory(aggregate_func=average_cost) for i in range(0, len(previous_configs)): smbo.runhistory.add(previous_configs[i], i, 10, 1) smbo.model = mock.Mock(spec=RandomForestWithInstances) smbo.model.predict_marginalized_over_instances.side_effect = side_effect_predict smbo.acquisition_func._compute = mock.Mock( spec=RandomForestWithInstances) smbo.acquisition_func._compute.side_effect = side_effect X = smbo.rng.rand(10, 2) Y = smbo.rng.rand(10, 1) challengers = smbo.choose_next(X, Y) # Convert challenger list (a generator) to a real list challengers = [c for c in challengers] self.assertEqual(smbo.model.train.call_count, 1) # For each configuration it is randomly sampled whether to take it from the list of challengers or to sample it # completely at random. Therefore, it is not guaranteed to obtain twice the number of configurations selected # by EI. self.assertEqual(len(challengers), 9913) num_random_search_sorted = 0 num_random_search = 0 num_local_search = 0 for c in challengers: self.assertIsInstance(c, Configuration) if 'Random Search (sorted)' == c.origin: num_random_search_sorted += 1 elif 'Random Search' == c.origin: num_random_search += 1 elif 'Local Search' == c.origin: num_local_search += 1 else: raise ValueError(c.origin) self.assertEqual(num_local_search, 10) self.assertEqual(num_random_search_sorted, 4990) self.assertEqual(num_random_search, 4913)