def test_choose_next_2(self):
def side_effect(X, derivative):
return np.mean(X, axis=1).reshape((-1, 1))
smbo = SMAC(self.scenario, rng=1).solver
smbo.incumbent = self.scenario.cs.sample_configuration()
smbo.runhistory = RunHistory(aggregate_func=average_cost)
smbo.model = mock.Mock(spec=RandomForestWithInstances)
smbo.acquisition_func._compute = mock.Mock(
spec=RandomForestWithInstances)
smbo.acquisition_func._compute.side_effect = side_effect
X = smbo.rng.rand(10, 2)
Y = smbo.rng.rand(10, 1)
x = smbo.choose_next(X, Y)
self.assertEqual(smbo.model.train.call_count, 1)
self.assertEqual(len(x), 2020)
num_random_search = 0
num_local_search = 0
for i in range(0, 2020, 2):
#print(x[i].origin)
self.assertIsInstance(x[i], Configuration)
if 'Random Search (sorted)' in x[i].origin:
num_random_search += 1
elif 'Local Search' in x[i].origin:
num_local_search += 1
# number of local search configs has to be least 10
# since x can have duplicates
# which can be associated with the local search
self.assertGreaterEqual(num_local_search, 10)
for i in range(1, 2020, 2):
self.assertIsInstance(x[i], Configuration)
self.assertEqual(x[i].origin, 'Random Search')
def test_choose_next_3(self):
# Test with ten configurations in the runhistory
def side_effect(X):
return np.mean(X, axis=1).reshape((-1, 1))
def side_effect_predict(X):
m, v = np.ones((X.shape[0], 1)), None
return m, v
smbo = SMAC(self.scenario, rng=1).solver
smbo.incumbent = self.scenario.cs.sample_configuration()
previous_configs = [smbo.incumbent] + [
self.scenario.cs.sample_configuration() for i in range(0, 20)
]
smbo.runhistory = RunHistory(aggregate_func=average_cost)
for i in range(0, len(previous_configs)):
smbo.runhistory.add(previous_configs[i], i, 10, 1)
smbo.model = mock.Mock(spec=RandomForestWithInstances)
smbo.model.predict_marginalized_over_instances.side_effect = side_effect_predict
smbo.acquisition_func._compute = mock.Mock(
spec=RandomForestWithInstances)
smbo.acquisition_func._compute.side_effect = side_effect
X = smbo.rng.rand(10, 2)
Y = smbo.rng.rand(10, 1)
challengers = smbo.choose_next(X, Y)
# Convert challenger list (a generator) to a real list
challengers = [c for c in challengers]
self.assertEqual(smbo.model.train.call_count, 1)
# For each configuration it is randomly sampled whether to take it from the list of challengers or to sample it
# completely at random. Therefore, it is not guaranteed to obtain twice the number of configurations selected
# by EI.
self.assertEqual(len(challengers), 9913)
num_random_search_sorted = 0
num_random_search = 0
num_local_search = 0
for c in challengers:
self.assertIsInstance(c, Configuration)
if 'Random Search (sorted)' == c.origin:
num_random_search_sorted += 1
elif 'Random Search' == c.origin:
num_random_search += 1
elif 'Local Search' == c.origin:
num_local_search += 1
else:
raise ValueError(c.origin)
self.assertEqual(num_local_search, 10)
self.assertEqual(num_random_search_sorted, 4990)
self.assertEqual(num_random_search, 4913)
