def test_choose_next_2(self):
def side_effect(X, derivative):
return np.mean(X, axis=1).reshape((-1, 1))
smbo = SMAC(self.scenario, rng=1).solver
smbo.incumbent = self.scenario.cs.sample_configuration()
smbo.runhistory = RunHistory(aggregate_func=average_cost)
smbo.model = mock.Mock(spec=RandomForestWithInstances)
smbo.acquisition_func._compute = mock.Mock(
spec=RandomForestWithInstances)
smbo.acquisition_func._compute.side_effect = side_effect
X = smbo.rng.rand(10, 2)
Y = smbo.rng.rand(10, 1)
x = smbo.choose_next(X, Y)
self.assertEqual(smbo.model.train.call_count, 1)
self.assertEqual(len(x), 2020)
num_random_search = 0
num_local_search = 0
for i in range(0, 2020, 2):
#print(x[i].origin)
self.assertIsInstance(x[i], Configuration)
if 'Random Search (sorted)' in x[i].origin:
num_random_search += 1
elif 'Local Search' in x[i].origin:
num_local_search += 1
# number of local search configs has to be least 10
# since x can have duplicates
# which can be associated with the local search
self.assertGreaterEqual(num_local_search, 10)
for i in range(1, 2020, 2):
self.assertIsInstance(x[i], Configuration)
self.assertEqual(x[i].origin, 'Random Search')
def test_get_next_by_local_search(self, patch):
# Without known incumbent
class SideEffect(object):
def __init__(self):
self.call_number = 0
def __call__(self, *args, **kwargs):
rval = 9 - self.call_number
self.call_number += 1
return (ConfigurationMock(rval), [rval])
patch.side_effect = SideEffect()
smbo = SMAC(self.scenario, rng=1).solver
rand_confs = smbo.config_space.sample_configuration(size=9)
rval = smbo._get_next_by_local_search(init_points=rand_confs)
self.assertEqual(len(rval), 9)
self.assertEqual(patch.call_count, 9)
for i in range(9):
self.assertIsInstance(rval[i][1], ConfigurationMock)
self.assertEqual(rval[i][1].value, 9 - i)
self.assertEqual(rval[i][0], 9 - i)
self.assertEqual(rval[i][1].origin, 'Local Search')
# With known incumbent
patch.side_effect = SideEffect()
smbo.incumbent = 'Incumbent'
rval = smbo._get_next_by_local_search(init_points=[smbo.incumbent] +
rand_confs)
self.assertEqual(len(rval), 10)
self.assertEqual(patch.call_count, 19)
# Only the first local search in each iteration starts from the
# incumbent
self.assertEqual(patch.call_args_list[9][0][0], 'Incumbent')
for i in range(10):
self.assertEqual(rval[i][1].origin, 'Local Search')
def test_choose_next(self):
seed = 42
smbo = SMAC(self.scenario, rng=seed).solver
smbo.runhistory = RunHistory(aggregate_func=average_cost)
X = self.scenario.cs.sample_configuration().get_array()[None, :]
smbo.incumbent = self.scenario.cs.sample_configuration()
Y = self.branin(X)
x = smbo.choose_next(X, Y)[0].get_array()
assert x.shape == (2, )
def test_choose_next_3(self):
# Test with ten configurations in the runhistory
def side_effect(X):
return np.mean(X, axis=1).reshape((-1, 1))
def side_effect_predict(X):
m, v = np.ones((X.shape[0], 1)), None
return m, v
smbo = SMAC(self.scenario, rng=1).solver
smbo.incumbent = self.scenario.cs.sample_configuration()
previous_configs = [smbo.incumbent] + [
self.scenario.cs.sample_configuration() for i in range(0, 20)
]
smbo.runhistory = RunHistory(aggregate_func=average_cost)
for i in range(0, len(previous_configs)):
smbo.runhistory.add(previous_configs[i], i, 10, 1)
smbo.model = mock.Mock(spec=RandomForestWithInstances)
smbo.model.predict_marginalized_over_instances.side_effect = side_effect_predict
smbo.acquisition_func._compute = mock.Mock(
spec=RandomForestWithInstances)
smbo.acquisition_func._compute.side_effect = side_effect
X = smbo.rng.rand(10, 2)
Y = smbo.rng.rand(10, 1)
challengers = smbo.choose_next(X, Y)
# Convert challenger list (a generator) to a real list
challengers = [c for c in challengers]
self.assertEqual(smbo.model.train.call_count, 1)
# For each configuration it is randomly sampled whether to take it from the list of challengers or to sample it
# completely at random. Therefore, it is not guaranteed to obtain twice the number of configurations selected
# by EI.
self.assertEqual(len(challengers), 9913)
num_random_search_sorted = 0
num_random_search = 0
num_local_search = 0
for c in challengers:
self.assertIsInstance(c, Configuration)
if 'Random Search (sorted)' == c.origin:
num_random_search_sorted += 1
elif 'Random Search' == c.origin:
num_random_search += 1
elif 'Local Search' == c.origin:
num_local_search += 1
else:
raise ValueError(c.origin)
self.assertEqual(num_local_search, 10)
self.assertEqual(num_random_search_sorted, 4990)
self.assertEqual(num_random_search, 4913)
