def test_impute_inactive_hyperparameters(self):
cs = ConfigurationSpace()
a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1]))
b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1]))
c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1))
cs.add_condition(EqualsCondition(b, a, 1))
cs.add_condition(EqualsCondition(c, a, 0))
cs.seed(1)
configs = cs.sample_configuration(size=100)
config_array = convert_configurations_to_array(configs)
for line in config_array:
if line[0] == 0:
self.assertTrue(np.isnan(line[1]))
elif line[0] == 1:
self.assertTrue(np.isnan(line[2]))
gp = get_gp(3, np.random.RandomState(1))
config_array = gp._impute_inactive(config_array)
for line in config_array:
if line[0] == 0:
self.assertEqual(line[1], -1)
elif line[0] == 1:
self.assertEqual(line[2], -1)
class TestLocalSearch(unittest.TestCase):
def setUp(self):
current_dir = os.path.dirname(__file__)
self.test_files_dir = os.path.join(current_dir, '..', 'test_files')
seed = np.random.randint(1, 100000)
self.cs = ConfigurationSpace(seed=seed)
x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5)
self.cs.add_hyperparameter(x1)
x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5)
self.cs.add_hyperparameter(x2)
x3 = CategoricalHyperparameter("x3",
[5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4],
default_value=5)
self.cs.add_hyperparameter(x3)
x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5)
self.cs.add_hyperparameter(x4)
def test_local_search(self):
def acquisition_function(points):
rval = []
for point in points:
opt = np.array([1, 1, 1, 1])
rval.append([-euclidean(point.get_array(), opt)])
return np.array(rval)
ls = LocalSearch(acquisition_function, self.cs, max_steps=100)
start_point = self.cs.sample_configuration()
acq_val_start_point = acquisition_function([start_point])
acq_val_incumbent, _ = ls._do_search(start_point)[0]
# Local search needs to find something that is as least as good as the
# start point
self.assertLessEqual(acq_val_start_point, acq_val_incumbent)
@unittest.mock.patch.object(LocalSearch, '_get_initial_points')
def test_local_search_2(
self,
_get_initial_points_patch,
):
pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs")
seed = np.random.randint(1, 100000)
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
with open(pcs_file) as fh:
config_space = pcs.read(fh.readlines())
config_space.seed(seed)
def acquisition_function(points):
return np.array([[np.count_nonzero(point.get_array())]
for point in points])
start_point = config_space.get_default_configuration()
_get_initial_points_patch.return_value = [start_point]
ls = LocalSearch(acquisition_function, config_space, max_steps=100000)
# To have some data in a mock runhistory
ls.runhistory = [None] * 1000
acq_val_incumbent, incumbent = ls._maximize(runhistory, None, 1)[0]
np.testing.assert_allclose(
incumbent.get_array(),
np.ones(len(config_space.get_hyperparameters())))
@unittest.mock.patch.object(LocalSearch, '_do_search')
@unittest.mock.patch.object(LocalSearch, '_get_initial_points')
def test_get_next_by_local_search(self, _get_initial_points_patch, patch):
# Without known incumbent
class SideEffect(object):
def __call__(self, *args, **kwargs):
rval = []
for i in range(len(args[0])):
rval.append((i, ConfigurationMock(i)))
return rval
patch.side_effect = SideEffect()
cs = test_helpers.get_branin_config_space()
rand_confs = cs.sample_configuration(size=9)
_get_initial_points_patch.return_value = rand_confs
acq_func = EI(None)
ls = LocalSearch(acq_func, cs)
# To have some data in a mock runhistory
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
rval = ls._maximize(runhistory, None, 9)
self.assertEqual(len(rval), 9)
self.assertEqual(patch.call_count, 1)
for i in range(9):
self.assertIsInstance(rval[i][1], ConfigurationMock)
self.assertEqual(rval[i][1].value, 8 - i)
self.assertEqual(rval[i][0], 8 - i)
self.assertEqual(rval[i][1].origin, 'Local Search')
# Check that the known 'incumbent' is transparently passed through
patch.side_effect = SideEffect()
_get_initial_points_patch.return_value = ['Incumbent'] + rand_confs
rval = ls._maximize(runhistory, None, 10)
self.assertEqual(len(rval), 10)
self.assertEqual(patch.call_count, 2)
# Only the first local search in each iteration starts from the
# incumbent
self.assertEqual(patch.call_args_list[1][0][0][0], 'Incumbent')
for i in range(10):
self.assertEqual(rval[i][1].origin, 'Local Search')
def test_local_search_finds_minimum(self):
class AcquisitionFunction:
model = None
def __call__(self, arrays):
rval = []
for array in arrays:
rval.append([-rosenbrock_4d(array)])
return np.array(rval)
ls = LocalSearch(
acquisition_function=AcquisitionFunction(),
config_space=self.cs,
n_steps_plateau_walk=10,
max_steps=np.inf,
)
runhistory = RunHistory()
self.cs.seed(1)
random_configs = self.cs.sample_configuration(size=100)
costs = [
rosenbrock_4d(random_config) for random_config in random_configs
]
self.assertGreater(np.min(costs), 100)
for random_config, cost in zip(random_configs, costs):
runhistory.add(config=random_config,
cost=cost,
time=0,
status=StatusType.SUCCESS)
minimizer = ls.maximize(runhistory, None, 10)
minima = [-rosenbrock_4d(m) for m in minimizer]
self.assertGreater(minima[0], -0.05)
def test_get_initial_points_moo(self):
class Model:
def predict_marginalized_over_instances(self, X):
return X, X
class AcquisitionFunction:
model = Model()
def __call__(self, X):
return np.array([x.get_array().sum() for x in X]).reshape(
(-1, 1))
ls = LocalSearch(
acquisition_function=AcquisitionFunction(),
config_space=self.cs,
n_steps_plateau_walk=10,
max_steps=np.inf,
)
runhistory = RunHistory()
random_configs = self.cs.sample_configuration(size=100)
costs = np.array(
[rosenbrock_4d(random_config) for random_config in random_configs])
for random_config, cost in zip(random_configs, costs):
runhistory.add(config=random_config,
cost=cost,
time=0,
status=StatusType.SUCCESS)
points = ls._get_initial_points(num_points=5,
runhistory=runhistory,
additional_start_points=None)
self.assertEqual(len(points), 10)
class TestLocalSearch(unittest.TestCase):
def setUp(self):
current_dir = os.path.dirname(__file__)
self.test_files_dir = os.path.join(current_dir, '..', 'test_files')
seed = np.random.randint(1, 100000)
self.cs = ConfigurationSpace(seed=seed)
x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5)
self.cs.add_hyperparameter(x1)
x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5)
self.cs.add_hyperparameter(x2)
x3 = CategoricalHyperparameter("x3",
[5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4],
default_value=5)
self.cs.add_hyperparameter(x3)
x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5)
self.cs.add_hyperparameter(x4)
def test_local_search(self):
def acquisition_function(point):
point = [p.get_array() for p in point]
opt = np.array([1, 1, 1, 1])
dist = [euclidean(point, opt)]
return np.array([-np.min(dist)])
l = LocalSearch(acquisition_function,
self.cs,
epsilon=1e-10,
max_iterations=100000)
start_point = self.cs.sample_configuration()
acq_val_start_point = acquisition_function([start_point])
acq_val_incumbent, _ = l._one_iter(start_point)
# Local search needs to find something that is as least as good as the
# start point
self.assertLessEqual(acq_val_start_point, acq_val_incumbent)
@unittest.mock.patch.object(LocalSearch, '_get_initial_points')
@unittest.mock.patch.object(LocalSearch, '_calculate_num_points')
def test_local_search_2(
self,
_calculate_num_points_patch,
_get_initial_points_patch,
):
pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs")
seed = np.random.randint(1, 100000)
with open(pcs_file) as fh:
config_space = pcs.read(fh.readlines())
config_space.seed(seed)
def acquisition_function(point):
return np.array([np.count_nonzero(point[0].get_array())])
start_point = config_space.get_default_configuration()
_calculate_num_points_patch.return_value = 1
_get_initial_points_patch.return_value = [start_point]
l = LocalSearch(acquisition_function,
config_space,
epsilon=0.01,
max_iterations=100000)
acq_val_incumbent, incumbent = l._maximize(None, None, 10)[0]
np.testing.assert_allclose(
incumbent.get_array(),
np.ones(len(config_space.get_hyperparameters())))
@unittest.mock.patch.object(LocalSearch, '_one_iter')
@unittest.mock.patch.object(LocalSearch, '_get_initial_points')
@unittest.mock.patch.object(LocalSearch, '_calculate_num_points')
def test_get_next_by_local_search(self, _calculate_num_points_patch,
_get_initial_points_patch, patch):
# Without known incumbent
class SideEffect(object):
def __init__(self):
self.call_number = 0
def __call__(self, *args, **kwargs):
rval = 9 - self.call_number
self.call_number += 1
return (rval, ConfigurationMock(rval))
patch.side_effect = SideEffect()
cs = test_helpers.get_branin_config_space()
rand_confs = cs.sample_configuration(size=9)
_get_initial_points_patch.return_value = rand_confs
_calculate_num_points_patch.return_value = 9
acq_func = EI(None)
ls = LocalSearch(acq_func, cs)
rval = ls._maximize(None, None, 9)
self.assertEqual(len(rval), 9)
self.assertEqual(patch.call_count, 9)
for i in range(9):
self.assertIsInstance(rval[i][1], ConfigurationMock)
self.assertEqual(rval[i][1].value, 9 - i)
self.assertEqual(rval[i][0], 9 - i)
self.assertEqual(rval[i][1].origin, 'Local Search')
# With known incumbent
patch.side_effect = SideEffect()
_get_initial_points_patch.return_value = ['Incumbent'] + rand_confs
_calculate_num_points_patch.return_value = 10
rval = ls._maximize(None, None, 10)
self.assertEqual(len(rval), 10)
self.assertEqual(patch.call_count, 19)
# Only the first local search in each iteration starts from the
# incumbent
self.assertEqual(patch.call_args_list[9][0][0], 'Incumbent')
for i in range(10):
self.assertEqual(rval[i][1].origin, 'Local Search')
class TestLocalSearch(unittest.TestCase):
def setUp(self):
current_dir = os.path.dirname(__file__)
self.test_files_dir = os.path.join(current_dir, '..', 'test_files')
seed = np.random.randint(1, 100000)
self.cs = ConfigurationSpace(seed=seed)
x1 = UniformFloatHyperparameter("x1", -5, 5, default=5)
self.cs.add_hyperparameter(x1)
x2 = UniformIntegerHyperparameter("x2", -5, 5, default=5)
self.cs.add_hyperparameter(x2)
x3 = CategoricalHyperparameter("x3",
[5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4],
default=5)
self.cs.add_hyperparameter(x3)
x4 = UniformIntegerHyperparameter("x4", -5, 5, default=5)
self.cs.add_hyperparameter(x4)
def test_local_search(self):
def acquisition_function(point):
opt = np.array([1, 1, 1, 1])
dist = [euclidean(point, opt)]
return -np.min(dist)
l = LocalSearch(acquisition_function,
self.cs,
epsilon=1e-10,
max_iterations=100000)
start_point = self.cs.sample_configuration()
acq_val_start_point = acquisition_function(start_point.get_array())
_, acq_val_incumbent = l.maximize(start_point)
# Local search needs to find something that is as least as good as the
# start point
self.assertLessEqual(acq_val_start_point, acq_val_incumbent)
def test_local_search_2(self):
pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs")
seed = np.random.randint(1, 100000)
with open(pcs_file) as fh:
config_space = pcs.read(fh.readlines())
config_space.seed(seed)
def acquisition_function(point):
return np.count_nonzero(np.array(point))
start_point = config_space.get_default_configuration()
l = LocalSearch(acquisition_function,
config_space,
epsilon=0.01,
max_iterations=100000)
incumbent, acq_val_incumbent = l.maximize(start_point)
self.assertEqual(acq_val_incumbent, len(start_point.get_array()))
self.assertTrue(
np.all(incumbent.get_array() == np.ones([acq_val_incumbent])))
##
step3_child__xgboost_eta = InCondition(child=hyper_xgboost_eta,
parent=step3,
values=["xgboost"])
step3_child__xgboost_max_depth = InCondition(child=hyper_xgboost_max_depth,
parent=step3,
values=["xgboost"])
step3_child__xgboost_subsample = InCondition(child=hyper_xgboost_subsample,
parent=step3,
values=["xgboost"])
step3_child__xgboost_colsample_bytree = InCondition(
child=hyper_xgboost_colsample_bytree, parent=step3, values=["xgboost"])
step3_child__xgboost_min_child_weight = InCondition(
child=hyper_xgboost_min_child_weight, parent=step3, values=["xgboost"])
##
step3_child__naiveBayes_laplace = InCondition(child=hyper_naiveBayes,
parent=step3,
values=["naiveBayes"])
cs.add_conditions([
step3_child_kknn, step3_child_ksvm_c, step3_child_ksvm_sigma,
step3_child_ranger_mtry, step3_child_ranger_sample_fraction,
step3_child__xgboost_eta, step3_child__xgboost_subsample,
step3_child__xgboost_max_depth, step3_child__xgboost_colsample_bytree,
step3_child__xgboost_min_child_weight, step3_child__naiveBayes_laplace
])
cfg = cs.sample_configuration()
stub = {k: cfg[k] for k in cfg if cfg[k]}
