def config_space(self):
"""Decision tree hyperparameter space."""
n_estimators = UniformIntegerHyperparameter(
'n_estimators', lower=10, upper=200, default_value=100
)
criterion = CategoricalHyperparameter(
'criterion', ['gini', 'entropy'], default_value='gini'
)
max_depth = CategoricalHyperparameter(
'max_depth', [5, 10, 20, 'none'], default_value='none'
)
max_features = CategoricalHyperparameter(
'max_features', ['auto', 'sqrt', 'log2', 'none'], default_value='none'
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters(
(
n_estimators,
criterion,
max_depth,
max_features
)
)
return config
def config_space(self):
"""Decision tree hyperparameter space."""
criterion = CategoricalHyperparameter(
'criterion', ['gini', 'entropy'], default_value='gini'
)
# NOTE: Default value = 'none' is translated to None in base class.
# ConfigSpace does not allow for None values as default.
max_depth = CategoricalHyperparameter(
'max_depth', [5, 10, 20, 'none'], default_value='none'
)
# NOTE: Default value = 'none' is translated to None in base class.
# ConfigSpace does not allow for None values as default.
max_features = CategoricalHyperparameter(
'max_features', ['auto', 'sqrt', 'log2', 'none'],
default_value='none'
)
min_samples_leaf = UniformIntegerHyperparameter(
'min_samples_leaf', lower=2, upper=5, default_value=3
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters(
(
criterion,
max_depth,
max_features,
min_samples_leaf
)
)
return config
def config_space(self):
"""Logistic regression hyperparameter space."""
reg_param = UniformFloatHyperparameter(
'reg_param', lower=1e-9, upper=1-1e-9, default_value=1e-3
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameter(reg_param)
return config
def config_space(self):
"""Logistic regression hyperparameter space."""
alpha = UniformFloatHyperparameter(
'alpha', lower=1e-8, upper=100, default_value=1.0
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameter(alpha)
return config
def config_space(self):
"""Logistic regression hyperparameter space."""
n_components = UniformIntegerHyperparameter(
'n_components', lower=2, upper=50, default_value=10
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameter(n_components)
return config
def config_space(self):
"""Returns the Fisher score hyperparameter configuration space."""
num_features = UniformIntegerHyperparameter('num_features',
lower=2,
upper=50,
default_value=20)
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameter(num_features)
return config
def config_space(self):
"""Logistic regression hyperparameter space."""
C_param = UniformFloatHyperparameter(
'C', lower=1e-8, upper=1000.0, default_value=1.0
)
penalty = CategoricalHyperparameter(
'penalty', ['l1', 'l2'], default_value='l1'
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters((C_param, penalty))
return config
def config_space(self):
"""Returns the MI hyperparameter configuration space."""
num_neighbors = UniformIntegerHyperparameter('num_neighbors',
lower=10,
upper=100,
default_value=20)
num_features = UniformIntegerHyperparameter('num_features',
lower=2,
upper=50,
default_value=20)
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters((num_neighbors, num_features))
return config
def _get_acm_cs(self):
"""
returns a configuration space
designed for querying ~smac.optimizer.smbo._component_builder
Returns
-------
ConfigurationSpace
"""
cs = ConfigurationSpace()
cs.seed(self.rng.randint(0,2**20))
model = CategoricalHyperparameter("model", choices=("RF", "GP"))
num_trees = Constant("num_trees", value=10)
bootstrap = CategoricalHyperparameter("do_bootstrapping", choices=(True, False), default_value=True)
ratio_features = CategoricalHyperparameter("ratio_features", choices=(3 / 6, 4 / 6, 5 / 6, 1), default_value=1)
min_split = UniformIntegerHyperparameter("min_samples_to_split", lower=1, upper=10, default_value=2)
min_leaves = UniformIntegerHyperparameter("min_samples_in_leaf", lower=1, upper=10, default_value=1)
cs.add_hyperparameters([model, num_trees, bootstrap, ratio_features, min_split, min_leaves])
inc_num_trees = InCondition(num_trees, model, ["RF"])
inc_bootstrap = InCondition(bootstrap, model, ["RF"])
inc_ratio_features = InCondition(ratio_features, model, ["RF"])
inc_min_split = InCondition(min_split, model, ["RF"])
inc_min_leavs = InCondition(min_leaves, model, ["RF"])
cs.add_conditions([inc_num_trees, inc_bootstrap, inc_ratio_features, inc_min_split, inc_min_leavs])
acq = CategoricalHyperparameter("acq_func", choices=("EI", "LCB", "PI", "LogEI"))
par_ei = UniformFloatHyperparameter("par_ei", lower=-10, upper=10)
par_pi = UniformFloatHyperparameter("par_pi", lower=-10, upper=10)
par_logei = UniformFloatHyperparameter("par_logei", lower=0.001, upper=100, log=True)
par_lcb = UniformFloatHyperparameter("par_lcb", lower=0.0001, upper=0.9999)
cs.add_hyperparameters([acq, par_ei, par_pi, par_logei, par_lcb])
inc_par_ei = InCondition(par_ei, acq, ["EI"])
inc_par_pi = InCondition(par_pi, acq, ["PI"])
inc_par_logei = InCondition(par_logei, acq, ["LogEI"])
inc_par_lcb = InCondition(par_lcb, acq, ["LCB"])
cs.add_conditions([inc_par_ei, inc_par_pi, inc_par_logei, inc_par_lcb])
return cs
def config_space(self):
"""SVC hyperparameter space."""
C_param = UniformFloatHyperparameter(
'C', lower=1e-8, upper=100.0, default_value=1.0
)
shrinking = CategoricalHyperparameter(
'shrinking', [True, False], default_value=True
)
kernel = CategoricalHyperparameter(
'kernel', ['linear', 'rbf', 'poly', 'sigmoid'],
)
degree = UniformIntegerHyperparameter(
'degree', lower=1, upper=5, default_value=2
)
coef0 = UniformFloatHyperparameter(
'coef0', lower=0.0, upper=10.0, default_value=0.0
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters(
(
C_param,
shrinking,
kernel,
degree,
coef0,
)
)
# Conditionals on hyperparameters specific to kernels.
config.add_conditions(
(
InCondition(child=degree, parent=kernel, values=['poly']),
InCondition(
child=coef0, parent=kernel, values=['poly', 'sigmoid']
)
)
)
return config
def config_space(self):
"""XGBoost hyperparameter space."""
# The mumber of Decision Trees.
n_estimators = UniformIntegerHyperparameter(
'n_estimators', lower=10, upper=200, default_value=100
)
# The maximum depth of each decision tree. Generally, boosting
# algorithms are configured with weak learners = shallow decision trees.
max_depth = UniformIntegerHyperparameter(
'max_depth', lower=5, upper=500, default_value=100
)
# L1 regularization term on weights.
reg_alpha = UniformFloatHyperparameter(
'reg_alpha', lower=1e-8, upper=100, default_value=1e-3
)
# L2 regularization term on weights.
reg_lambda = UniformFloatHyperparameter(
'reg_lambda', lower=1e-8, upper=100, default_value=1e-3
)
learning_rate = UniformFloatHyperparameter(
'learning_rate', lower=1e-8, upper=50, default_value=0.01
)
min_data_in_leaf = UniformIntegerHyperparameter(
'min_data_in_leaf', lower=2, upper=5, default_value=3
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters(
(
n_estimators,
min_data_in_leaf,
max_depth,
reg_alpha,
reg_lambda,
learning_rate
)
)
return config
def config_space(self):
"""KNN hyperparameter space."""
n_neighbors = UniformIntegerHyperparameter(
'n_neighbors', lower=1, upper=100, default_value=5
)
leaf_size = UniformIntegerHyperparameter(
'leaf_size', lower=1, upper=100, default_value=20
)
metric = CategoricalHyperparameter(
'metric',
['euclidean', 'manhattan', 'chebyshev', 'minkowski'],
default_value='euclidean'
)
p_param = UniformIntegerHyperparameter('p', 1, 5, default_value=2)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters((n_neighbors, leaf_size, metric, p_param))
# Conditionals on hyperparameters specific to kernels.
config.add_condition(
InCondition(child=p_param, parent=metric, values=['minkowski'])
)
return config
def test_impute_inactive_hyperparameters(self):
cs = ConfigurationSpace()
a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1]))
b = cs.add_hyperparameter(CategoricalHyperparameter('b', [0, 1]))
c = cs.add_hyperparameter(UniformFloatHyperparameter('c', 0, 1))
cs.add_condition(EqualsCondition(b, a, 1))
cs.add_condition(EqualsCondition(c, a, 0))
cs.seed(1)
configs = cs.sample_configuration(size=100)
config_array = convert_configurations_to_array(configs)
for line in config_array:
if line[0] == 0:
self.assertTrue(np.isnan(line[1]))
elif line[0] == 1:
self.assertTrue(np.isnan(line[2]))
gp = get_gp(3, np.random.RandomState(1))
config_array = gp._impute_inactive(config_array)
for line in config_array:
if line[0] == 0:
self.assertEqual(line[1], -1)
elif line[0] == 1:
self.assertEqual(line[2], -1)
class TestLocalSearch(unittest.TestCase):
def setUp(self):
current_dir = os.path.dirname(__file__)
self.test_files_dir = os.path.join(current_dir, '..', 'test_files')
seed = np.random.randint(1, 100000)
self.cs = ConfigurationSpace(seed=seed)
x1 = UniformFloatHyperparameter("x1", -5, 5, default_value=5)
self.cs.add_hyperparameter(x1)
x2 = UniformIntegerHyperparameter("x2", -5, 5, default_value=5)
self.cs.add_hyperparameter(x2)
x3 = CategoricalHyperparameter("x3",
[5, 2, 0, 1, -1, -2, 4, -3, 3, -5, -4],
default_value=5)
self.cs.add_hyperparameter(x3)
x4 = UniformIntegerHyperparameter("x4", -5, 5, default_value=5)
self.cs.add_hyperparameter(x4)
def test_local_search(self):
def acquisition_function(points):
rval = []
for point in points:
opt = np.array([1, 1, 1, 1])
rval.append([-euclidean(point.get_array(), opt)])
return np.array(rval)
ls = LocalSearch(acquisition_function, self.cs, max_steps=100)
start_point = self.cs.sample_configuration()
acq_val_start_point = acquisition_function([start_point])
acq_val_incumbent, _ = ls._do_search(start_point)[0]
# Local search needs to find something that is as least as good as the
# start point
self.assertLessEqual(acq_val_start_point, acq_val_incumbent)
@unittest.mock.patch.object(LocalSearch, '_get_initial_points')
def test_local_search_2(
self,
_get_initial_points_patch,
):
pcs_file = os.path.join(self.test_files_dir, "test_local_search.pcs")
seed = np.random.randint(1, 100000)
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
with open(pcs_file) as fh:
config_space = pcs.read(fh.readlines())
config_space.seed(seed)
def acquisition_function(points):
return np.array([[np.count_nonzero(point.get_array())]
for point in points])
start_point = config_space.get_default_configuration()
_get_initial_points_patch.return_value = [start_point]
ls = LocalSearch(acquisition_function, config_space, max_steps=100000)
# To have some data in a mock runhistory
ls.runhistory = [None] * 1000
acq_val_incumbent, incumbent = ls._maximize(runhistory, None, 1)[0]
np.testing.assert_allclose(
incumbent.get_array(),
np.ones(len(config_space.get_hyperparameters())))
@unittest.mock.patch.object(LocalSearch, '_do_search')
@unittest.mock.patch.object(LocalSearch, '_get_initial_points')
def test_get_next_by_local_search(self, _get_initial_points_patch, patch):
# Without known incumbent
class SideEffect(object):
def __call__(self, *args, **kwargs):
rval = []
for i in range(len(args[0])):
rval.append((i, ConfigurationMock(i)))
return rval
patch.side_effect = SideEffect()
cs = test_helpers.get_branin_config_space()
rand_confs = cs.sample_configuration(size=9)
_get_initial_points_patch.return_value = rand_confs
acq_func = EI(None)
ls = LocalSearch(acq_func, cs)
# To have some data in a mock runhistory
runhistory = unittest.mock.Mock()
runhistory.data = [None] * 1000
rval = ls._maximize(runhistory, None, 9)
self.assertEqual(len(rval), 9)
self.assertEqual(patch.call_count, 1)
for i in range(9):
self.assertIsInstance(rval[i][1], ConfigurationMock)
self.assertEqual(rval[i][1].value, 8 - i)
self.assertEqual(rval[i][0], 8 - i)
self.assertEqual(rval[i][1].origin, 'Local Search')
# Check that the known 'incumbent' is transparently passed through
patch.side_effect = SideEffect()
_get_initial_points_patch.return_value = ['Incumbent'] + rand_confs
rval = ls._maximize(runhistory, None, 10)
self.assertEqual(len(rval), 10)
self.assertEqual(patch.call_count, 2)
# Only the first local search in each iteration starts from the
# incumbent
self.assertEqual(patch.call_args_list[1][0][0][0], 'Incumbent')
for i in range(10):
self.assertEqual(rval[i][1].origin, 'Local Search')
def test_local_search_finds_minimum(self):
class AcquisitionFunction:
model = None
def __call__(self, arrays):
rval = []
for array in arrays:
rval.append([-rosenbrock_4d(array)])
return np.array(rval)
ls = LocalSearch(
acquisition_function=AcquisitionFunction(),
config_space=self.cs,
n_steps_plateau_walk=10,
max_steps=np.inf,
)
runhistory = RunHistory()
self.cs.seed(1)
random_configs = self.cs.sample_configuration(size=100)
costs = [
rosenbrock_4d(random_config) for random_config in random_configs
]
self.assertGreater(np.min(costs), 100)
for random_config, cost in zip(random_configs, costs):
runhistory.add(config=random_config,
cost=cost,
time=0,
status=StatusType.SUCCESS)
minimizer = ls.maximize(runhistory, None, 10)
minima = [-rosenbrock_4d(m) for m in minimizer]
self.assertGreater(minima[0], -0.05)
def test_get_initial_points_moo(self):
class Model:
def predict_marginalized_over_instances(self, X):
return X, X
class AcquisitionFunction:
model = Model()
def __call__(self, X):
return np.array([x.get_array().sum() for x in X]).reshape(
(-1, 1))
ls = LocalSearch(
acquisition_function=AcquisitionFunction(),
config_space=self.cs,
n_steps_plateau_walk=10,
max_steps=np.inf,
)
runhistory = RunHistory()
random_configs = self.cs.sample_configuration(size=100)
costs = np.array(
[rosenbrock_4d(random_config) for random_config in random_configs])
for random_config, cost in zip(random_configs, costs):
runhistory.add(config=random_config,
cost=cost,
time=0,
status=StatusType.SUCCESS)
points = ls._get_initial_points(num_points=5,
runhistory=runhistory,
additional_start_points=None)
self.assertEqual(len(points), 10)
