class Dummy():
configuration_space = ConfigurationSpace(seed=1)
flt = UniformFloatHyperparameter("flt", lower=0.0, upper=1.0)
cat = CategoricalHyperparameter("cat", choices=(1, "a"))
itg = UniformIntegerHyperparameter("itg", lower=0, upper=10)
configuration_space.add_hyperparameters([flt, cat, itg])
fidelity_space = ConfigurationSpace(seed=1)
f1 = UniformFloatHyperparameter("f_flt", lower=0.0, upper=1.0)
f2 = CategoricalHyperparameter("f_cat", choices=(1, "a"))
f3 = UniformIntegerHyperparameter("f_itg", lower=0, upper=10)
fidelity_space.add_hyperparameters([f1, f2, f3])
def get_fidelity_space(self):
return self.fidelity_space
def test_add_hyperparameters_with_equal_names(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
self.assertRaisesRegex(
ValueError, "Hyperparameter 'name' is already in the "
"configuration space.", cs.add_hyperparameter, hp)
def int_quniform(label: str, low: int, high: int, q: int = None, default=None):
if not q:
q = math.gcd(low, high)
kwargs = {}
if default:
kwargs.update({'default_value': default})
return UniformIntegerHyperparameter(label, low, high, q=q, **kwargs)
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(
CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab')
)
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space.",
):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(
ForbiddenValueError,
"Given vector violates forbidden clause Forbidden: x3 == 2",
):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def get_config_space_from_dict(space_dict: dict):
cs = ConfigurationSpace()
params_dict = space_dict['parameters']
for key in params_dict:
param_dict = params_dict[key]
param_type = param_dict['type']
if param_type in ['float', 'int']:
bound = param_dict['bound']
optional_args = dict()
if 'default' in param_dict:
optional_args['default_value'] = param_dict['default']
elif 'log' in param_dict:
optional_args['log'] = parse_bool(param_dict['log'])
elif 'q' in param_dict:
optional_args['q'] = param_dict['q']
if param_type == 'float':
param = UniformFloatHyperparameter(key, bound[0], bound[1], **optional_args)
else:
param = UniformIntegerHyperparameter(key, bound[0], bound[1], **optional_args)
elif param_type == 'cat':
choices = param_dict['choice']
optional_args = dict()
if 'default' in param_dict:
optional_args['default_value'] = param_dict['default']
param = CategoricalHyperparameter(key, choices, **optional_args)
else:
raise ValueError("Parameter type %s not supported!" % param_type)
cs.add_hyperparameter(param)
return cs
def test_acts_as_mapping(self):
"""
Test that ConfigurationSpace can act as a mapping with iteration,
indexing and items, values, keys.
"""
cs = ConfigurationSpace()
names = [f"name{i}" for i in range(5)]
hyperparameters = [
UniformIntegerHyperparameter(name, 0, 10) for name in names
]
cs.add_hyperparameters(hyperparameters)
# Test indexing
assert cs['name3'] == hyperparameters[3]
# Test dict methods
assert list(cs.keys()) == names
assert list(cs.values()) == hyperparameters
assert list(cs.items()) == list(zip(names, hyperparameters))
assert len(cs) == 5
# Test __iter__
assert list(iter(cs)) == names
# Test unpacking
d = {**cs}
assert list(d.keys()) == names
assert list(d.values()) == hyperparameters
assert list(d.items()) == list(zip(names, hyperparameters))
assert len(d) == 5
def int_qloguniform(label: str, low: int, high: int, q: int = None, default=None):
if not q:
q = min(low, 1)
kwargs = {'log': True}
if default:
kwargs.update({'default_value': default})
return UniformIntegerHyperparameter(label, low, high, q=q, **kwargs)
def get_hyperparameter_search_space():
cs = ConfigurationSpace()
BaseImageClassificationModel.set_training_space(cs)
BaseImageClassificationModel.set_optimizer_space(cs)
xception_middle_blocks = UniformIntegerHyperparameter(
'xception_middle_blocks', 6, 10, default_value=8)
cs.add_hyperparameter(xception_middle_blocks)
return cs
def test_multi_sample_quantized_uihp(self):
# This unit test covers a problem with sampling multiple entries at a time from a configuration space with at
# least one UniformIntegerHyperparameter which is quantized.
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformIntegerHyperparameter("uihp", lower=1, upper=100, q=2, log=False))
self.assertIsNotNone(cs.sample_configuration(size=1))
self.assertEqual(10, len(cs.sample_configuration(size=10)))
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def _get_configuration_space(self) -> ConfigurationSpace:
"""Get the configuration space for the random forest.
Returns
-------
ConfigurationSpace
"""
cfg = ConfigurationSpace()
cfg.seed(int(self.rs.randint(0, 1000)))
num_trees = Constant("num_trees", value=N_TREES)
bootstrap = CategoricalHyperparameter(
"do_bootstrapping", choices=(self.bootstrap,), default_value=self.bootstrap,
)
max_feats = CategoricalHyperparameter("max_features", choices=(3 / 6, 4 / 6, 5 / 6, 1), default_value=1)
min_split = UniformIntegerHyperparameter("min_samples_to_split", lower=1, upper=10, default_value=2)
min_leavs = UniformIntegerHyperparameter("min_samples_in_leaf", lower=1, upper=10, default_value=1)
cfg.add_hyperparameters([num_trees, bootstrap, max_feats, min_split, min_leavs])
return cfg
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def test_add_configuration_space_conjunctions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_configuration_space(prefix='test', configuration_space=cs1)
self.assertEqual(str(cs2).count('test:'), 10)
# Check that they're equal except for the "test:" prefix
self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def test_estimate_size(self):
cs = ConfigurationSpace()
self.assertEqual(cs.estimate_size(), 0)
cs.add_hyperparameter(Constant('constant', 0))
self.assertEqual(cs.estimate_size(), 1)
cs.add_hyperparameter(UniformIntegerHyperparameter('integer', 0, 5))
self.assertEqual(cs.estimate_size(), 6)
cs.add_hyperparameter(CategoricalHyperparameter('cat', [0, 1, 2]))
self.assertEqual(cs.estimate_size(), 18)
cs.add_hyperparameter(UniformFloatHyperparameter('float', 0, 1))
self.assertTrue(np.isinf(cs.estimate_size()))
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegex(ValueError, r"Hyperparameter configuration "
r"contains a cycle \[\['child', 'parent'\]\]",
cs.add_condition, cond2)
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparamforbidden_clauseseters(self):
cs = ConfigurationSpace()
self.assertEqual(0, len(cs.get_hyperparameters()))
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
self.assertEqual([hp1], cs.get_hyperparameters())
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 1)
cs.add_condition(cond1)
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
# TODO: I need more tests for the topological sort!
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def test_random_neighborhood_int(self):
hp = UniformIntegerHyperparameter('a', 1, 10)
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor['a'] for neighbor in all_neighbors]
self.assertAlmostEqual(5.8125, np.mean(all_neighbors), places=2)
self.assertAlmostEqual(5.60234375, np.var(all_neighbors), places=2)
hp = UniformIntegerHyperparameter('a', 1, 10, log=True)
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor['a'] for neighbor in all_neighbors]
# Default value is 3.16
self.assertAlmostEqual(3.9425, np.mean(all_neighbors), places=2)
self.assertAlmostEqual(5.91, np.var(all_neighbors), places=2)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp)
for val in range(1, 11):
config = Configuration(cs, values={'a': val})
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, 1)
neighbors = [neighbor['a'] for neighbor in neighborhood]
self.assertEqual(len(neighbors), len(np.unique(neighbors)))
self.assertNotIn(val, neighbors)
def get_configspace(self, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, default_value=500, q=50)
num_leaves = UniformIntegerHyperparameter("num_leaves", 31, 2047, default_value=128)
max_depth = Constant('max_depth', 15)
learning_rate = UniformFloatHyperparameter("learning_rate", 1e-3, 0.3, default_value=0.1, log=True)
min_child_samples = UniformIntegerHyperparameter("min_child_samples", 5, 30, default_value=20)
subsample = UniformFloatHyperparameter("subsample", 0.7, 1, default_value=1, q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.7, 1, default_value=1, q=0.1)
cs.add_hyperparameters([n_estimators, num_leaves, max_depth, learning_rate, min_child_samples, subsample,
colsample_bytree])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': (hp.randint('lgb_n_estimators', 19) + 2) * 50,
'num_leaves': hp.randint('lgb_num_leaves', 2017) + 31,
'max_depth': 15,
'learning_rate': hp.loguniform('lgb_learning_rate', np.log(1e-3), np.log(0.3)),
'min_child_samples': hp.randint('lgb_min_child_samples', 26) + 5,
'subsample': (hp.randint('lgb_subsample', 4) + 7) * 0.1,
'colsample_bytree': (hp.randint('lgb_colsample_bytree', 4) + 7) * 0.1,
}
return space
elif optimizer == 'gpflowopt':
from gpflowopt.domain import ContinuousParameter
domain = (
ContinuousParameter('n_estimators', 100, 1000) +
ContinuousParameter('num_leaves', 31, 2047) +
ContinuousParameter('max_depth', 15, 16) +
ContinuousParameter("learning_rate", 1e-3, 0.3) +
ContinuousParameter("min_child_samples", 5, 30) +
ContinuousParameter("subsample", 0.7, 1) +
ContinuousParameter("colsample_bytree", 0.7, 1)
)
return domain
else:
raise ValueError('Unknown optimizer %s when getting configspace' % optimizer)
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def test_with_ordinal(self):
cs = smac.configspace.ConfigurationSpace()
_ = cs.add_hyperparameter(
CategoricalHyperparameter('a', [0, 1], default_value=0))
_ = cs.add_hyperparameter(
OrdinalHyperparameter('b', [0, 1], default_value=1))
_ = cs.add_hyperparameter(
UniformFloatHyperparameter('c',
lower=0.,
upper=1.,
default_value=1))
_ = cs.add_hyperparameter(
UniformIntegerHyperparameter('d',
lower=0,
upper=10,
default_value=1))
cs.seed(1)
feat_array = np.array([0, 0, 0]).reshape(1, -1)
types, bounds = get_types(cs, feat_array)
model = RandomForestWithInstances(
configspace=cs,
types=types,
bounds=bounds,
instance_features=feat_array,
seed=1,
ratio_features=1.0,
pca_components=9,
)
self.assertEqual(bounds[0][0], 2)
self.assertTrue(bounds[0][1] is np.nan)
self.assertEqual(bounds[1][0], 0)
self.assertEqual(bounds[1][1], 1)
self.assertEqual(bounds[2][0], 0.)
self.assertEqual(bounds[2][1], 1.)
self.assertEqual(bounds[3][0], 0.)
self.assertEqual(bounds[3][1], 1.)
X = np.array(
[[0., 0., 0., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0.],
[0., 1., 0., 9., 0., 0., 0.], [0., 1., 1., 4., 0., 0., 0.]],
dtype=np.float64)
y = np.array([0, 1, 2, 3], dtype=np.float64)
X_train = np.vstack((X, X, X, X, X, X, X, X, X, X))
y_train = np.vstack((y, y, y, y, y, y, y, y, y, y))
model.train(X_train, y_train.reshape((-1, 1)))
mean, _ = model.predict(X)
for idx, m in enumerate(mean):
self.assertAlmostEqual(y[idx], m, 0.05)
def test_add_conditions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs2.add_hyperparameter(hp1)
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
cs2.add_hyperparameter(hp2)
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
cs2.add_hyperparameter(hp3)
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cs2.add_hyperparameter(hp4)
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_condition(cond1)
cs2.add_condition(andCond)
self.assertEqual(str(cs1), str(cs2))
def string2hyperparameter(hp_desc: str):
# Only support type, range, default_value, log, q
# Sample: x2, Type: UniformInteger, Range: [1, 15], Default: 4, on log-scale, Q: 2
q = -1
log = None
default_value = None
params = hp_desc.split(',')
cur_idx = -1
while default_value is None:
if q == -1:
if 'Q:' in params[cur_idx]:
q = float(params[cur_idx][4:])
cur_idx -= 1
continue
else:
q = None
if log is None:
if 'log-scale' in params[cur_idx]:
log = True
cur_idx -= 1
continue
else:
log = False
if default_value is None:
default_value = str(params[cur_idx][10:])
cur_idx -= 1
prefix_params = ','.join(params[:cur_idx + 1])
range_str = prefix_params.split(':')[-1]
if range_str[-1] == ']':
element_list = range_str[2:-1].split(',')
range = [float(element_list[0]), float(element_list[1])]
else:
element_list = range_str[1:-1].split(',')
range = [element[1:] for element in element_list]
type_str = prefix_params.split(':')[-2].split(',')[0][1:]
name_str = ':'.join(prefix_params.split(':')[:-2])
name = ','.join(name_str.split(',')[:-1])[4:]
if type_str == 'UniformFloat':
return UniformFloatHyperparameter(name, range[0], range[1], default_value=float(default_value), log=log, q=q)
elif type_str == 'UniformInteger':
return UniformIntegerHyperparameter(name, range[0], range[1], default_value=int(default_value), log=log, q=q)
elif type_str == 'Categorical':
return CategoricalHyperparameter(name, range, default_value=default_value)
else:
raise ValueError('Hyperparameter type %s not supported!' % type)
def test_remove_inactive_parameter():
configuration_space = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("hp1", choices=[0, 1])
hp2 = CategoricalHyperparameter("hp2", choices=['a'])
hp3 = UniformIntegerHyperparameter("hp3",
lower=0,
upper=5,
default_value=5)
configuration_space.add_hyperparameters([hp1, hp2, hp3])
# If hp1 = 0, then don't allow hp2
not_condition = NotEqualsCondition(hp2, hp1, 0)
configuration_space.add_condition(not_condition)
allowed_cfg = Configuration(configuration_space, {
'hp1': 1,
'hp2': 'a',
'hp3': 5
})
not_allowed = {'hp1': 0, 'hp2': 'a', 'hp3': 5}
with pytest.raises(ValueError):
Configuration(configuration_space, not_allowed)
# No inactive hp - case: config is CS.configuration
transformed = AbstractBenchmark._check_and_cast_configuration(
allowed_cfg, configuration_space)
assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5}
# No inactive hp - case: config is dict
transformed = AbstractBenchmark._check_and_cast_configuration(
allowed_cfg.get_dictionary(), configuration_space)
assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5}
# Remove inactive: - case: config is CS.configuration
not_allowed_cs = Configuration(configuration_space, {
'hp1': 0,
'hp2': 'a',
'hp3': 5
},
allow_inactive_with_values=True)
transformed = AbstractBenchmark._check_and_cast_configuration(
not_allowed_cs, configuration_space)
assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
# Remove inactive: - case: config is dict
transformed = AbstractBenchmark._check_and_cast_configuration(
not_allowed, configuration_space)
assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
def test_meta_field(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp",
lower=1,
upper=10,
meta=dict(uihp=True)))
cs.add_hyperparameter(
NormalIntegerHyperparameter("nihp",
mu=0,
sigma=1,
meta=dict(nihp=True)))
cs.add_hyperparameter(
UniformFloatHyperparameter("ufhp",
lower=1,
upper=10,
meta=dict(ufhp=True)))
cs.add_hyperparameter(
NormalFloatHyperparameter("nfhp",
mu=0,
sigma=1,
meta=dict(nfhp=True)))
cs.add_hyperparameter(
CategoricalHyperparameter("chp",
choices=['1', '2', '3'],
meta=dict(chp=True)))
cs.add_hyperparameter(
OrdinalHyperparameter("ohp",
sequence=['1', '2', '3'],
meta=dict(ohp=True)))
cs.add_hyperparameter(Constant("const", value=1,
meta=dict(const=True)))
parent = ConfigurationSpace()
parent.add_configuration_space("sub", cs, delimiter=':')
self.assertEqual(
parent.get_hyperparameter("sub:uihp").meta, dict(uihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nihp").meta, dict(nihp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ufhp").meta, dict(ufhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:nfhp").meta, dict(nfhp=True))
self.assertEqual(
parent.get_hyperparameter("sub:chp").meta, dict(chp=True))
self.assertEqual(
parent.get_hyperparameter("sub:ohp").meta, dict(ohp=True))
self.assertEqual(
parent.get_hyperparameter("sub:const").meta, dict(const=True))
def branin(n_eval):
evaluator = Branin()
name_tag = '_'.join(
['branin', datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f")])
cs = ConfigurationSpace()
for i in range(len(evaluator.n_vertices)):
car_var = UniformIntegerHyperparameter('x' + str(i + 1).zfill(2),
0,
int(evaluator.n_vertices[i]) -
1,
default_value=25)
cs.add_hyperparameter(car_var)
init_points_numpy = evaluator.suggested_init.long().numpy()
init_points = []
for i in range(init_points_numpy.shape[0]):
init_points.append(
Configuration(
cs, {
'x' + str(j + 1).zfill(2): int(init_points_numpy[i][j])
for j in range(len(evaluator.n_vertices))
}))
def evaluate(x):
x_tensor = torch.LongTensor([
int(x['x' + str(j + 1).zfill(2)])
for j in range(len(evaluator.n_vertices))
])
return evaluator.evaluate(x_tensor).item()
print('Began at ' + datetime.now().strftime("%H:%M:%S"))
scenario = Scenario({
"run_obj": "quality",
"runcount-limit": n_eval,
"cs": cs,
"deterministic": "true",
'output_dir': os.path.join(EXP_DIR, name_tag)
})
smac = SMAC(scenario=scenario,
tae_runner=evaluate,
initial_configurations=init_points)
smac.optimize()
evaluations, optimum = evaluations_from_smac(smac)
print('Finished at ' + datetime.now().strftime("%H:%M:%S"))
return optimum
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5))
pcs.add_hyperparameter(UniformFloatHyperparameter('x1', 0.5, 2.55))
pcs.add_hyperparameter(
CategoricalHyperparameter('x2', ['ab', 'bc', 'cd', 'de']))
conf = pcs.sample_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError,
'Illegal value 0 for hyperparameter x1'):
conf['x1'] = 0
with self.assertRaisesRegex(ValueError,
'Illegal value 2.5 for hyperparameter x0'):
conf['x0'] = 2.5
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space."
):
conf['x_0'] = 1
# successful operation 1
x1_old = conf['x1']
if x1_old == 1.5:
conf['x1'] = 2.1
else:
conf['x1'] = 1.5
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
# successful operation 2
x2_old = conf['x2']
if x2_old == 'ab':
conf['x2'] = 'cd'
else:
conf['x2'] = 'ab'
x2_new = conf['x2']
self.assertNotEqual(x2_old, x2_new)
