def test_uniformfloat_transform(self):
"""This checks whether a value sampled through the configuration
space (it does not happend when the variable is sampled alone) stays
equal when it is serialized via JSON and the deserialized again."""
cs = ConfigurationSpace()
a = cs.add_hyperparameter(UniformFloatHyperparameter('a', -5, 10))
b = cs.add_hyperparameter(NormalFloatHyperparameter('b', 1, 2,
log=True))
for i in range(100):
config = cs.sample_configuration()
value = OrderedDict(sorted(config.get_dictionary().items()))
string = json.dumps(value)
saved_value = json.loads(string)
saved_value = OrderedDict(sorted(byteify(saved_value).items()))
self.assertEqual(repr(value), repr(saved_value))
# Next, test whether the truncation also works when initializing the
# Configuration with a dictionary
for i in range(100):
rs = np.random.RandomState(1)
value_a = a.sample(rs)
value_b = b.sample(rs)
values_dict = {'a': value_a, 'b': value_b}
config = Configuration(cs, values=values_dict)
string = json.dumps(config.get_dictionary())
saved_value = json.loads(string)
saved_value = byteify(saved_value)
self.assertEqual(values_dict, saved_value)
def test_add_hyperparameters_with_equal_names(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
self.assertRaisesRegex(ValueError,
"Hyperparameter 'name' is already in the "
"configuration space.",
cs.add_hyperparameter, hp)
def setUp(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("parent", [0, 1]))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
hp3 = cs.add_hyperparameter(
UniformIntegerHyperparameter("friend", 0, 5))
self.cs = cs
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def _get_configuration_space(
search_space: Dict[str, BaseDistribution]) -> ConfigurationSpace:
config_space = ConfigurationSpace()
for name, distribution in search_space.items():
config_space.add_hyperparameter(
_distribution_to_hyperparameter(name, distribution))
return config_space
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def get_config_space():
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformIntegerHyperparameter(name='a',
lower=0,
upper=100))
cs.add_hyperparameter(UniformIntegerHyperparameter(name='b',
lower=0,
upper=100))
return cs
def testRandomImputation(self):
rs = numpy.random.RandomState(1)
for i in range(0, 150, 15):
# First random imputation sanity check
num_samples = max(1, i * 10)
num_feat = max(1, i)
num_censored = int(num_samples * 0.1)
X = rs.rand(num_samples, num_feat)
y = numpy.sin(X[:, 0:1])
cutoff = max(y) * 0.9
y[y > cutoff] = cutoff
# We have some cen data
cen_X = X[:num_censored, :]
cen_y = y[:num_censored]
uncen_X = X[num_censored:, :]
uncen_y = y[num_censored:]
cen_y /= 2
cs = ConfigurationSpace()
for i in range(num_feat):
cs.add_hyperparameter(
UniformFloatHyperparameter(name="a_%d" % i,
lower=0,
upper=1,
default_value=0.5))
types, bounds = get_types(cs, None)
self.model = RandomForestWithInstances(
configspace=cs,
types=types,
bounds=bounds,
instance_features=None,
seed=1234567980,
)
imputor = rfr_imputator.RFRImputator(rng=rs,
cutoff=cutoff,
threshold=cutoff * 10,
change_threshold=0.01,
max_iter=5,
model=self.model)
imp_y = imputor.impute(censored_X=cen_X,
censored_y=cen_y,
uncensored_X=uncen_X,
uncensored_y=uncen_y)
if imp_y is None:
continue
for idx in range(cen_y.shape[0]):
self.assertGreater(imp_y[idx], cen_y[idx])
self.assertTrue(numpy.isfinite(imp_y).all())
def test_multi_sample_quantized_uihp(self):
# This unit test covers a problem with sampling multiple entries at a time from a
# configuration space with at least one UniformIntegerHyperparameter which is quantized.
cs = ConfigurationSpace()
cs.add_hyperparameter(
UniformIntegerHyperparameter("uihp", lower=1, upper=101, q=2, log=False)
)
self.assertIsNotNone(cs.sample_configuration(size=1))
self.assertEqual(10, len(cs.sample_configuration(size=10)))
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegex(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def test_check_forbidden_with_sampled_vector_configuration(self):
cs = ConfigurationSpace()
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
cs.add_hyperparameter(metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, vector=np.ones(1, dtype=float))
self.assertRaisesRegex(ValueError, "violates forbidden clause",
cs._check_forbidden, configuration.get_array())
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def get_configspace():
""" Returns the configuration space for the network to be configured in the example. """
# BOHB does not support Ordinal -> convert to Cat
cs_invalid = get_complete_configspace()
cs = ConfigurationSpace()
for hp in cs_invalid.get_hyperparameters():
if isinstance(hp, OrdinalHyperparameter):
cs.add_hyperparameter(CategoricalHyperparameter(hp.name, choices=hp.sequence))
else:
cs.add_hyperparameter(hp)
return cs
def _test_random_neigbor(self, hp):
cs = ConfigurationSpace()
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
cs.seed(1)
config = cs.sample_configuration()
for i in range(100):
new_config = get_random_neighbor(config, i)
self.assertNotEqual(config, new_config)
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def test_illegal_default_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("loss", ["l1", "l2"], default='l1')
hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"], default='l1')
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
forb1 = ForbiddenEqualsClause(hp1, "l1")
forb2 = ForbiddenEqualsClause(hp2, "l1")
forb3 = ForbiddenAndConjunction(forb1, forb2)
# cs.add_forbidden_clause(forb3)
self.assertRaisesRegexp(ValueError, "Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && "
"Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_illegal_default_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("loss", ["l1", "l2"], default_value='l1')
hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value='l1')
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
forb1 = ForbiddenEqualsClause(hp1, "l1")
forb2 = ForbiddenEqualsClause(hp2, "l1")
forb3 = ForbiddenAndConjunction(forb1, forb2)
# cs.add_forbidden_clause(forb3)
self.assertRaisesRegexp(ValueError, "Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && "
"Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegex(ValueError, r"Hyperparameter configuration "
r"contains a cycle \[\['child', 'parent'\]\]",
cs.add_condition, cond2)
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegex(ValueError, r"Hyperparameter configuration "
r"contains a cycle \[\['child', 'parent'\]\]",
cs.add_condition, cond2)
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(KeyError, "Hyperparameter 'x_0' does not exist in this configuration space."):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_get_hyperparameters_topological_sort(self):
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
# More top-level hyperparameters
hp7 = CategoricalHyperparameter("input7", [0, 1])
# Somewhat shuffled
hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7]
for hp in hyperparameters:
cs.add_hyperparameter(hp)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
hps = cs.get_hyperparameters()
# AND is moved to the front because of alphabetical sorting
for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(cond5)
hps = cs.get_hyperparameters()
for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(conj3)
hps = cs.get_hyperparameters()
# print(hps, hyperparameters)
for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]):
# print(hp, idx)
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(KeyError, "Hyperparameter 'x_0' does not exist in this configuration space."):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_get_hyperparameters_topological_sort(self):
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
# More top-level hyperparameters
hp7 = CategoricalHyperparameter("input7", [0, 1])
# Somewhat shuffled
hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7]
for hp in hyperparameters:
cs.add_hyperparameter(hp)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
hps = cs.get_hyperparameters()
# AND is moved to the front because of alphabetical sorting
for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(cond5)
hps = cs.get_hyperparameters()
for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(conj3)
hps = cs.get_hyperparameters()
# print(hps, hyperparameters)
for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]):
# print(hp, idx)
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_merge_foreign_data(self):
''' test smac.utils.merge_foreign_data '''
scenario = Scenario(self.test_scenario_dict)
scenario_2 = Scenario(self.test_scenario_dict)
scenario_2.feature_dict = {"inst_new": [4]}
# init cs
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformIntegerHyperparameter(name='a',
lower=0,
upper=100))
cs.add_hyperparameter(UniformIntegerHyperparameter(name='b',
lower=0,
upper=100))
# build runhistory
rh_merge = RunHistory(aggregate_func=average_cost)
config = Configuration(cs, values={'a': 1, 'b': 2})
rh_merge.add(config=config, instance_id="inst_new", cost=10, time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
# "d" is an instance in <scenario>
rh_merge.add(config=config, instance_id="d", cost=5, time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
# build empty rh
rh_base = RunHistory(aggregate_func=average_cost)
merge_foreign_data(scenario=scenario, runhistory=rh_base,
in_scenario_list=[scenario_2], in_runhistory_list=[rh_merge])
# both runs should be in the runhistory
# but we should not use the data to update the cost of config
self.assertTrue(len(rh_base.data) == 2)
self.assertTrue(np.isnan(rh_base.get_cost(config)))
# we should not get direct access to external run data
runs = rh_base.get_runs_for_config(config)
self.assertTrue(len(runs) == 0)
rh_merge.add(config=config, instance_id="inst_new_2", cost=10, time=20,
status=StatusType.SUCCESS,
seed=None,
additional_info=None)
self.assertRaises(ValueError, merge_foreign_data, **{
"scenario": scenario, "runhistory": rh_base, "in_scenario_list": [scenario_2], "in_runhistory_list": [rh_merge]})
def test_get_hyperparamforbidden_clauseseters(self):
cs = ConfigurationSpace()
self.assertEqual(0, len(cs.get_hyperparameters()))
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
self.assertEqual([hp1], cs.get_hyperparameters())
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 1)
cs.add_condition(cond1)
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
# TODO: I need more tests for the topological sort!
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def test_get_hyperparamforbidden_clauseseters(self):
cs = ConfigurationSpace()
self.assertEqual(0, len(cs.get_hyperparameters()))
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
self.assertEqual([hp1], cs.get_hyperparameters())
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 1)
cs.add_condition(cond1)
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
# TODO: I need more tests for the topological sort!
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def sacred_space_to_configspace(space):
"""
Convert a Labwatch searchspace to a ConfigSpace.
Parameters
----------
space: labwatch.searchspace.SearchSpace
A labwatch searchspace to be converted.
Returns
-------
ConfigSpace.ConfigurationSpace:
A ConfigurationSpace equivalent to the given SeachSpace.
"""
# first convert all non conditionals
non_conditions = {}
conditions = []
for name in space.non_conditions:
param = space.parameters[name]
converted_param = convert_simple_param(name, param)
non_conditions[name] = converted_param
for name in space.conditions:
param = space.parameters[name]
converted_result = convert_simple_param(name, param["result"])
# next build the condition as required by the ConfigSpace
condition = param["condition"]
condition_name = space.uids_to_names[condition["uid"]]
if condition_name not in non_conditions:
raise ValueError("Unknown parameter in Condition")
converted_condition = non_conditions[condition_name]
converted_choices = []
for choice in condition["choices"]:
if isinstance(choice, dict):
if choice["_class"] != "Constant":
raise ValueError("Invalid choice encountered in Condition")
converted_choices.append(choice["value"])
else:
converted_choices.append(choice)
cond = InCondition(converted_result,
converted_condition,
values=converted_choices)
non_conditions[name] = converted_result
conditions.append(cond)
# finally build the ConfigSpace
cs = ConfigurationSpace(seed=np.random.seed())
for _name, param in non_conditions.items():
cs.add_hyperparameter(param)
for cond in conditions:
cs.add_condition(cond)
return cs
def test_add_forbidden_clause(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
forb = ForbiddenEqualsClause(hp1, 1)
# TODO add checking whether a forbidden clause makes sense at all
cs.add_forbidden_clause(forb)
# TODO add something to properly retrieve the forbidden clauses
self.assertEqual(str(cs), "Configuration space object:\n "
"Hyperparameters:\n input1, "
"Type: Categorical, Choices: {0, 1}, "
"Default: 0\n"
" Forbidden Clauses:\n"
" Forbidden: input1 == 1\n")
def test_add_forbidden_clause(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
forb = ForbiddenEqualsClause(hp1, 1)
# TODO add checking whether a forbidden clause makes sense at all
cs.add_forbidden_clause(forb)
# TODO add something to properly retrieve the forbidden clauses
self.assertEqual(str(cs), "Configuration space object:\n "
"Hyperparameters:\n input1, "
"Type: Categorical, Choices: {0, 1}, "
"Default: 0\n"
" Forbidden Clauses:\n"
" Forbidden: input1 == 1\n")
def get_cash_cs(include_algorithms=None, task_type=CLASSIFICATION):
_candidates = get_combined_candidtates(_classifiers, _addons)
if include_algorithms is not None:
_candidates = set(include_algorithms).intersection(set(_candidates.keys()))
if len(_candidates) == 0:
raise ValueError("No algorithms included! Please check the spelling of the included algorithms!")
cs = ConfigurationSpace()
algo = CategoricalHyperparameter('algorithm', list(_candidates))
cs.add_hyperparameter(algo)
for estimator_id in _candidates:
estimator_cs = get_hpo_cs(estimator_id)
parent_hyperparameter = {'parent': algo,
'value': estimator_id}
cs.add_configuration_space(estimator_id, estimator_cs, parent_hyperparameter=parent_hyperparameter)
return cs
def create_cs_from_pandaframe(self, data):
# TODO use from pyimp after https://github.com/automl/ParameterImportance/issues/72 is implemented
warnings.warn(
"No parameter configuration space (pcs) provided! "
"Interpreting all parameters as floats. This might lead "
"to suboptimal analysis.", RuntimeWarning)
self.logger.debug("Interpreting as parameters: %s", data.columns)
minima = data.min() # to define ranges of hyperparameter
maxima = data.max()
cs = ConfigurationSpace(seed=42)
for p in data.columns:
cs.add_hyperparameter(
UniformFloatHyperparameter(p,
lower=minima[p] - 1,
upper=maxima[p] + 1))
def test_get_one_exchange_neighbourhood(self):
# test fixed_dims
cs = ConfigurationSpace()
cs.add_hyperparameter(
CategoricalHyperparameter('0', [0, 1], default_value=0))
cs.add_hyperparameter(
CategoricalHyperparameter('1', [0, 1], default_value=0))
cs.add_hyperparameter(
CategoricalHyperparameter('2', [0, 1], default_value=0))
cs.add_hyperparameter(
CategoricalHyperparameter('3', [0, 1], default_value=0))
cs.add_hyperparameter(
CategoricalHyperparameter('4', [0, 1, 2, 3, 4], default_value=0))
fixed_dims = {'0': 1, '1': 0, '2': 0, '3': 0}
conf = cs.sample_configuration(fixed_dims=fixed_dims)
neighborhood_iter = get_one_exchange_neighbourhood(
conf, seed=0, fixed_dims=fixed_dims)
tmp = next(neighborhood_iter)
self.assertEqual(tmp['0'], fixed_dims['0'])
self.assertEqual(tmp['1'], fixed_dims['1'])
self.assertEqual(tmp['2'], fixed_dims['2'])
self.assertEqual(tmp['3'], fixed_dims['3'])
tmp = next(neighborhood_iter)
self.assertEqual(tmp['0'], fixed_dims['0'])
self.assertEqual(tmp['1'], fixed_dims['1'])
self.assertEqual(tmp['2'], fixed_dims['2'])
self.assertEqual(tmp['3'], fixed_dims['3'])
tmp = next(neighborhood_iter)
self.assertEqual(tmp['0'], fixed_dims['0'])
self.assertEqual(tmp['1'], fixed_dims['1'])
self.assertEqual(tmp['2'], fixed_dims['2'])
self.assertEqual(tmp['3'], fixed_dims['3'])
tmp = next(neighborhood_iter)
self.assertEqual(tmp['0'], fixed_dims['0'])
self.assertEqual(tmp['1'], fixed_dims['1'])
self.assertEqual(tmp['2'], fixed_dims['2'])
self.assertEqual(tmp['3'], fixed_dims['3'])
# StopIteration
with self.assertRaises(StopIteration):
tmp = next(neighborhood_iter)
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_estimate_size(self):
cs = ConfigurationSpace()
self.assertEqual(cs.estimate_size(), 0)
cs.add_hyperparameter(Constant('constant', 0))
self.assertEqual(cs.estimate_size(), 1)
cs.add_hyperparameter(UniformIntegerHyperparameter('integer', 0, 5))
self.assertEqual(cs.estimate_size(), 6)
cs.add_hyperparameter(CategoricalHyperparameter('cat', [0, 1, 2]))
self.assertEqual(cs.estimate_size(), 18)
cs.add_hyperparameter(UniformFloatHyperparameter('float', 0, 1))
self.assertTrue(np.isinf(cs.estimate_size()))
class TestLHDesign(unittest.TestCase):
def setUp(self):
def get_uniform_param(name: str):
return UniformFloatHyperparameter(name, 0, 1)
def get_constant_param(name: str):
return Constant(name, 0.)
def get_categorical_param(name: str):
return CategoricalHyperparameter(name, choices=["a", "b", "c"])
def get_ordinal_param(name: str):
return OrdinalHyperparameter(name, [8, 6, 4, 2])
get_params = [
get_uniform_param,
get_constant_param,
get_categorical_param,
get_ordinal_param
]
self.cs = ConfigurationSpace()
for j, get_param in enumerate(get_params):
param_name = f"x{j}"
self.cs.add_hyperparameter(get_param(param_name))
for i in range(5):
self.cs.add_hyperparameter(UniformFloatHyperparameter('x%d' % (i + len(get_params)), 0, 1))
def test_latin_hypercube_design(self):
kwargs = dict(
rng=np.random.RandomState(1),
traj_logger=unittest.mock.Mock(),
ta_run_limit=1000,
configs=None,
n_configs_x_params=None,
max_config_fracs=0.25,
init_budget=1,
)
LHDesign(
cs=self.cs,
**kwargs
).select_configurations()
def test_condition_without_added_hyperparameters(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
self.assertRaisesRegex(ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition,
cond)
cs.add_hyperparameter(hp1)
self.assertRaisesRegex(ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition,
cond)
# Test also the parent hyperparameter
cs2 = ConfigurationSpace()
cs2.add_hyperparameter(hp2)
self.assertRaisesRegex(ValueError, "Parent hyperparameter 'parent' "
"not in configuration space.",
cs2.add_condition, cond)
def evaluate(_config):
_config = _config.get_dictionary()
# print(_config)
arm = None
cs = ConfigurationSpace()
for key in _config:
key_str = key.split(":")
if key_str[0] == 'classifier':
if key_str[1] == '__choice__':
arm = _config[key]
cs.add_hyperparameter(
UnParametrizedHyperparameter("estimator",
_config[key]))
else:
cs.add_hyperparameter(
UnParametrizedHyperparameter(key_str[2], _config[key]))
if arm in first_bandit.arms:
transformed_node = apply_metalearning_fe(
first_bandit.sub_bandits[arm].optimizer['fe'], _config)
default_config = cs.sample_configuration(1)
hpo_evaluator = ClassificationEvaluator(
None,
data_node=transformed_node,
name='hpo',
resampling_strategy=first_bandit.eval_type,
seed=first_bandit.seed)
start_time = time.time()
score1 = -hpo_evaluator(default_config)
time_cost1 = time.time() - start_time
# Evaluate the default config
start_time = time.time()
score2 = -hpo_evaluator(
first_bandit.sub_bandits[arm].default_config)
time_cost2 = time.time() - start_time
transformed_node.score2 = max(score1, score2)
return (arm, score1, default_config, transformed_node,
time_cost1), (arm, score2,
first_bandit.sub_bandits[arm].default_config,
transformed_node, time_cost2)
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegex(
ValueError, r"Adding a second condition \(different\) for a "
r"hyperparameter is ambigouos and "
r"therefore forbidden. Add a conjunction "
r"instead!", cs.add_condition, cond2)
def get_configspace(self, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
for i in range(self.dim):
xi = UniformFloatHyperparameter("x%d" % i, self.lb, self.ub)
cs.add_hyperparameter(xi)
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'x%d' % i: hp.uniform('hp_x%d' % i, self.lb, self.ub) for i in range(self.dim)}
return space
elif optimizer == 'gpflowopt':
import gpflowopt
domain = np.sum([
gpflowopt.domain.ContinuousParameter('x%d' % i, self.lb, self.ub) for i in range(self.dim)
])
return domain
else:
raise ValueError('Unknown optimizer %s when getting configspace' % optimizer)
def test_condition_without_added_hyperparameters(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
self.assertRaisesRegex(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
cs.add_hyperparameter(hp1)
self.assertRaisesRegex(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
# Test also the parent hyperparameter
cs2 = ConfigurationSpace()
cs2.add_hyperparameter(hp2)
self.assertRaisesRegex(
ValueError, "Parent hyperparameter 'parent' "
"not in configuration space.", cs2.add_condition, cond)
def test_choose_next(self):
configspace = ConfigurationSpace()
configspace.add_hyperparameter(UniformFloatHyperparameter('a', 0, 1))
configspace.add_hyperparameter(UniformFloatHyperparameter('b', 0, 1))
dataset_name = 'foo'
func_eval_time_limit = 15
total_walltime_limit = 15
memory_limit = 3072
auto = AutoMLSMBO(config_space=None,
dataset_name=dataset_name,
backend=None,
func_eval_time_limit=func_eval_time_limit,
total_walltime_limit=total_walltime_limit,
memory_limit=memory_limit,
watcher=None,
metric=accuracy)
auto.config_space = configspace
scenario = Scenario({
'cs': configspace,
'cutoff-time': func_eval_time_limit,
'wallclock-limit': total_walltime_limit,
'memory-limit': memory_limit,
'run-obj': 'quality'
})
smac = SMAC(scenario)
self.assertRaisesRegex(
ValueError, 'Cannot use SMBO algorithm on '
'empty runhistory', auto.choose_next, smac)
config = Configuration(configspace, values={'a': 0.1, 'b': 0.2})
# TODO make sure the incumbent is always set?
smac.solver.incumbent = config
runhistory = smac.solver.runhistory
runhistory.add(config=config,
cost=0.5,
time=0.5,
status=StatusType.SUCCESS)
auto.choose_next(smac)
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegex(ValueError,
r"Adding a second condition \(different\) for a "
r"hyperparameter is ambigouos and "
r"therefore forbidden. Add a conjunction "
r"instead!",
cs.add_condition, cond2)
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_repr(self):
cs1 = ConfigurationSpace()
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n",
retval)
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs1.add_hyperparameter(hp1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n" % str(hp1), retval)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n %s\n Conditions:\n %s\n" %
(str(hp2), str(hp1), str(cond1)), retval)
def test_add_configuration_space_conjunctions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_configuration_space(prefix='test', configuration_space=cs1)
self.assertEqual(str(cs2).count('test:'), 10)
# Check that they're equal except for the "test:" prefix
self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def test_check_configuration2(self):
# Test that hyperparameters which are not active must not be set and
# that evaluating forbidden clauses does not choke on missing
# hyperparameters
cs = ConfigurationSpace()
classifier = CategoricalHyperparameter("classifier",
["k_nearest_neighbors", "extra_trees"])
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2])
metric_depends_on_classifier = EqualsCondition(metric, classifier,
"k_nearest_neighbors")
p_depends_on_metric = EqualsCondition(p, metric, "minkowski")
cs.add_hyperparameter(metric)
cs.add_hyperparameter(p)
cs.add_hyperparameter(classifier)
cs.add_condition(metric_depends_on_classifier)
cs.add_condition(p_depends_on_metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, dict(classifier="extra_trees"))
# check backward compatibility with checking configurations instead of vectors
cs.check_configuration(configuration)
def _test_get_one_exchange_neighbourhood(self, hp):
cs = ConfigurationSpace()
num_neighbors = 0
if not isinstance(hp, list):
hp = [hp]
for hp_ in hp:
cs.add_hyperparameter(hp_)
if np.isinf(hp_.get_num_neighbors()):
num_neighbors += 4
else:
num_neighbors += hp_.get_num_neighbors()
cs.seed(1)
config = cs.get_default_configuration()
all_neighbors = []
for i in range(100):
neighborhood = get_one_exchange_neighbourhood(config, i)
for new_config in neighborhood:
self.assertNotEqual(config, new_config)
all_neighbors.append(new_config)
return all_neighbors
def test_get_parent_and_children_of(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([hp1], cs.get_parents_of(hp2.name))
self.assertEqual([hp1], cs.get_parents_of(hp2))
self.assertEqual([hp2], cs.get_children_of(hp1.name))
self.assertEqual([hp2], cs.get_children_of(hp1))
self.assertRaisesRegex(KeyError,
"Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_parents_of,
"Foo")
self.assertRaisesRegex(KeyError,
"Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_children_of,
"Foo")
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def test_setting_illegal_value(self):
cs = ConfigurationSpace()
cs.add_hyperparameter(UniformFloatHyperparameter('x', 0, 1))
configuration = {'x': 2}
self.assertRaises(ValueError, Configuration, cs, configuration)
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_add_hyperparameter(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
def test_check_configuration(self):
# TODO this is only a smoke test
# TODO actually, this rather tests the evaluate methods in the
# conditions module!
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
cs.add_condition(conj3)
expected_outcomes = [False, False, False, False, False,
False, False, True, False, False,
False, False, False, False, False,
True, False, False, False, True,
False, False, False, True, False,
False, False, False, False, False,
False, True]
for idx, values in enumerate(product([0, 1], repeat=5)):
# The hyperparameters aren't sorted, but the test assumes them to
# be sorted.
hyperparameters = sorted(cs.get_hyperparameters(),
key=lambda t: t.name)
instantiations = {hyperparameters[jdx+1].name: values[jdx]
for jdx in range(len(values))}
evaluation = conj3.evaluate(instantiations)
self.assertEqual(expected_outcomes[idx], evaluation)
if not evaluation:
self.assertRaisesRegex(ValueError,
r"Inactive hyperparameter 'AND' must "
r"not be specified, but has the vector value: "
r"'0.0'.",
Configuration, cs, values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True"})
else:
Configuration(cs, values={"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True"})
def test_add_conditions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs2.add_hyperparameter(hp1)
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
cs2.add_hyperparameter(hp2)
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
cs2.add_hyperparameter(hp3)
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cs2.add_hyperparameter(hp4)
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_condition(cond1)
cs2.add_condition(andCond)
self.assertEqual(str(cs1), str(cs2))