def test_check_configuration2(self):
# Test that hyperparameters which are not active must not be set and
# that evaluating forbidden clauses does not choke on missing
# hyperparameters
cs = ConfigurationSpace()
classifier = CategoricalHyperparameter("classifier",
["k_nearest_neighbors", "extra_trees"])
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2])
metric_depends_on_classifier = EqualsCondition(metric, classifier,
"k_nearest_neighbors")
p_depends_on_metric = EqualsCondition(p, metric, "minkowski")
cs.add_hyperparameter(metric)
cs.add_hyperparameter(p)
cs.add_hyperparameter(classifier)
cs.add_condition(metric_depends_on_classifier)
cs.add_condition(p_depends_on_metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, dict(classifier="extra_trees"))
# check backward compatibility with checking configurations instead of vectors
cs.check_configuration(configuration)
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegex(ValueError, r"Hyperparameter configuration "
r"contains a cycle \[\['child', 'parent'\]\]",
cs.add_condition, cond2)
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(KeyError, "Hyperparameter 'x_0' does not exist in this configuration space."):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_get_hyperparameters(self):
cs = ConfigurationSpace()
self.assertEqual(0, len(cs.get_hyperparameters()))
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
self.assertEqual([hp1], cs.get_hyperparameters())
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 1)
cs.add_condition(cond1)
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
# TODO: I need more tests for the topological sort!
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def test_get_hyperparamforbidden_clauseseters(self):
cs = ConfigurationSpace()
self.assertEqual(0, len(cs.get_hyperparameters()))
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
self.assertEqual([hp1], cs.get_hyperparameters())
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 1)
cs.add_condition(cond1)
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
# TODO: I need more tests for the topological sort!
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegexp(
ValueError, "Hyperparameter configuration "
"contains a cycle \[\['child', 'parent'\]\]", cs.add_condition,
cond2)
def sacred_space_to_configspace(space):
"""
Convert a Labwatch searchspace to a ConfigSpace.
Parameters
----------
space: labwatch.searchspace.SearchSpace
A labwatch searchspace to be converted.
Returns
-------
ConfigSpace.ConfigurationSpace:
A ConfigurationSpace equivalent to the given SeachSpace.
"""
# first convert all non conditionals
non_conditions = {}
conditions = []
for name in space.non_conditions:
param = space.parameters[name]
converted_param = convert_simple_param(name, param)
non_conditions[name] = converted_param
for name in space.conditions:
param = space.parameters[name]
converted_result = convert_simple_param(name, param["result"])
# next build the condition as required by the ConfigSpace
condition = param["condition"]
condition_name = space.uids_to_names[condition["uid"]]
if condition_name not in non_conditions:
raise ValueError("Unknown parameter in Condition")
converted_condition = non_conditions[condition_name]
converted_choices = []
for choice in condition["choices"]:
if isinstance(choice, dict):
if choice["_class"] != "Constant":
raise ValueError("Invalid choice encountered in Condition")
converted_choices.append(choice["value"])
else:
converted_choices.append(choice)
cond = InCondition(converted_result,
converted_condition,
values=converted_choices)
non_conditions[name] = converted_result
conditions.append(cond)
# finally build the ConfigSpace
cs = ConfigurationSpace(seed=np.random.seed())
for _name, param in non_conditions.items():
cs.add_hyperparameter(param)
for cond in conditions:
cs.add_condition(cond)
return cs
def __activate(self, value: Dict, store: Dict, cs: ConfigurationSpace):
assert isinstance(value, dict)
for k, v in value.items():
assert isinstance(v, dict)
reversed_dict = self.reverse_dict(v)
reversed_dict = self.pop_covered_item(reversed_dict, len(v))
for sk, sv in reversed_dict.items():
cond = self.__condition(
{
"_child": sk,
"_values": sv,
"_parent": k
}, store)
cs.add_condition(cond)
def test_remove_inactive_parameter():
configuration_space = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("hp1", choices=[0, 1])
hp2 = CategoricalHyperparameter("hp2", choices=['a'])
hp3 = UniformIntegerHyperparameter("hp3",
lower=0,
upper=5,
default_value=5)
configuration_space.add_hyperparameters([hp1, hp2, hp3])
# If hp1 = 0, then don't allow hp2
not_condition = NotEqualsCondition(hp2, hp1, 0)
configuration_space.add_condition(not_condition)
allowed_cfg = Configuration(configuration_space, {
'hp1': 1,
'hp2': 'a',
'hp3': 5
})
not_allowed = {'hp1': 0, 'hp2': 'a', 'hp3': 5}
with pytest.raises(ValueError):
Configuration(configuration_space, not_allowed)
# No inactive hp - case: config is CS.configuration
transformed = AbstractBenchmark._check_and_cast_configuration(
allowed_cfg, configuration_space)
assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5}
# No inactive hp - case: config is dict
transformed = AbstractBenchmark._check_and_cast_configuration(
allowed_cfg.get_dictionary(), configuration_space)
assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5}
# Remove inactive: - case: config is CS.configuration
not_allowed_cs = Configuration(configuration_space, {
'hp1': 0,
'hp2': 'a',
'hp3': 5
},
allow_inactive_with_values=True)
transformed = AbstractBenchmark._check_and_cast_configuration(
not_allowed_cs, configuration_space)
assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
# Remove inactive: - case: config is dict
transformed = AbstractBenchmark._check_and_cast_configuration(
not_allowed, configuration_space)
assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10))
cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_keys(self):
# A regression test to make sure issue #49 does no longer pop up. By
# iterating over the configuration in the for loop, it should not raise
# a KeyError if the child hyperparameter is inactive.
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default_value="None",
)
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5,
)
cs.add_hyperparameters([shrinkage, shrinkage_factor])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
for i in range(10):
config = cs.sample_configuration()
{hp_name: config[hp_name] for hp_name in config if config[hp_name] is not None}
def test_keys(self):
# A regression test to make sure issue #49 does no longer pop up. By
# iterating over the configuration in the for loop, it should not raise
# a KeyError if the child hyperparameter is inactive.
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default_value="None",
)
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5,
)
cs.add_hyperparameters([shrinkage, shrinkage_factor])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
for i in range(10):
config = cs.sample_configuration()
{hp_name: config[hp_name] for hp_name in config if config[hp_name] is not None}
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegex(
ValueError, r"Adding a second condition \(different\) for a "
r"hyperparameter is ambigouos and "
r"therefore forbidden. Add a conjunction "
r"instead!", cs.add_condition, cond2)
def test_get_parent_and_children_of(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([hp1], cs.get_parents_of(hp2.name))
self.assertEqual([hp1], cs.get_parents_of(hp2))
self.assertEqual([hp2], cs.get_children_of(hp1.name))
self.assertEqual([hp2], cs.get_children_of(hp1))
self.assertRaisesRegex(
KeyError, "Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_parents_of, "Foo")
self.assertRaisesRegex(
KeyError, "Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_children_of, "Foo")
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_repr(self):
cs1 = ConfigurationSpace()
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n",
retval)
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs1.add_hyperparameter(hp1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n" % str(hp1), retval)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n %s\n Conditions:\n %s\n" %
(str(hp2), str(hp1), str(cond1)), retval)
def test_repr(self):
cs1 = ConfigurationSpace()
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n",
retval)
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs1.add_hyperparameter(hp1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n" % str(hp1), retval)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n"
" %s\n %s\n Conditions:\n %s\n" %
(str(hp2), str(hp1), str(cond1)), retval)
def test_add_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2, cond3)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_hyperparameter(hp4)
cs.add_condition(andconj1)
self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_add_second_condition_wo_conjunction(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = Constant("And", "True")
cond1 = EqualsCondition(hp3, hp1, 1)
cond2 = EqualsCondition(hp3, hp2, 1)
cs = ConfigurationSpace()
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
cs.add_hyperparameter(hp3)
cs.add_condition(cond1)
self.assertRaisesRegex(ValueError,
r"Adding a second condition \(different\) for a "
r"hyperparameter is ambigouos and "
r"therefore forbidden. Add a conjunction "
r"instead!",
cs.add_condition, cond2)
def test_check_neighbouring_config_diamond_str(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', ['red', 'green'])
left = CategoricalHyperparameter('left', ['red', 'green'])
right = CategoricalHyperparameter('right', ['red', 'green', 'blue', 'yellow'])
bottom = CategoricalHyperparameter('bottom', ['red', 'green'])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 'red'))
diamond.add_condition(EqualsCondition(right, head, 'red'))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 'green'),
EqualsCondition(bottom, right, 'green')))
config = Configuration(diamond, {'bottom': 'red', 'head': 'red', 'left': 'green', 'right': 'green'})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = ConfigSpace.c_util.change_hp_value(
diamond,
config.get_array(),
hp_name,
neighbor_value,
index
)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def test_check_neighbouring_config_diamond_str(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', ['red', 'green'])
left = CategoricalHyperparameter('left', ['red', 'green'])
right = CategoricalHyperparameter('right',
['red', 'green', 'blue', 'yellow'])
bottom = CategoricalHyperparameter('bottom', ['red', 'green'])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 'red'))
diamond.add_condition(EqualsCondition(right, head, 'red'))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 'green'),
EqualsCondition(bottom, right, 'green')))
config = Configuration(diamond, {
'bottom': 'red',
'head': 'red',
'left': 'green',
'right': 'green'
})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = ConfigSpace.c_util.change_hp_value(diamond,
config.get_array(),
hp_name, neighbor_value,
index)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def test_check_neighbouring_config_diamond(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1, 2, 3])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 1),
EqualsCondition(bottom, right, 1)))
config = Configuration(diamond, {
'bottom': 0,
'head': 0,
'left': 1,
'right': 1
})
hp_name = "head"
index = diamond.get_idx_by_hyperparameter_name(hp_name)
neighbor_value = 1
new_array = change_hp_value(diamond, config.get_array(), hp_name,
neighbor_value, index)
expected_array = np.array([1, np.nan, np.nan, np.nan])
np.testing.assert_almost_equal(new_array, expected_array)
def test_sample_configuration_with_or_conjunction(self):
cs = ConfigurationSpace(seed=1)
hyper_params = {}
hyper_params["hp5"] = CategoricalHyperparameter("hp5", ['0', '1', '2'])
hyper_params["hp7"] = CategoricalHyperparameter("hp7", ['3', '4', '5'])
hyper_params["hp8"] = CategoricalHyperparameter("hp8", ['6', '7', '8'])
for key in hyper_params:
cs.add_hyperparameter(hyper_params[key])
cs.add_condition(
InCondition(hyper_params["hp5"], hyper_params["hp8"], ['6']))
cs.add_condition(
OrConjunction(
InCondition(hyper_params["hp7"], hyper_params["hp8"], ['7']),
InCondition(hyper_params["hp7"], hyper_params["hp5"], ['1'])))
for cfg, fixture in zip(cs.sample_configuration(10),
[[1, np.NaN, 2], [0, 2, np.NaN], [0, 1, 1],
[1, np.NaN, 2], [1, np.NaN, 2]]):
np.testing.assert_array_almost_equal(cfg.get_array(), fixture)
def test_get_parent_and_children_of(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([hp1], cs.get_parents_of(hp2.name))
self.assertEqual([hp1], cs.get_parents_of(hp2))
self.assertEqual([hp2], cs.get_children_of(hp1.name))
self.assertEqual([hp2], cs.get_children_of(hp1))
self.assertRaisesRegex(KeyError,
"Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_parents_of,
"Foo")
self.assertRaisesRegex(KeyError,
"Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_children_of,
"Foo")
def get_combined_cs(estimator_id, node, task_type=0):
cs = ConfigurationSpace()
hpo_cs = get_hpo_cs(estimator_id, task_type)
fe_cs = get_fe_cs(estimator_id, node, task_type)
config_cand = ['placeholder']
config_option = CategoricalHyperparameter('hpo', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = hpo_cs
parent_hyperparameter = {'parent': config_option, 'value': config_item}
cs.add_configuration_space(config_item,
sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
model = UnParametrizedHyperparameter("estimator", estimator_id)
cs.add_hyperparameter(model)
return cs
def test_check_configuration2(self):
# Test that hyperparameters which are not active must not be set and
# that evaluating forbidden clauses does not choke on missing
# hyperparameters
cs = ConfigurationSpace()
classifier = CategoricalHyperparameter("classifier",
["k_nearest_neighbors", "extra_trees"])
metric = CategoricalHyperparameter("metric", ["minkowski", "other"])
p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2])
metric_depends_on_classifier = EqualsCondition(metric, classifier,
"k_nearest_neighbors")
p_depends_on_metric = EqualsCondition(p, metric, "minkowski")
cs.add_hyperparameter(metric)
cs.add_hyperparameter(p)
cs.add_hyperparameter(classifier)
cs.add_condition(metric_depends_on_classifier)
cs.add_condition(p_depends_on_metric)
forbidden = ForbiddenEqualsClause(metric, "other")
cs.add_forbidden_clause(forbidden)
configuration = Configuration(cs, dict(classifier="extra_trees"))
def get_complete_configspace():
"""Creates a configspace that includes all kinds of parameters with
complicated values. The idea is to provide a configspace that can be
used to check modules using ConfigSpace as a dependency to check
compatibility with e.g. Constants, log-scale, etc.
Returns
-------
cs: ConfigurationSpace
cs containing all kinds of parameters
"""
cs = ConfigurationSpace()
hp = {}
# Add Constants for all allowed types ('int', 'float', 'string')
hp['alpha'] = Constant("alpha", 0.0001)
hp['tol'] = Constant("tol", '0.0001')
hp['verbose'] = Constant("verbose", 1)
# Add numericals
# Add Floats
hp['beta1'] = UniformFloatHyperparameter("beta1", 0.85, 0.95, log=False)
hp['power_t'] = NormalFloatHyperparameter("power_t", mu=0.5, sigma=0.1, log=False)
# Add Ints
hp['momentum'] = UniformIntegerHyperparameter("momentum", 0, 100, False)
hp['beta2'] = NormalIntegerHyperparameter("beta2", mu=1, sigma=0.001, log=False)
# Add Floats (log)
hp['learning_rate_init'] = UniformFloatHyperparameter("learning_rate_init", 0.0001, 0.1, log=True)
hp['random1'] = NormalFloatHyperparameter("NormalFloat", mu=0, sigma=1, default_value=1, log=True)
# Add Ints (log)
hp['random2'] = UniformIntegerHyperparameter("UniformInt", 2, 100, log=True)
hp['random3'] = NormalIntegerHyperparameter("NormalInt", mu=0, sigma=1, default_value=1, log=True)
# Add Categorical for allowed types
hp['activation'] = CategoricalHyperparameter('activation', choices=['identity', 'logistic', 'tanh', 'relu'])
hp['solver'] = CategoricalHyperparameter('solver', choices=[-2, 0, 2]) # corrresponds to: ‘lbfgs’, ‘sgd’, ‘adam’
hp['batch_size_auto'] = CategoricalHyperparameter('batch_size_auto', choices=[True, False])
hp['learning_rate'] = CategoricalHyperparameter('learning_rate', choices=[-0.5, 0.0, 0.5]) # corresponds to {‘constant’, ‘invscaling’, ‘adaptive’}
# Add Ordinal
hp['batch_size'] = OrdinalHyperparameter('batch_size', sequence=[32, 64.0, '128'])
for k, v in hp.items():
cs.add_hyperparameter(v)
# learning_rate only with sgd
c = InCondition(hp['learning_rate'], hp['solver'], [0])
c = EqualsCondition(hp['momentum'], hp['solver'], 0)
# learning_rate_init only with sgd or adam
cs.add_condition(OrConjunction(EqualsCondition(hp['learning_rate'], hp['solver'], 0), # sgd
EqualsCondition(hp['learning_rate'], hp['solver'], 2))) # adam
# batch_size only with not batch_size_auto
cs.add_condition(NotEqualsCondition(hp['batch_size'], hp['batch_size_auto'], True))
# complicated way for solver == sgd
#cs.add_condition(AndConjunction(LessThanCondition(hp['power_t'], hp['solver'], 1),
# GreaterThanCondition(hp['power_t'], hp['solver'], -1)))
# betas with adam
cs.add_condition(EqualsCondition(hp['beta1'], hp['solver'], 2))
cs.add_condition(EqualsCondition(hp['beta2'], hp['solver'], 2))
return cs
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def test_add_conditions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs2.add_hyperparameter(hp1)
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
cs2.add_hyperparameter(hp2)
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
cs2.add_hyperparameter(hp3)
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cs2.add_hyperparameter(hp4)
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_condition(cond1)
cs2.add_condition(andCond)
self.assertEqual(str(cs1), str(cs2))
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def test_add_conditions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1]))
cs2.add_hyperparameter(hp1)
hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1]))
cs2.add_hyperparameter(hp2)
hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10))
cs2.add_hyperparameter(hp3)
hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10))
cs2.add_hyperparameter(hp4)
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_condition(cond1)
cs2.add_condition(andCond)
self.assertEqual(str(cs1), str(cs2))
def get_hyperparameter_search_space():
C = UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 10000)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[C, kernel, degree, gamma, coef0, shrinking, tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_combined_cs(node, task_type=0):
cs = ConfigurationSpace()
config_cand = []
cand_space = {}
for estimator_id in _classifiers:
cand_space[estimator_id] = get_hpo_cs(estimator_id, task_type)
config_cand.append(estimator_id)
config_option = CategoricalHyperparameter('estimator', config_cand)
cs.add_hyperparameter(config_option)
for config_item in config_cand:
sub_configuration_space = cand_space[config_item]
parent_hyperparameter = {'parent': config_option,
'value': config_item}
cs.add_configuration_space(config_item, sub_configuration_space,
parent_hyperparameter=parent_hyperparameter)
fe_cs = get_fe_cs(estimator_id, node, task_type)
for hp in fe_cs.get_hyperparameters():
cs.add_hyperparameter(hp)
for cond in fe_cs.get_conditions():
cs.add_condition(cond)
for bid in fe_cs.get_forbiddens():
cs.add_forbidden_clause(bid)
return cs
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparameters_topological_sort(self):
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
# More top-level hyperparameters
hp7 = CategoricalHyperparameter("input7", [0, 1])
# Somewhat shuffled
hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7]
for hp in hyperparameters:
cs.add_hyperparameter(hp)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
hps = cs.get_hyperparameters()
# AND is moved to the front because of alphabetical sorting
for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(cond5)
hps = cs.get_hyperparameters()
for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(conj3)
hps = cs.get_hyperparameters()
# print(hps, hyperparameters)
for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]):
# print(hp, idx)
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparameters_topological_sort(self):
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
# More top-level hyperparameters
hp7 = CategoricalHyperparameter("input7", [0, 1])
# Somewhat shuffled
hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7]
for hp in hyperparameters:
cs.add_hyperparameter(hp)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
hps = cs.get_hyperparameters()
# AND is moved to the front because of alphabetical sorting
for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(cond5)
hps = cs.get_hyperparameters()
for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]):
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
cs.add_condition(conj3)
hps = cs.get_hyperparameters()
# print(hps, hyperparameters)
for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]):
# print(hp, idx)
self.assertEqual(hps.index(hp), idx)
self.assertEqual(cs._hyperparameter_idx[hp.name], idx)
self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_get_types_with_inactive(self):
cs = ConfigurationSpace()
a = cs.add_hyperparameter(CategoricalHyperparameter('a', ['a', 'b']))
b = cs.add_hyperparameter(UniformFloatHyperparameter('b', 1, 5))
c = cs.add_hyperparameter(UniformIntegerHyperparameter('c', 3, 7))
d = cs.add_hyperparameter(Constant('d', -5))
e = cs.add_hyperparameter(OrdinalHyperparameter('e', ['cold', 'hot']))
f = cs.add_hyperparameter(CategoricalHyperparameter('f', ['x', 'y']))
cs.add_condition(EqualsCondition(b, a, 'a'))
cs.add_condition(EqualsCondition(c, a, 'a'))
cs.add_condition(EqualsCondition(d, a, 'a'))
cs.add_condition(EqualsCondition(e, a, 'a'))
cs.add_condition(EqualsCondition(f, a, 'a'))
types, bounds = get_types(cs, None)
np.testing.assert_array_equal(types, [2, 0, 0, 2, 0, 3])
self.assertEqual(bounds[0][0], 2)
self.assertFalse(np.isfinite(bounds[0][1]))
np.testing.assert_array_equal(bounds[1], [-1, 1])
np.testing.assert_array_equal(bounds[2], [-1, 1])
self.assertEqual(bounds[3][0], 2)
self.assertFalse(np.isfinite(bounds[3][1]))
np.testing.assert_array_equal(bounds[4], [0, 2])
self.assertEqual(bounds[5][0], 3)
self.assertFalse(np.isfinite(bounds[5][1]))
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(
UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(
CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'],
default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError,
"Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space.",
):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(
ForbiddenValueError,
"Given vector violates forbidden clause Forbidden: x3 == 2",
):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def recursion(self, hdl: Dict, path=()) -> ConfigurationSpace:
cs = ConfigurationSpace()
# 检测一下这个dict是否在直接描述超参
key_list = list(hdl.keys())
if len(key_list) == 0:
cs.add_hyperparameter(Constant("placeholder", "placeholder"))
return cs
else:
sample_key = key_list[0]
sample_value = hdl[sample_key]
if is_hdl_bottom(sample_key, sample_value):
store = {}
conditions_dict = {}
for key, value in hdl.items():
if purify_key(key).startswith("__"):
conditions_dict[key] = value
else:
hp = self.__parse_dict_to_config(key, value)
cs.add_hyperparameter(hp)
store[key] = hp
for key, value in conditions_dict.items():
if SERIES_CONNECT_LEADER_TOKEN in key:
leader_model, condition_indicator = key.split(
SERIES_CONNECT_LEADER_TOKEN)
else:
leader_model, condition_indicator = None, key
if condition_indicator == "__condition":
assert isinstance(value, list)
for item in value:
cond = self.__condition(item, store, leader_model)
cs.add_condition(cond)
elif condition_indicator == "__activate":
self.__activate(value, store, cs, leader_model)
elif condition_indicator == "__forbidden":
self.__forbidden(value, store, cs, leader_model)
elif condition_indicator == "__rely_model":
RelyModels.info.append([value, deepcopy(path)])
return cs
pattern = re.compile(r"(.*)\((.*)\)")
for key, value in hdl.items():
mat = pattern.match(key)
if mat:
groups = mat.groups()
assert len(groups) == 2
prefix_name, method = groups
value_list = list(value.keys())
assert len(value_list) >= 1
if method == "choice":
pass
else:
raise NotImplementedError()
cur_cs = ConfigurationSpace()
assert isinstance(value, dict)
# 不能用constant，会报错
choice2proba = {}
not_specific_proba_choices = []
sum_proba = 0
for k in value_list:
v = value[k]
if isinstance(v, dict) and "__proba" in v:
proba = v.pop("__proba")
choice2proba[k] = proba
sum_proba += proba
else:
not_specific_proba_choices.append(k)
if sum_proba <= 1:
if len(not_specific_proba_choices) > 0:
p_rest = (1 -
sum_proba) / len(not_specific_proba_choices)
for not_specific_proba_choice in not_specific_proba_choices:
choice2proba[not_specific_proba_choice] = p_rest
else:
choice2proba = {k: 1 / len(value_list) for k in value_list}
proba_list = [choice2proba[k] for k in value_list]
value_list = list(map(smac_hdl._encode,
value_list)) # choices must be str
option_param = CategoricalHyperparameter(
'__choice__', value_list,
weights=proba_list) # todo : default
cur_cs.add_hyperparameter(option_param)
for sub_key, sub_value in value.items():
assert isinstance(sub_value, dict)
sub_cs = self.recursion(sub_value,
path=list(path) +
[prefix_name, sub_key])
parent_hyperparameter = {
'parent': option_param,
'value': sub_key
}
cur_cs.add_configuration_space(
sub_key,
sub_cs,
parent_hyperparameter=parent_hyperparameter)
cs.add_configuration_space(prefix_name, cur_cs)
elif isinstance(value, dict):
sub_cs = self.recursion(value, path=list(path) + [key])
cs.add_configuration_space(key, sub_cs)
else:
raise NotImplementedError()
return cs
def test_check_configuration(self):
# TODO this is only a smoke test
# TODO actually, this rather tests the evaluate methods in the
# conditions module!
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
cs.add_condition(conj3)
expected_outcomes = [False, False, False, False, False,
False, False, True, False, False,
False, False, False, False, False,
True, False, False, False, True,
False, False, False, True, False,
False, False, False, False, False,
False, True]
for idx, values in enumerate(product([0, 1], repeat=5)):
# The hyperparameters aren't sorted, but the test assumes them to
# be sorted.
hyperparameters = sorted(cs.get_hyperparameters(),
key=lambda t: t.name)
instantiations = {hyperparameters[jdx+1].name: values[jdx]
for jdx in range(len(values))}
evaluation = conj3.evaluate(instantiations)
self.assertEqual(expected_outcomes[idx], evaluation)
if not evaluation:
self.assertRaisesRegex(ValueError,
r"Inactive hyperparameter 'AND' must "
r"not be specified, but has the vector value: "
r"'0.0'.",
Configuration, cs, values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True"})
else:
Configuration(cs, values={"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True"})
x5 = CategoricalHyperparameter("x5", ['x1', 'x2', 'x3'])
x6 = CategoricalHyperparameter("x6", ['x1', 'x2', 'x3'])
x7 = CategoricalHyperparameter("x7", ['x1', 'x3', 'x2'], default_value='x2')
cs1.add_hyperparameters([x1, x2, x3, x7, x6, x5, x4])
cond1 = ForbiddenAndConjunction(
ForbiddenEqualsClause(x3, "x1"),
ForbiddenEqualsClause(x4, "x2"),
ForbiddenEqualsClause(x5, "x3")
)
cs1.add_forbidden_clause(cond1)
cond2 = EqualsCondition(x1, x5, "x1")
cond7 = EqualsCondition(x2, x6, "x1")
cond3 = InCondition(x2, x6, ["x1", "x2", "x3"])
cs1.add_condition(cond3)
cs1.add_condition(cond2)
cond4 = ForbiddenEqualsClause(x4, 'x3')
cond5 = ForbiddenInClause(x7, ['x1', 'x3'])
cs1.add_forbidden_clause(cond5)
cs1.add_forbidden_clause(cond4)
from litebo.utils.config_space.space_utils import config_space2string, string2config_space
str = config_space2string(cs1)
print(str)
cs2 = string2config_space(str)
print(cs2 == cs1)
exit()
def test_check_configuration(self):
# TODO this is only a smoke test
# TODO actually, this rather tests the evaluate methods in the
# conditions module!
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
cs.add_condition(conj3)
expected_outcomes = [
False, False, False, False, False, False, False, True, False,
False, False, False, False, False, False, True, False, False,
False, True, False, False, False, True, False, False, False, False,
False, False, False, True
]
for idx, values in enumerate(product([0, 1], repeat=5)):
# The hyperparameters aren't sorted, but the test assumes them to
# be sorted.
hyperparameters = sorted(cs.get_hyperparameters(),
key=lambda t: t.name)
instantiations = {
hyperparameters[jdx + 1].name: values[jdx]
for jdx in range(len(values))
}
evaluation = conj3.evaluate(instantiations)
self.assertEqual(expected_outcomes[idx], evaluation)
if not evaluation:
self.assertRaisesRegex(
ValueError,
r"Inactive hyperparameter 'AND' must "
r"not be specified, but has the vector value: "
r"'0.0'.",
Configuration,
cs,
values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True",
},
)
else:
Configuration(
cs,
values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True",
},
)
def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 1,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 1,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 1, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1,
'right': 0, 'bottom': 0},
diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def get_hyperparameter_search_space(dataset_properties=None):
max_num_layers = 7 # Maximum number of layers coded
# Hacky way to condition layers params based on the number of layers
# 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer
layer_choices = [
chr(i) for i in range(ord('c'),
ord('b') + max_num_layers)
]
batch_size = UniformIntegerHyperparameter("batch_size",
32,
4096,
log=True,
default=32)
number_epochs = UniformIntegerHyperparameter("number_epochs",
2,
80,
default=5)
num_layers = CategoricalHyperparameter("num_layers",
choices=layer_choices,
default='c')
lr = UniformFloatHyperparameter("learning_rate",
1e-6,
1.0,
log=True,
default=0.01)
l2 = UniformFloatHyperparameter("lambda2",
1e-7,
1e-2,
log=True,
default=1e-4)
dropout_output = UniformFloatHyperparameter("dropout_output",
0.0,
0.99,
default=0.5)
# Define basic hyperparameters and define the config space
# basic means that are independent from the number of layers
cs = ConfigurationSpace()
cs.add_hyperparameter(number_epochs)
cs.add_hyperparameter(batch_size)
cs.add_hyperparameter(num_layers)
cs.add_hyperparameter(lr)
cs.add_hyperparameter(l2)
cs.add_hyperparameter(dropout_output)
# Define parameters with different child parameters and conditions
solver_choices = [
"adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov",
"smorm3s"
]
solver = CategoricalHyperparameter(name="solver",
choices=solver_choices,
default="smorm3s")
beta1 = UniformFloatHyperparameter("beta1",
1e-4,
0.1,
log=True,
default=0.1)
beta2 = UniformFloatHyperparameter("beta2",
1e-4,
0.1,
log=True,
default=0.01)
rho = UniformFloatHyperparameter("rho",
0.05,
0.99,
log=True,
default=0.95)
momentum = UniformFloatHyperparameter("momentum",
0.3,
0.999,
default=0.9)
# TODO: Add policy based on this sklearn sgd
policy_choices = ['fixed', 'inv', 'exp', 'step']
lr_policy = CategoricalHyperparameter(name="lr_policy",
choices=policy_choices,
default='fixed')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-3,
upper=1e-1,
default=1e-2)
power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5)
epoch_step = UniformIntegerHyperparameter("epoch_step",
2,
20,
default=5)
cs.add_hyperparameter(solver)
cs.add_hyperparameter(beta1)
cs.add_hyperparameter(beta2)
cs.add_hyperparameter(momentum)
cs.add_hyperparameter(rho)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(power)
cs.add_hyperparameter(epoch_step)
# Define parameters that are needed it for each layer
output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh']
activations_choices = [
'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear'
]
weight_choices = [
'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform',
'he_normal', 'he_uniform', 'ortogonal', 'sparse'
]
# Iterate over parameters that are used in each layer
for i in range(1, max_num_layers):
layer_units = UniformIntegerHyperparameter("num_units_layer_" +
str(i),
64,
4096,
log=True,
default=128)
cs.add_hyperparameter(layer_units)
layer_dropout = UniformFloatHyperparameter("dropout_layer_" +
str(i),
0.0,
0.99,
default=0.5)
cs.add_hyperparameter(layer_dropout)
weight_initialization = CategoricalHyperparameter(
'weight_init_' + str(i),
choices=weight_choices,
default='he_normal')
cs.add_hyperparameter(weight_initialization)
layer_std = UniformFloatHyperparameter("std_layer_" + str(i),
1e-6,
0.1,
log=True,
default=0.005)
cs.add_hyperparameter(layer_std)
layer_activation = CategoricalHyperparameter(
"activation_layer_" + str(i),
choices=activations_choices,
default="relu")
cs.add_hyperparameter(layer_activation)
layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' +
str(i),
0.01,
0.99,
default=0.3)
cs.add_hyperparameter(layer_leakiness)
layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' +
str(i),
0.5,
1.0,
default=2. / 3.)
cs.add_hyperparameter(layer_tanh_alpha)
layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' +
str(i),
1.1,
3.0,
log=True,
default=1.7159)
cs.add_hyperparameter(layer_tanh_beta)
# TODO: Could be in a function in a new module
for i in range(2, max_num_layers):
# Condition layers parameter on layer choice
layer_unit_param = cs.get_hyperparameter("num_units_layer_" +
str(i))
layer_cond = InCondition(child=layer_unit_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition dropout parameter on layer choice
layer_dropout_param = cs.get_hyperparameter("dropout_layer_" +
str(i))
layer_cond = InCondition(child=layer_dropout_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition weight initialization on layer choice
layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i))
layer_cond = InCondition(child=layer_weight_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition std parameter on weight layer initialization choice
layer_std_param = cs.get_hyperparameter("std_layer_" + str(i))
weight_cond = EqualsCondition(child=layer_std_param,
parent=layer_weight_param,
value='normal')
cs.add_condition(weight_cond)
# Condition activation parameter on layer choice
layer_activation_param = cs.get_hyperparameter(
"activation_layer_" + str(i))
layer_cond = InCondition(child=layer_activation_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition leakiness on activation choice
layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_leakiness_param,
parent=layer_activation_param,
value='leaky')
cs.add_condition(activation_cond)
# Condition tanh on activation choice
layer_tanh_alpha_param = cs.get_hyperparameter(
"tanh_alpha_layer_" + str(i))
activation_cond = EqualsCondition(child=layer_tanh_alpha_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_tanh_beta_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
# Conditioning on solver
momentum_depends_on_solver = InCondition(
momentum, solver, values=["momentum", "nesterov"])
beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam")
beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam")
rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta")
cs.add_condition(momentum_depends_on_solver)
cs.add_condition(beta1_depends_on_solver)
cs.add_condition(beta2_depends_on_solver)
cs.add_condition(rho_depends_on_solver)
# Conditioning on learning rate policy
lr_policy_depends_on_solver = InCondition(
lr_policy, solver,
["adadelta", "adagrad", "sgd", "momentum", "nesterov"])
gamma_depends_on_policy = InCondition(child=gamma,
parent=lr_policy,
values=["inv", "exp", "step"])
power_depends_on_policy = EqualsCondition(power, lr_policy, "inv")
epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy,
"step")
cs.add_condition(lr_policy_depends_on_solver)
cs.add_condition(gamma_depends_on_policy)
cs.add_condition(power_depends_on_policy)
cs.add_condition(epoch_step_depends_on_policy)
return cs