def test_add_conditions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(
CategoricalHyperparameter("input1", [0, 1]))
cs2.add_hyperparameter(hp1)
hp2 = cs1.add_hyperparameter(
CategoricalHyperparameter("input2", [0, 1]))
cs2.add_hyperparameter(hp2)
hp3 = cs1.add_hyperparameter(
UniformIntegerHyperparameter("child1", 0, 10))
cs2.add_hyperparameter(hp3)
hp4 = cs1.add_hyperparameter(
UniformIntegerHyperparameter("child2", 0, 10))
cs2.add_hyperparameter(hp4)
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_condition(cond1)
cs2.add_condition(andCond)
self.assertEqual(str(cs1), str(cs2))
def setUp(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter(
"parent", [0, 1]))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
hp3 = cs.add_hyperparameter(
UniformIntegerHyperparameter("friend", 0, 5))
self.cs = cs
def test_random_neighborhood_int(self):
hp = UniformIntegerHyperparameter('a', 1, 10)
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor['a'] for neighbor in all_neighbors]
self.assertAlmostEqual(5.79, np.mean(all_neighbors), places=2)
self.assertAlmostEqual(4.99, np.var(all_neighbors), places=2)
hp = UniformIntegerHyperparameter('a', 1, 10, log=True)
all_neighbors = self._test_get_one_exchange_neighbourhood(hp)
all_neighbors = [neighbor['a'] for neighbor in all_neighbors]
# Default value is 3.16
self.assertAlmostEqual(3.55, np.mean(all_neighbors), places=2)
self.assertAlmostEqual(5.91, np.var(all_neighbors), places=2)
def test_add_hyperparameters_with_equal_names(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
self.assertRaisesRegexp(
ValueError, "Hyperparameter 'name' is already in the "
"configuration space.", cs.add_hyperparameter, hp)
def test_hyperparameters_with_valid_condition(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond)
self.assertEqual(len(cs._hyperparameters), 2)
def test_get_conditions(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
self.assertEqual([], cs.get_conditions())
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([cond1], cs.get_conditions())
def test_get_hyperparameters_topological_sort_simple(self):
for iteration in range(10):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
def test_get_hyperparameter(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
retval = cs.get_hyperparameter("parent")
self.assertEqual(hp1, retval)
retval = cs.get_hyperparameter("child")
self.assertEqual(hp2, retval)
self.assertRaises(KeyError, cs.get_hyperparameter, "grandfather")
def test_sample_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace(seed=1)
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp5, hp3, 1)
cond5 = EqualsCondition(hp4, hp5, 1)
cond6 = EqualsCondition(hp6, hp4, 1)
cond7 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond6, cond7)
cs.add_condition(cond4)
cs.add_condition(cond5)
cs.add_condition(conj3)
samples = []
for i in range(5):
cs.seed(1)
samples.append([])
for j in range(100):
sample = cs.sample_configuration()
samples[-1].append(sample)
if i > 0:
for j in range(100):
self.assertEqual(samples[-1][j], samples[-2][j])
def test_condition_with_cycles(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
cond2 = EqualsCondition(hp1, hp2, 0)
self.assertRaisesRegexp(
ValueError, "Hyperparameter configuration "
"contains a cycle \[\['child', 'parent'\]\]", cs.add_condition,
cond2)
def test_get_hyperparamforbidden_clauseseters(self):
cs = ConfigurationSpace()
self.assertEqual(0, len(cs.get_hyperparameters()))
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
self.assertEqual([hp1], cs.get_hyperparameters())
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 1)
cs.add_condition(cond1)
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
# TODO: I need more tests for the topological sort!
self.assertEqual([hp1, hp2], cs.get_hyperparameters())
def test_eq(self):
# Compare empty configuration spaces
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
self.assertEqual(cs1, cs2)
# Compare to something which isn't a configuration space
self.assertTrue(not (cs1 == "ConfigurationSpace"))
# Compare to equal configuration spaces
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("friend", 0, 5)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp1)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
cs2.add_hyperparameter(hp1)
cs2.add_hyperparameter(hp2)
cs2.add_condition(cond1)
self.assertEqual(cs1, cs2)
cs1.add_hyperparameter(hp3)
self.assertFalse(cs1 == cs2)
def test_add_configuration_space_conjunctions(self):
cs1 = ConfigurationSpace()
cs2 = ConfigurationSpace()
hp1 = cs1.add_hyperparameter(
CategoricalHyperparameter("input1", [0, 1]))
hp2 = cs1.add_hyperparameter(
CategoricalHyperparameter("input2", [0, 1]))
hp3 = cs1.add_hyperparameter(
UniformIntegerHyperparameter("child1", 0, 10))
hp4 = cs1.add_hyperparameter(
UniformIntegerHyperparameter("child2", 0, 10))
cond1 = EqualsCondition(hp2, hp3, 0)
cond2 = EqualsCondition(hp1, hp3, 5)
cond3 = EqualsCondition(hp1, hp4, 1)
andCond = AndConjunction(cond2, cond3)
cs1.add_conditions([cond1, andCond])
cs2.add_configuration_space(prefix='test', configuration_space=cs1)
self.assertEqual(str(cs2).count('test:'), 10)
# Check that they're equal except for the "test:" prefix
self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def test_condition_without_added_hyperparameters(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
self.assertRaisesRegexp(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
cs.add_hyperparameter(hp1)
self.assertRaisesRegexp(
ValueError, "Child hyperparameter 'child' not "
"in configuration space.", cs.add_condition, cond)
# Test also the parent hyperparameter
cs2 = ConfigurationSpace()
cs2.add_hyperparameter(hp2)
self.assertRaisesRegexp(
ValueError, "Parent hyperparameter 'parent' "
"not in configuration space.", cs2.add_condition, cond)
def test_get_parent_and_children_of(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
self.assertEqual([hp1], cs.get_parents_of(hp2.name))
self.assertEqual([hp1], cs.get_parents_of(hp2))
self.assertEqual([hp2], cs.get_children_of(hp1.name))
self.assertEqual([hp2], cs.get_children_of(hp1))
self.assertRaisesRegexp(
KeyError, "Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_parents_of, "Foo")
self.assertRaisesRegexp(
KeyError, "Hyperparameter 'Foo' does not exist in this "
"configuration space.", cs.get_children_of, "Foo")
def test_add_configuration_space(self):
cs = ConfigurationSpace()
hp1 = cs.add_hyperparameter(CategoricalHyperparameter(
"input1", [0, 1]))
forb1 = cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1))
hp2 = cs.add_hyperparameter(
UniformIntegerHyperparameter("child", 0, 10))
cond = cs.add_condition(EqualsCondition(hp2, hp1, 0))
cs2 = ConfigurationSpace()
cs2.add_configuration_space('prefix', cs, delimiter='__')
self.assertEqual(
str(cs2), '''Configuration space object:
Hyperparameters:
prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5
prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0
Conditions:
prefix__child | prefix__input1 == 0
Forbidden Clauses:
Forbidden: prefix__input1 == 1
''')
def test_repr(self):
cs1 = ConfigurationSpace()
retval = cs1.__str__()
self.assertEqual("Configuration space object:\n Hyperparameters:\n",
retval)
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs1.add_hyperparameter(hp1)
retval = cs1.__str__()
self.assertEqual(
"Configuration space object:\n Hyperparameters:\n"
" %s\n" % str(hp1), retval)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond1 = EqualsCondition(hp2, hp1, 0)
cs1.add_hyperparameter(hp2)
cs1.add_condition(cond1)
retval = cs1.__str__()
self.assertEqual(
"Configuration space object:\n Hyperparameters:\n"
" %s\n %s\n Conditions:\n %s\n" %
(str(hp2), str(hp1), str(cond1)), retval)
def test_random_neighbor_int(self):
hp = UniformIntegerHyperparameter('a', 1, 10)
self._test_random_neigbor(hp)
hp = UniformIntegerHyperparameter('a', 1, 10, log=True)
self._test_random_neigbor(hp)
def test_setitem(self):
'''
Checks overriding a sampled configuration
'''
pcs = ConfigurationSpace()
pcs.add_hyperparameter(
UniformIntegerHyperparameter('x0', 1, 5, default_value=1))
x1 = pcs.add_hyperparameter(
CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'],
default_value='ab'))
# Condition
x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2]))
pcs.add_condition(EqualsCondition(x2, x1, 'ab'))
# Forbidden
x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2]))
pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2))
conf = pcs.get_default_configuration()
# failed because it's a invalid configuration
with self.assertRaisesRegex(ValueError,
"Illegal value '0' for hyperparameter x0"):
conf['x0'] = 0
# failed because the variable didn't exists
with self.assertRaisesRegex(
KeyError,
"Hyperparameter 'x_0' does not exist in this configuration space."
):
conf['x_0'] = 1
# failed because forbidden clause is violated
with self.assertRaisesRegex(
ForbiddenValueError,
"Given vector violates forbidden clause Forbidden: x3 == 2"):
conf['x3'] = 2
self.assertEqual(conf['x3'], 1)
# successful operation 1
x0_old = conf['x0']
if x0_old == 1:
conf['x0'] = 2
else:
conf['x0'] = 1
x0_new = conf['x0']
self.assertNotEqual(x0_old, x0_new)
pcs._check_configuration_rigorous(conf)
self.assertEqual(conf['x2'], 1)
# successful operation 2
x1_old = conf['x1']
if x1_old == 'ab':
conf['x1'] = 'cd'
else:
conf['x1'] = 'ab'
x1_new = conf['x1']
self.assertNotEqual(x1_old, x1_new)
pcs._check_configuration_rigorous(conf)
self.assertRaises(KeyError, conf.__getitem__, 'x2')
def test_deactivate_inactive_hyperparameters(self):
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
AndConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 1,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
diamond = ConfigurationSpace()
head = CategoricalHyperparameter('head', [0, 1])
left = CategoricalHyperparameter('left', [0, 1])
right = CategoricalHyperparameter('right', [0, 1])
bottom = CategoricalHyperparameter('bottom', [0, 1])
diamond.add_hyperparameters([head, left, right, bottom])
diamond.add_condition(EqualsCondition(left, head, 0))
diamond.add_condition(EqualsCondition(right, head, 0))
diamond.add_condition(
OrConjunction(EqualsCondition(bottom, left, 0),
EqualsCondition(bottom, right, 0)))
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 0,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 1,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
c = deactivate_inactive_hyperparameters(
{
'head': 0,
'left': 1,
'right': 0,
'bottom': 0
}, diamond)
diamond._check_configuration_rigorous(c)
plain = ConfigurationSpace()
a = UniformIntegerHyperparameter('a', 0, 10)
b = UniformIntegerHyperparameter('b', 0, 10)
plain.add_hyperparameters([a, b])
c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain)
plain.check_configuration(c)
def test_add_hyperparameter(self):
cs = ConfigurationSpace()
hp = UniformIntegerHyperparameter("name", 0, 10)
cs.add_hyperparameter(hp)
def test_check_configuration(self):
# TODO this is only a smoke test
# TODO actually, this rather tests the evaluate methods in the
# conditions module!
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cs.add_hyperparameter(hp2)
cond1 = EqualsCondition(hp2, hp1, 0)
cs.add_condition(cond1)
# This automatically checks the configuration!
Configuration(cs, dict(parent=0, child=5))
# and now for something more complicated
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("input1", [0, 1])
cs.add_hyperparameter(hp1)
hp2 = CategoricalHyperparameter("input2", [0, 1])
cs.add_hyperparameter(hp2)
hp3 = CategoricalHyperparameter("input3", [0, 1])
cs.add_hyperparameter(hp3)
hp4 = CategoricalHyperparameter("input4", [0, 1])
cs.add_hyperparameter(hp4)
hp5 = CategoricalHyperparameter("input5", [0, 1])
cs.add_hyperparameter(hp5)
hp6 = Constant("AND", "True")
cs.add_hyperparameter(hp6)
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = NotEqualsCondition(hp6, hp2, 1)
cond3 = InCondition(hp6, hp3, [1])
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
cs.add_condition(conj3)
expected_outcomes = [
False, False, False, False, False, False, False, True, False,
False, False, False, False, False, False, True, False, False,
False, True, False, False, False, True, False, False, False, False,
False, False, False, True
]
for idx, values in enumerate(product([0, 1], repeat=5)):
# The hyperparameters aren't sorted, but the test assumes them to
# be sorted.
hyperparameters = sorted(cs.get_hyperparameters(),
key=lambda t: t.name)
instantiations = {
hyperparameters[jdx + 1].name: values[jdx]
for jdx in range(len(values))
}
evaluation = conj3.evaluate(instantiations)
self.assertEqual(expected_outcomes[idx], evaluation)
if evaluation == False:
self.assertRaisesRegexp(
ValueError, "Inactive hyperparameter 'AND' must "
"not be specified, but has the vector value: "
"'0.0'.",
Configuration,
cs,
values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True"
})
else:
Configuration(cs,
values={
"input1": values[0],
"input2": values[1],
"input3": values[2],
"input4": values[3],
"input5": values[4],
"AND": "True"
})