def test_get_hyperparameters_topological_sort(self): # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") # More top-level hyperparameters hp7 = CategoricalHyperparameter("input7", [0, 1]) # Somewhat shuffled hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7] for hp in hyperparameters: cs.add_hyperparameter(hp) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) hps = cs.get_hyperparameters() # AND is moved to the front because of alphabetical sorting for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]): self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name) cs.add_condition(cond5) hps = cs.get_hyperparameters() for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]): self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name) cs.add_condition(conj3) hps = cs.get_hyperparameters() # print(hps, hyperparameters) for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]): # print(hp, idx) self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def test_add_conditions(self): cs1 = ConfigurationSpace() cs2 = ConfigurationSpace() hp1 = cs1.add_hyperparameter( CategoricalHyperparameter("input1", [0, 1])) cs2.add_hyperparameter(hp1) hp2 = cs1.add_hyperparameter( CategoricalHyperparameter("input2", [0, 1])) cs2.add_hyperparameter(hp2) hp3 = cs1.add_hyperparameter( UniformIntegerHyperparameter("child1", 0, 10)) cs2.add_hyperparameter(hp3) hp4 = cs1.add_hyperparameter( UniformIntegerHyperparameter("child2", 0, 10)) cs2.add_hyperparameter(hp4) cond1 = EqualsCondition(hp2, hp3, 0) cond2 = EqualsCondition(hp1, hp3, 5) cond3 = EqualsCondition(hp1, hp4, 1) andCond = AndConjunction(cond2, cond3) cs1.add_conditions([cond1, andCond]) cs2.add_condition(cond1) cs2.add_condition(andCond) self.assertEqual(str(cs1), str(cs2))
def test_check_neighbouring_config_diamond_str(self): diamond = ConfigurationSpace() head = CategoricalHyperparameter('head', ['red', 'green']) left = CategoricalHyperparameter('left', ['red', 'green']) right = CategoricalHyperparameter('right', ['red', 'green', 'blue', 'yellow']) bottom = CategoricalHyperparameter('bottom', ['red', 'green']) diamond.add_hyperparameters([head, left, right, bottom]) diamond.add_condition(EqualsCondition(left, head, 'red')) diamond.add_condition(EqualsCondition(right, head, 'red')) diamond.add_condition( AndConjunction(EqualsCondition(bottom, left, 'green'), EqualsCondition(bottom, right, 'green'))) config = Configuration(diamond, { 'bottom': 'red', 'head': 'red', 'left': 'green', 'right': 'green' }) hp_name = "head" index = diamond.get_idx_by_hyperparameter_name(hp_name) neighbor_value = 1 new_array = ConfigSpace.c_util.change_hp_value(diamond, config.get_array(), hp_name, neighbor_value, index) expected_array = np.array([1, np.nan, np.nan, np.nan]) np.testing.assert_almost_equal(new_array, expected_array)
def test_deactivate_inactive_hyperparameters(self): diamond = ConfigurationSpace() head = CategoricalHyperparameter('head', [0, 1]) left = CategoricalHyperparameter('left', [0, 1]) right = CategoricalHyperparameter('right', [0, 1]) bottom = CategoricalHyperparameter('bottom', [0, 1]) diamond.add_hyperparameters([head, left, right, bottom]) diamond.add_condition(EqualsCondition(left, head, 0)) diamond.add_condition(EqualsCondition(right, head, 0)) diamond.add_condition(AndConjunction(EqualsCondition(bottom, left, 0), EqualsCondition(bottom, right, 0))) c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0, 'right': 0, 'bottom': 0}, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters({'head': 1, 'left': 0, 'right': 0, 'bottom': 0}, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1, 'right': 0, 'bottom': 0}, diamond) diamond._check_configuration_rigorous(c) diamond = ConfigurationSpace() head = CategoricalHyperparameter('head', [0, 1]) left = CategoricalHyperparameter('left', [0, 1]) right = CategoricalHyperparameter('right', [0, 1]) bottom = CategoricalHyperparameter('bottom', [0, 1]) diamond.add_hyperparameters([head, left, right, bottom]) diamond.add_condition(EqualsCondition(left, head, 0)) diamond.add_condition(EqualsCondition(right, head, 0)) diamond.add_condition(OrConjunction(EqualsCondition(bottom, left, 0), EqualsCondition(bottom, right, 0))) c = deactivate_inactive_hyperparameters({'head': 0, 'left': 0, 'right': 0, 'bottom': 0}, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters({'head': 1, 'left': 1, 'right': 0, 'bottom': 0}, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters({'head': 0, 'left': 1, 'right': 0, 'bottom': 0}, diamond) diamond._check_configuration_rigorous(c) plain = ConfigurationSpace() a = UniformIntegerHyperparameter('a', 0, 10) b = UniformIntegerHyperparameter('b', 0, 10) plain.add_hyperparameters([a, b]) c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain) plain.check_configuration(c)
def test_check_neighbouring_config_diamond(self): diamond = ConfigurationSpace() head = CategoricalHyperparameter('head', [0, 1]) left = CategoricalHyperparameter('left', [0, 1]) right = CategoricalHyperparameter('right', [0, 1, 2, 3]) bottom = CategoricalHyperparameter('bottom', [0, 1]) diamond.add_hyperparameters([head, left, right, bottom]) diamond.add_condition(EqualsCondition(left, head, 0)) diamond.add_condition(EqualsCondition(right, head, 0)) diamond.add_condition( AndConjunction(EqualsCondition(bottom, left, 1), EqualsCondition(bottom, right, 1))) config = Configuration(diamond, { 'bottom': 0, 'head': 0, 'left': 1, 'right': 1 }) hp_name = "head" index = diamond.get_idx_by_hyperparameter_name(hp_name) neighbor_value = 1 new_array = change_hp_value(diamond, config.get_array(), hp_name, neighbor_value, index) expected_array = np.array([1, np.nan, np.nan, np.nan]) np.testing.assert_almost_equal(new_array, expected_array)
def test_add_conjunction(self): hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = Constant("And", "True") cond1 = EqualsCondition(hp4, hp1, 1) cond2 = EqualsCondition(hp4, hp2, 1) cond3 = EqualsCondition(hp4, hp3, 1) andconj1 = AndConjunction(cond1, cond2, cond3) cs = ConfigurationSpace() cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) cs.add_hyperparameter(hp3) cs.add_hyperparameter(hp4) cs.add_condition(andconj1) self.assertNotIn(hp4, cs.get_all_unconditional_hyperparameters())
def test_add_configuration_space_conjunctions(self): cs1 = ConfigurationSpace() cs2 = ConfigurationSpace() hp1 = cs1.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1])) hp2 = cs1.add_hyperparameter(CategoricalHyperparameter("input2", [0, 1])) hp3 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child1", 0, 10)) hp4 = cs1.add_hyperparameter(UniformIntegerHyperparameter("child2", 0, 10)) cond1 = EqualsCondition(hp2, hp3, 0) cond2 = EqualsCondition(hp1, hp3, 5) cond3 = EqualsCondition(hp1, hp4, 1) andCond = AndConjunction(cond2, cond3) cs1.add_conditions([cond1, andCond]) cs2.add_configuration_space(prefix='test', configuration_space=cs1) self.assertEqual(str(cs2).count('test:'), 10) # Check that they're equal except for the "test:" prefix self.assertEqual(str(cs1), str(cs2).replace('test:', ''))
def test_check_configuration(self): # TODO this is only a smoke test # TODO actually, this rather tests the evaluate methods in the # conditions module! cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) cs.add_condition(conj3) expected_outcomes = [ False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True ] for idx, values in enumerate(product([0, 1], repeat=5)): # The hyperparameters aren't sorted, but the test assumes them to # be sorted. hyperparameters = sorted(cs.get_hyperparameters(), key=lambda t: t.name) instantiations = { hyperparameters[jdx + 1].name: values[jdx] for jdx in range(len(values)) } evaluation = conj3.evaluate(instantiations) self.assertEqual(expected_outcomes[idx], evaluation) if not evaluation: self.assertRaisesRegex( ValueError, r"Inactive hyperparameter 'AND' must " r"not be specified, but has the vector value: " r"'0.0'.", Configuration, cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True", }, ) else: Configuration( cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True", }, )
def test_check_configuration(self): # TODO this is only a smoke test # TODO actually, this rather tests the evaluate methods in the # conditions module! cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) cs.add_condition(conj3) expected_outcomes = [False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True] for idx, values in enumerate(product([0, 1], repeat=5)): # The hyperparameters aren't sorted, but the test assumes them to # be sorted. hyperparameters = sorted(cs.get_hyperparameters(), key=lambda t: t.name) instantiations = {hyperparameters[jdx+1].name: values[jdx] for jdx in range(len(values))} evaluation = conj3.evaluate(instantiations) self.assertEqual(expected_outcomes[idx], evaluation) if not evaluation: self.assertRaisesRegex(ValueError, r"Inactive hyperparameter 'AND' must " r"not be specified, but has the vector value: " r"'0.0'.", Configuration, cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True"}) else: Configuration(cs, values={"input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True"})
def test_generate_grid(self): '''Test grid generation''' # Sub-test 1 cs = ConfigurationSpace(seed=1234) cat1 = CategoricalHyperparameter(name='cat1', choices=['T', 'F']) const1 = Constant(name='const1', value=4) float1 = UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) int1 = UniformIntegerHyperparameter(name='int1', lower=10, upper=100, log=True) ord1 = OrdinalHyperparameter(name='ord1', sequence=['1', '2', '3']) cs.add_hyperparameters([float1, int1, cat1, ord1, const1]) num_steps_dict = {'float1': 11, 'int1': 6} generated_grid = generate_grid(cs, num_steps_dict) # Check randomly pre-selected values in the generated_grid # 2 * 1 * 11 * 6 * 3 total diff. possible configurations self.assertEqual(len(generated_grid), 396) # Check 1st and last generated configurations completely: first_expected_dict = { 'cat1': 'T', 'const1': 4, 'float1': -1.0, 'int1': 10, 'ord1': '1' } last_expected_dict = { 'cat1': 'F', 'const1': 4, 'float1': 1.0, 'int1': 100, 'ord1': '3' } self.assertEqual(generated_grid[0].get_dictionary(), first_expected_dict) self.assertEqual(generated_grid[-1].get_dictionary(), last_expected_dict) self.assertEqual(generated_grid[198].get_dictionary()['cat1'], 'F') self.assertEqual(generated_grid[45].get_dictionary()['const1'], 4) # The 2 most frequently changing HPs (int1 and ord1) have 3 * 6 = 18 different values for # each value of float1, so the 4th value of float1 of -0.4 is reached after # 3 * 18 = 54 values in the generated_grid (and remains the same for the next 18 values): for i in range(18): self.assertAlmostEqual( generated_grid[54 + i].get_dictionary()['float1'], -0.4, places=2) # 5th diff. value for int1 after 4 * 3 = 12 values. Reasoning as above. self.assertEqual(generated_grid[12].get_dictionary()['int1'], 63) self.assertEqual(generated_grid[3].get_dictionary()['ord1'], '1') self.assertEqual(generated_grid[4].get_dictionary()['ord1'], '2') self.assertEqual(generated_grid[5].get_dictionary()['ord1'], '3') # Sub-test 2 # Test for extreme cases: only numerical cs = ConfigurationSpace(seed=1234) cs.add_hyperparameters([float1, int1]) num_steps_dict = {'float1': 11, 'int1': 6} generated_grid = generate_grid(cs, num_steps_dict) self.assertEqual(len(generated_grid), 66) # Check 1st and last generated configurations completely: first_expected_dict = {'float1': -1.0, 'int1': 10} last_expected_dict = {'float1': 1.0, 'int1': 100} self.assertEqual(generated_grid[0].get_dictionary(), first_expected_dict) self.assertEqual(generated_grid[-1].get_dictionary(), last_expected_dict) # Test: only categorical cs = ConfigurationSpace(seed=1234) cs.add_hyperparameters([cat1]) generated_grid = generate_grid(cs) self.assertEqual(len(generated_grid), 2) # Check 1st and last generated configurations completely: self.assertEqual(generated_grid[0].get_dictionary()['cat1'], 'T') self.assertEqual(generated_grid[-1].get_dictionary()['cat1'], 'F') # Test: only constant cs = ConfigurationSpace(seed=1234) cs.add_hyperparameters([const1]) generated_grid = generate_grid(cs) self.assertEqual(len(generated_grid), 1) # Check 1st and only generated configuration completely: self.assertEqual(generated_grid[0].get_dictionary()['const1'], 4) # Test: no hyperparameters yet cs = ConfigurationSpace(seed=1234) generated_grid = generate_grid(cs, num_steps_dict) # For the case of no hyperparameters, in get_cartesian_product, itertools.product() returns # a single empty tuple element which leads to a single empty Configuration. self.assertEqual(len(generated_grid), 0) # Sub-test 3 # Tests for quantization and conditional spaces. num_steps_dict supports specifying steps # for only some of the int and float HPs. The rest are taken from the 'q' member variables # of these HPs. The conditional space tested has 2 levels of conditions. cs2 = ConfigurationSpace(seed=123) float1 = UniformFloatHyperparameter(name='float1', lower=-1, upper=1, log=False) int1 = UniformIntegerHyperparameter(name='int1', lower=0, upper=1000, log=False, q=500) cs2.add_hyperparameters([float1, int1]) int2_cond = UniformIntegerHyperparameter(name='int2_cond', lower=10, upper=100, log=True) cs2.add_hyperparameters([int2_cond]) cond_1 = AndConjunction(LessThanCondition(int2_cond, float1, -0.5), GreaterThanCondition(int2_cond, int1, 600)) cs2.add_conditions([cond_1]) cat1_cond = CategoricalHyperparameter(name='cat1_cond', choices=['apple', 'orange']) cs2.add_hyperparameters([cat1_cond]) cond_2 = AndConjunction(GreaterThanCondition(cat1_cond, int1, 300), LessThanCondition(cat1_cond, int1, 700), GreaterThanCondition(cat1_cond, float1, -0.5), LessThanCondition(cat1_cond, float1, 0.5)) cs2.add_conditions([cond_2]) float2_cond = UniformFloatHyperparameter(name='float2_cond', lower=10., upper=100., log=True) # 2nd level dependency in ConfigurationSpace tree being tested cs2.add_hyperparameters([float2_cond]) cond_3 = GreaterThanCondition(float2_cond, int2_cond, 50) cs2.add_conditions([cond_3]) num_steps_dict1 = {'float1': 4, 'int2_cond': 3, 'float2_cond': 3} generated_grid = generate_grid(cs2, num_steps_dict1) self.assertEqual(len(generated_grid), 18) # RR: I manually generated the grid and verified the values were correct. # Check 1st and last generated configurations completely: first_expected_dict = {'float1': -1.0, 'int1': 0} last_expected_dict = { 'float1': -1.0, 'int1': 1000, 'int2_cond': 100, 'float2_cond': 100.0 } self.assertEqual(generated_grid[0].get_dictionary(), first_expected_dict) self.assertEqual(generated_grid[-1].get_dictionary(), last_expected_dict) # Here, we test that a few randomly chosen values in the generated grid # correspond to the ones I checked. self.assertEqual(generated_grid[3].get_dictionary()['int1'], 1000) self.assertEqual(generated_grid[12].get_dictionary()['cat1_cond'], 'orange') self.assertAlmostEqual( generated_grid[-2].get_dictionary()['float2_cond'], 31.622776601683803, places=3) # Sub-test 4 # Test: only a single hyperparameter and num_steps_dict is None cs = ConfigurationSpace(seed=1234) cs.add_hyperparameters([float1]) num_steps_dict = {'float1': 11} try: generated_grid = generate_grid(cs) except ValueError as e: assert str(e) == "num_steps_dict is None or doesn't contain " \ "the number of points to divide float1 into. And its quantization " \ "factor is None. Please provide/set one of these values." generated_grid = generate_grid(cs, num_steps_dict) self.assertEqual(len(generated_grid), 11) # Check 1st and last generated configurations completely: self.assertEqual(generated_grid[0].get_dictionary()['float1'], -1.0) self.assertEqual(generated_grid[-1].get_dictionary()['float1'], 1.0)