def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter(name="alpha", lower=0.01, upper=0.5, default_value=0.1) if dataset_properties is not None and dataset_properties.get('sparse'): choices = ['mutual_info_regression', 'f_regression'] else: choices = ['f_regression'] score_func = CategoricalHyperparameter(name="score_func", choices=choices, default_value="f_regression") mode = CategoricalHyperparameter('mode', ['fpr', 'fdr', 'fwe'], 'fpr') cs = ConfigurationSpace() cs.add_hyperparameter(alpha) cs.add_hyperparameter(score_func) cs.add_hyperparameter(mode) # Mutual info consistently crashes if percentile is not the mode if 'mutual_info_regression' in choices: cond = NotEqualsCondition(mode, score_func, 'mutual_info_regression') cs.add_condition(cond) return cs
def get_hyperparameter_search_space(dataset_properties=None): alpha = UniformFloatHyperparameter(name="alpha", lower=0.01, upper=0.5, default_value=0.1) if dataset_properties is not None and dataset_properties.get('sparse'): choices = ['chi2', 'mutual_info_classif'] else: choices = ['chi2', 'f_classif', 'mutual_info_classif'] score_func = CategoricalHyperparameter(name="score_func", choices=choices, default_value="chi2") mode = CategoricalHyperparameter('mode', ['fpr', 'fdr', 'fwe'], 'fpr') cs = ConfigurationSpace() cs.add_hyperparameter(alpha) cs.add_hyperparameter(score_func) cs.add_hyperparameter(mode) # mutual_info_classif constantly crashes if mode is not percentile # as a WA, fix the mode for this score cond = NotEqualsCondition(mode, score_func, 'mutual_info_classif') cs.add_condition(cond) return cs
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def test_get_hyperparameters_topological_sort(self): # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") # More top-level hyperparameters hp7 = CategoricalHyperparameter("input7", [0, 1]) # Somewhat shuffled hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7] for hp in hyperparameters: cs.add_hyperparameter(hp) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) hps = cs.get_hyperparameters() # AND is moved to the front because of alphabetical sorting for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]): self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name) cs.add_condition(cond5) hps = cs.get_hyperparameters() for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]): self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name) cs.add_condition(conj3) hps = cs.get_hyperparameters() # print(hps, hyperparameters) for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]): # print(hp, idx) self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_remove_inactive_parameter(): configuration_space = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("hp1", choices=[0, 1]) hp2 = CategoricalHyperparameter("hp2", choices=['a']) hp3 = UniformIntegerHyperparameter("hp3", lower=0, upper=5, default_value=5) configuration_space.add_hyperparameters([hp1, hp2, hp3]) # If hp1 = 0, then don't allow hp2 not_condition = NotEqualsCondition(hp2, hp1, 0) configuration_space.add_condition(not_condition) allowed_cfg = Configuration(configuration_space, { 'hp1': 1, 'hp2': 'a', 'hp3': 5 }) not_allowed = {'hp1': 0, 'hp2': 'a', 'hp3': 5} with pytest.raises(ValueError): Configuration(configuration_space, not_allowed) # No inactive hp - case: config is CS.configuration transformed = AbstractBenchmark._check_and_cast_configuration( allowed_cfg, configuration_space) assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5} # No inactive hp - case: config is dict transformed = AbstractBenchmark._check_and_cast_configuration( allowed_cfg.get_dictionary(), configuration_space) assert transformed.get_dictionary() == {'hp1': 1, 'hp2': 'a', 'hp3': 5} # Remove inactive: - case: config is CS.configuration not_allowed_cs = Configuration(configuration_space, { 'hp1': 0, 'hp2': 'a', 'hp3': 5 }, allow_inactive_with_values=True) transformed = AbstractBenchmark._check_and_cast_configuration( not_allowed_cs, configuration_space) assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5} # Remove inactive: - case: config is dict transformed = AbstractBenchmark._check_and_cast_configuration( not_allowed, configuration_space) assert transformed.get_dictionary() == {'hp1': 0, 'hp3': 5}
def test_check_configuration(self): # TODO this is only a smoke test # TODO actually, this rather tests the evaluate methods in the # conditions module! cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) cs.add_condition(conj3) expected_outcomes = [ False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True ] for idx, values in enumerate(product([0, 1], repeat=5)): # The hyperparameters aren't sorted, but the test assumes them to # be sorted. hyperparameters = sorted(cs.get_hyperparameters(), key=lambda t: t.name) instantiations = { hyperparameters[jdx + 1].name: values[jdx] for jdx in range(len(values)) } evaluation = conj3.evaluate(instantiations) self.assertEqual(expected_outcomes[idx], evaluation) if not evaluation: self.assertRaisesRegex( ValueError, r"Inactive hyperparameter 'AND' must " r"not be specified, but has the vector value: " r"'0.0'.", Configuration, cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True", }, ) else: Configuration( cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True", }, )