def test_get_hyperparameters_topological_sort(self): # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) hp2 = CategoricalHyperparameter("input2", [0, 1]) hp3 = CategoricalHyperparameter("input3", [0, 1]) hp4 = CategoricalHyperparameter("input4", [0, 1]) hp5 = CategoricalHyperparameter("input5", [0, 1]) hp6 = Constant("AND", "True") # More top-level hyperparameters hp7 = CategoricalHyperparameter("input7", [0, 1]) # Somewhat shuffled hyperparameters = [hp1, hp2, hp3, hp4, hp5, hp6, hp7] for hp in hyperparameters: cs.add_hyperparameter(hp) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) hps = cs.get_hyperparameters() # AND is moved to the front because of alphabetical sorting for hp, idx in zip(hyperparameters, [1, 2, 3, 4, 6, 0, 5]): self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name) cs.add_condition(cond5) hps = cs.get_hyperparameters() for hp, idx in zip(hyperparameters, [1, 2, 3, 6, 5, 0, 4]): self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name) cs.add_condition(conj3) hps = cs.get_hyperparameters() print(hps, hyperparameters) for hp, idx in zip(hyperparameters, [0, 1, 2, 5, 4, 6, 3]): print(hp, idx) self.assertEqual(hps.index(hp), idx) self.assertEqual(cs._hyperparameter_idx[hp.name], idx) self.assertEqual(cs._idx_to_hyperparameter[idx], hp.name)
def test_sample_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace(seed=1) hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp5, hp3, 1) cond5 = EqualsCondition(hp4, hp5, 1) cond6 = EqualsCondition(hp6, hp4, 1) cond7 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond6, cond7) cs.add_condition(cond4) cs.add_condition(cond5) cs.add_condition(conj3) samples = [] for i in range(5): cs.seed(1) samples.append([]) for j in range(100): sample = cs.sample_configuration() samples[-1].append(sample) if i > 0: for j in range(100): self.assertEqual(samples[-1][j], samples[-2][j])
def test_add_forbidden_illegal(self): cs = ConfigurationSpace() hp = CategoricalHyperparameter("input1", [0, 1]) forb = ForbiddenEqualsClause(hp, 1) self.assertRaisesRegex( ValueError, "Cannot add clause '%s'" % forb, cs.add_forbidden_clause, forb, ) forb2 = ForbiddenEqualsClause(hp, 0) self.assertRaisesRegex( ValueError, "Cannot add clause '%s'" % forb, cs.add_forbidden_clauses, [forb, forb2], )
def test_keys(self): # A regression test to make sure issue #49 does no longer pop up. By # iterating over the configuration in the for loop, it should not raise # a KeyError if the child hyperparameter is inactive. cs = ConfigurationSpace() shrinkage = CategoricalHyperparameter( "shrinkage", ["None", "auto", "manual"], default_value="None", ) shrinkage_factor = UniformFloatHyperparameter( "shrinkage_factor", 0., 1., 0.5, ) cs.add_hyperparameters([shrinkage, shrinkage_factor]) cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual")) for i in range(10): config = cs.sample_configuration() {hp_name: config[hp_name] for hp_name in config if config[hp_name] is not None}
def ising(n_eval, lamda, random_seed_pair): evaluator = Ising(lamda, random_seed_pair) name_tag = '_'.join([ 'ising', ('%.2E' % lamda), datetime.now().strftime("%Y-%m-%d-%H:%M:%S:%f") ]) cs = ConfigurationSpace() for i in range(ISING_N_EDGES): car_var = CategoricalHyperparameter('x' + str(i + 1).zfill(2), [str(elm) for elm in range(2)], default_value='0') cs.add_hyperparameter(car_var) init_points_numpy = evaluator.suggested_init.long().numpy() init_points = [] for i in range(init_points_numpy.shape[0]): init_points.append( Configuration( cs, { 'x' + str(j + 1).zfill(2): str(init_points_numpy[i][j]) for j in range(ISING_N_EDGES) })) def evaluate(x): x_tensor = torch.LongTensor( [int(x['x' + str(j + 1).zfill(2)]) for j in range(ISING_N_EDGES)]) return evaluator.evaluate(x_tensor).item() print('Began at ' + datetime.now().strftime("%H:%M:%S")) scenario = Scenario({ "run_obj": "quality", "runcount-limit": n_eval, "cs": cs, "deterministic": "true", 'output_dir': os.path.join(EXP_DIR, name_tag) }) smac = SMAC(scenario=scenario, tae_runner=evaluate, initial_configurations=init_points) smac.optimize() evaluations, optimum = evaluations_from_smac(smac) print('Finished at ' + datetime.now().strftime("%H:%M:%S")) return optimum
def test_condition_without_added_hyperparameters(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond = EqualsCondition(hp2, hp1, 0) self.assertRaisesRegex( ValueError, "Child hyperparameter 'child' not " "in configuration space.", cs.add_condition, cond) cs.add_hyperparameter(hp1) self.assertRaisesRegex( ValueError, "Child hyperparameter 'child' not " "in configuration space.", cs.add_condition, cond) # Test also the parent hyperparameter cs2 = ConfigurationSpace() cs2.add_hyperparameter(hp2) self.assertRaisesRegex( ValueError, "Parent hyperparameter 'parent' " "not in configuration space.", cs2.add_condition, cond)
def get_cash_cs(include_algorithms=None, task_type=CLASSIFICATION): _candidates = get_combined_candidtates(_classifiers, _addons) if include_algorithms is not None: _candidates = set(include_algorithms).intersection( set(_candidates.keys())) if len(_candidates) == 0: raise ValueError( "No algorithms included! Please check the spelling of the included algorithms!" ) cs = ConfigurationSpace() algo = CategoricalHyperparameter('algorithm', list(_candidates)) cs.add_hyperparameter(algo) for estimator_id in _candidates: estimator_cs = get_hpo_cs(estimator_id) parent_hyperparameter = {'parent': algo, 'value': estimator_id} cs.add_configuration_space(estimator_id, estimator_cs, parent_hyperparameter=parent_hyperparameter) return cs
def test_repr_roundtrip(self): cs = ConfigurationSpace() cs.add_hyperparameter( UniformIntegerHyperparameter("uihp", lower=1, upper=10)) cs.add_hyperparameter( NormalIntegerHyperparameter("nihp", mu=0, sigma=1)) cs.add_hyperparameter( UniformFloatHyperparameter("ufhp", lower=1, upper=10)) cs.add_hyperparameter(NormalFloatHyperparameter("nfhp", mu=0, sigma=1)) cs.add_hyperparameter( CategoricalHyperparameter("chp", choices=['1', '2', '3'])) cs.add_hyperparameter( OrdinalHyperparameter("ohp", sequence=['1', '2', '3'])) cs.add_hyperparameter(Constant("const", value=1)) default = cs.get_default_configuration() repr = default.__repr__() repr = repr.replace('})', '}, configuration_space=cs)') config = eval(repr) self.assertEqual(default, config)
def test_add_configuration_space(self): cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter( "input1", [0, 1])) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) hp2 = cs.add_hyperparameter( UniformIntegerHyperparameter("child", 0, 10)) cs.add_condition(EqualsCondition(hp2, hp1, 0)) cs2 = ConfigurationSpace() cs2.add_configuration_space('prefix', cs, delimiter='__') self.assertEqual( str(cs2), '''Configuration space object: Hyperparameters: prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5 prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0 Conditions: prefix__child | prefix__input1 == 0 Forbidden Clauses: Forbidden: prefix__input1 == 1 ''')
def test_get_parent_and_children_of(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) self.assertEqual([hp1], cs.get_parents_of(hp2.name)) self.assertEqual([hp1], cs.get_parents_of(hp2)) self.assertEqual([hp2], cs.get_children_of(hp1.name)) self.assertEqual([hp2], cs.get_children_of(hp1)) self.assertRaisesRegex( KeyError, "Hyperparameter 'Foo' does not exist in this " "configuration space.", cs.get_parents_of, "Foo") self.assertRaisesRegex( KeyError, "Hyperparameter 'Foo' does not exist in this " "configuration space.", cs.get_children_of, "Foo")
def test_repr(self): cs1 = ConfigurationSpace() retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n", retval) hp1 = CategoricalHyperparameter("parent", [0, 1]) cs1.add_hyperparameter(hp1) retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n" " %s\n" % str(hp1), retval) hp2 = UniformIntegerHyperparameter("child", 0, 10) cond1 = EqualsCondition(hp2, hp1, 0) cs1.add_hyperparameter(hp2) cs1.add_condition(cond1) retval = cs1.__str__() self.assertEqual("Configuration space object:\n Hyperparameters:\n" " %s\n %s\n Conditions:\n %s\n" % (str(hp2), str(hp1), str(cond1)), retval)
def test_with_ordinal(self): cs = smac.configspace.ConfigurationSpace() a = cs.add_hyperparameter(CategoricalHyperparameter('a', [0, 1], default_value=0)) b = cs.add_hyperparameter(OrdinalHyperparameter('b', [0, 1], default_value=1)) b = cs.add_hyperparameter(UniformFloatHyperparameter('c', lower=0., upper=1., default_value=1)) b = cs.add_hyperparameter(UniformIntegerHyperparameter('d', lower=0, upper=10, default_value=1)) cs.seed(1) feat_array = np.array([0, 0, 0]).reshape(1, -1) types, bounds = get_types(cs, feat_array) model = RandomForestWithInstances( configspace=cs, types=types, bounds=bounds, instance_features=feat_array, seed=1, ratio_features=1.0, pca_components=9, ) self.assertEqual(bounds[0][0], 2) self.assertTrue(bounds[0][1] is np.nan) self.assertEqual(bounds[1][0], 0) self.assertEqual(bounds[1][1], 1) self.assertEqual(bounds[2][0], 0.) self.assertEqual(bounds[2][1], 1.) self.assertEqual(bounds[3][0], 0.) self.assertEqual(bounds[3][1], 1.) X = np.array([ [0., 0., 0., 0., 0., 0., 0.], [0., 0., 1., 0., 0., 0., 0.], [0., 1., 0., 9., 0., 0., 0.], [0., 1., 1., 4., 0., 0., 0.]], dtype=np.float64) y = np.array([0, 1, 2, 3], dtype=np.float64) X_train = np.vstack((X, X, X, X, X, X, X, X, X, X)) y_train = np.vstack((y, y, y, y, y, y, y, y, y, y)) model.train(X_train, y_train.reshape((-1, 1))) mean, _ = model.predict(X) for idx, m in enumerate(mean): self.assertAlmostEqual(y[idx], m, 0.05)
def get_combined_cs(estimator_id, node, task_type=0): cs = ConfigurationSpace() hpo_cs = get_hpo_cs(estimator_id, task_type) fe_cs = get_fe_cs(estimator_id, node, task_type) config_cand = ['placeholder'] config_option = CategoricalHyperparameter('hpo', config_cand) cs.add_hyperparameter(config_option) for config_item in config_cand: sub_configuration_space = hpo_cs parent_hyperparameter = {'parent': config_option, 'value': config_item} cs.add_configuration_space(config_item, sub_configuration_space, parent_hyperparameter=parent_hyperparameter) for hp in fe_cs.get_hyperparameters(): cs.add_hyperparameter(hp) for cond in fe_cs.get_conditions(): cs.add_condition(cond) for bid in fe_cs.get_forbiddens(): cs.add_forbidden_clause(bid) model = UnParametrizedHyperparameter("estimator", estimator_id) cs.add_hyperparameter(model) return cs
def test_add_configuration_space_probabilities(self): from copy import copy, deepcopy from pickle import dumps, loads weights = [0.25, 0.5, 0.25] hp = CategoricalHyperparameter("B", ["1", "2", "3"], weights=weights) sub_cs = ConfigurationSpace() sub_cs.add_hyperparameter(hp) cs = ConfigurationSpace() cs.add_configuration_space("A", sub_cs) self.assertEqual( deepcopy(sub_cs).get_hyperparameter("B").probabilities, weights) self.assertEqual( copy(sub_cs).get_hyperparameter("B").probabilities, weights) self.assertEqual( loads(dumps(sub_cs)).get_hyperparameter("B").probabilities, weights) self.assertEqual(cs.get_hyperparameter("A:B").probabilities, weights) self.assertEqual( deepcopy(cs).get_hyperparameter("A:B").probabilities, weights) self.assertEqual( copy(cs).get_hyperparameter("A:B").probabilities, weights) self.assertEqual( loads(dumps(cs)).get_hyperparameter("A:B").probabilities, weights)
def test_eq(self): # Compare empty configuration spaces cs1 = ConfigurationSpace() cs2 = ConfigurationSpace() self.assertEqual(cs1, cs2) # Compare to something which isn't a configuration space self.assertTrue(not (cs1 == "ConfigurationSpace")) # Compare to equal configuration spaces hp1 = CategoricalHyperparameter("parent", [0, 1]) hp2 = UniformIntegerHyperparameter("child", 0, 10) hp3 = UniformIntegerHyperparameter("friend", 0, 5) cond1 = EqualsCondition(hp2, hp1, 0) cs1.add_hyperparameter(hp1) cs1.add_hyperparameter(hp2) cs1.add_condition(cond1) cs2.add_hyperparameter(hp1) cs2.add_hyperparameter(hp2) cs2.add_condition(cond1) self.assertEqual(cs1, cs2) cs1.add_hyperparameter(hp3) self.assertFalse(cs1 == cs2)
def get_config_space_from_dict(space_dict: dict): cs = ConfigurationSpace() params_dict = space_dict['parameters'] for key in params_dict: param_dict = params_dict[key] param_type = param_dict['type'] if param_type in ['float', 'int']: bound = param_dict['bound'] optional_args = dict() if 'default' in param_dict: optional_args['default_value'] = param_dict['default'] if 'log' in param_dict: optional_args['log'] = parse_bool(param_dict['log']) if 'q' in param_dict: optional_args['q'] = param_dict['q'] if param_type == 'float': param = UniformFloatHyperparameter(key, bound[0], bound[1], **optional_args) else: param = UniformIntegerHyperparameter(key, bound[0], bound[1], **optional_args) elif param_type == 'cat': choices = param_dict['choice'] optional_args = dict() if 'default' in param_dict: optional_args['default_value'] = param_dict['default'] param = CategoricalHyperparameter(key, choices, **optional_args) elif param_type == 'const': value = param_dict['value'] param = Constant(key, value) else: raise ValueError("Parameter type %s not supported!" % param_type) cs.add_hyperparameter(param) return cs
def get_combined_cs(node, task_type=0): cs = ConfigurationSpace() config_cand = [] cand_space = {} for estimator_id in _classifiers: cand_space[estimator_id] = get_hpo_cs(estimator_id, task_type) config_cand.append(estimator_id) config_option = CategoricalHyperparameter('estimator', config_cand) cs.add_hyperparameter(config_option) for config_item in config_cand: sub_configuration_space = cand_space[config_item] parent_hyperparameter = {'parent': config_option, 'value': config_item} cs.add_configuration_space(config_item, sub_configuration_space, parent_hyperparameter=parent_hyperparameter) fe_cs = get_fe_cs(estimator_id, node, task_type) for hp in fe_cs.get_hyperparameters(): cs.add_hyperparameter(hp) for cond in fe_cs.get_conditions(): cs.add_condition(cond) for bid in fe_cs.get_forbiddens(): cs.add_forbidden_clause(bid) return cs
def test_subspace_switches(self): # create a switch to select one of two algorithms algo_switch = CategoricalHyperparameter(name="switch", choices=["algo1", "algo2"], weights=[0.25, 0.75], default_value="algo1") # create sub-configuration space for algorithm 1 algo1_cs = ConfigurationSpace() hp1 = CategoricalHyperparameter(name="algo1_param1", choices=["A", "B"], weights=[0.3, 0.7], default_value="B") algo1_cs.add_hyperparameter(hp1) # create sub-configuration space for algorithm 2 algo2_cs = ConfigurationSpace() hp2 = CategoricalHyperparameter(name="algo2_param1", choices=["X", "Y"], default_value="Y") algo2_cs.add_hyperparameter(hp2) # create a configuration space and populate it with both the switch # and the two sub-configuration spaces cs = ConfigurationSpace() cs.add_hyperparameter(algo_switch) cs.add_configuration_space(prefix="algo1_subspace", configuration_space=algo1_cs, parent_hyperparameter={ 'parent': algo_switch, 'value': "algo1" }) cs.add_configuration_space(prefix="algo2_subspace", configuration_space=algo2_cs, parent_hyperparameter={ 'parent': algo_switch, 'value': "algo2" }) # check choices in the final configuration space self.assertEqual(("algo1", "algo2"), cs.get_hyperparameter("switch").choices) self.assertEqual( ("A", "B"), cs.get_hyperparameter("algo1_subspace:algo1_param1").choices) self.assertEqual( ("X", "Y"), cs.get_hyperparameter("algo2_subspace:algo2_param1").choices) # check probabilities in the final configuration space self.assertEqual((0.25, 0.75), cs.get_hyperparameter("switch").probabilities) self.assertEqual( (0.3, 0.7), cs.get_hyperparameter("algo1_subspace:algo1_param1").probabilities) self.assertEqual( None, cs.get_hyperparameter("algo2_subspace:algo2_param1").probabilities) # check default values in the final configuration space self.assertEqual("algo1", cs.get_hyperparameter("switch").default_value) self.assertEqual( "B", cs.get_hyperparameter("algo1_subspace:algo1_param1").default_value) self.assertEqual( "Y", cs.get_hyperparameter("algo2_subspace:algo2_param1").default_value)
def test_check_configuration(self): # TODO this is only a smoke test # TODO actually, this rather tests the evaluate methods in the # conditions module! cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("parent", [0, 1]) cs.add_hyperparameter(hp1) hp2 = UniformIntegerHyperparameter("child", 0, 10) cs.add_hyperparameter(hp2) cond1 = EqualsCondition(hp2, hp1, 0) cs.add_condition(cond1) # This automatically checks the configuration! Configuration(cs, dict(parent=0, child=5)) # and now for something more complicated cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) hp2 = CategoricalHyperparameter("input2", [0, 1]) cs.add_hyperparameter(hp2) hp3 = CategoricalHyperparameter("input3", [0, 1]) cs.add_hyperparameter(hp3) hp4 = CategoricalHyperparameter("input4", [0, 1]) cs.add_hyperparameter(hp4) hp5 = CategoricalHyperparameter("input5", [0, 1]) cs.add_hyperparameter(hp5) hp6 = Constant("AND", "True") cs.add_hyperparameter(hp6) cond1 = EqualsCondition(hp6, hp1, 1) cond2 = NotEqualsCondition(hp6, hp2, 1) cond3 = InCondition(hp6, hp3, [1]) cond4 = EqualsCondition(hp6, hp4, 1) cond5 = EqualsCondition(hp6, hp5, 1) conj1 = AndConjunction(cond1, cond2) conj2 = OrConjunction(conj1, cond3) conj3 = AndConjunction(conj2, cond4, cond5) cs.add_condition(conj3) expected_outcomes = [ False, False, False, False, False, False, False, True, False, False, False, False, False, False, False, True, False, False, False, True, False, False, False, True, False, False, False, False, False, False, False, True ] for idx, values in enumerate(product([0, 1], repeat=5)): # The hyperparameters aren't sorted, but the test assumes them to # be sorted. hyperparameters = sorted(cs.get_hyperparameters(), key=lambda t: t.name) instantiations = { hyperparameters[jdx + 1].name: values[jdx] for jdx in range(len(values)) } evaluation = conj3.evaluate(instantiations) self.assertEqual(expected_outcomes[idx], evaluation) if not evaluation: self.assertRaisesRegex( ValueError, r"Inactive hyperparameter 'AND' must " r"not be specified, but has the vector value: " r"'0.0'.", Configuration, cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True", }, ) else: Configuration( cs, values={ "input1": values[0], "input2": values[1], "input3": values[2], "input4": values[3], "input5": values[4], "AND": "True", }, )
def recursion(self, hdl: Dict, path=()) -> ConfigurationSpace: cs = ConfigurationSpace() # 检测一下这个dict是否在直接描述超参 key_list = list(hdl.keys()) if len(key_list) == 0: cs.add_hyperparameter(Constant("placeholder", "placeholder")) return cs else: sample_key = key_list[0] sample_value = hdl[sample_key] if is_hdl_bottom(sample_key, sample_value): store = {} conditions_dict = {} for key, value in hdl.items(): if purify_key(key).startswith("__"): conditions_dict[key] = value else: hp = self.__parse_dict_to_config(key, value) cs.add_hyperparameter(hp) store[key] = hp for key, value in conditions_dict.items(): if SERIES_CONNECT_LEADER_TOKEN in key: leader_model, condition_indicator = key.split( SERIES_CONNECT_LEADER_TOKEN) else: leader_model, condition_indicator = None, key if condition_indicator == "__condition": assert isinstance(value, list) for item in value: cond = self.__condition(item, store, leader_model) cs.add_condition(cond) elif condition_indicator == "__activate": self.__activate(value, store, cs, leader_model) elif condition_indicator == "__forbidden": self.__forbidden(value, store, cs, leader_model) elif condition_indicator == "__rely_model": RelyModels.info.append([value, deepcopy(path)]) return cs pattern = re.compile(r"(.*)\((.*)\)") for key, value in hdl.items(): mat = pattern.match(key) if mat: groups = mat.groups() assert len(groups) == 2 prefix_name, method = groups value_list = list(value.keys()) assert len(value_list) >= 1 if method == "choice": pass else: raise NotImplementedError() cur_cs = ConfigurationSpace() assert isinstance(value, dict) # 不能用constant,会报错 choice2proba = {} not_specific_proba_choices = [] sum_proba = 0 for k in value_list: v = value[k] if isinstance(v, dict) and "__proba" in v: proba = v.pop("__proba") choice2proba[k] = proba sum_proba += proba else: not_specific_proba_choices.append(k) if sum_proba <= 1: if len(not_specific_proba_choices) > 0: p_rest = (1 - sum_proba) / len(not_specific_proba_choices) for not_specific_proba_choice in not_specific_proba_choices: choice2proba[not_specific_proba_choice] = p_rest else: choice2proba = {k: 1 / len(value_list) for k in value_list} proba_list = [choice2proba[k] for k in value_list] value_list = list(map(smac_hdl._encode, value_list)) # choices must be str option_param = CategoricalHyperparameter( '__choice__', value_list, weights=proba_list) # todo : default cur_cs.add_hyperparameter(option_param) for sub_key, sub_value in value.items(): assert isinstance(sub_value, dict) sub_cs = self.recursion(sub_value, path=list(path) + [prefix_name, sub_key]) parent_hyperparameter = { 'parent': option_param, 'value': sub_key } cur_cs.add_configuration_space( sub_key, sub_cs, parent_hyperparameter=parent_hyperparameter) cs.add_configuration_space(prefix_name, cur_cs) elif isinstance(value, dict): sub_cs = self.recursion(value, path=list(path) + [key]) cs.add_configuration_space(key, sub_cs) else: raise NotImplementedError() return cs
from ConfigSpace import ConfigurationSpace, UniformFloatHyperparameter, CategoricalHyperparameter, \ UniformIntegerHyperparameter from ConfigSpace import EqualsCondition, InCondition, ForbiddenAndConjunction, ForbiddenEqualsClause, ForbiddenInClause import os import sys sys.path.append(os.getcwd()) cs1 = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformIntegerHyperparameter("x2", 1, 15, q=2.0, log=True) x3 = CategoricalHyperparameter("x3", ['x1', 'x2', 'x3']) x4 = CategoricalHyperparameter("x9", ['x1', 'x2', 'x3']) x5 = CategoricalHyperparameter("x5", ['x1', 'x2', 'x3']) x6 = CategoricalHyperparameter("x6", ['x1', 'x2', 'x3']) x7 = CategoricalHyperparameter("x7", ['x1', 'x3', 'x2'], default_value='x2') cs1.add_hyperparameters([x1, x2, x3, x7, x6, x5, x4]) cond1 = ForbiddenAndConjunction( ForbiddenEqualsClause(x3, "x1"), ForbiddenEqualsClause(x4, "x2"), ForbiddenEqualsClause(x5, "x3") ) cs1.add_forbidden_clause(cond1) cond2 = EqualsCondition(x1, x5, "x1") cond7 = EqualsCondition(x2, x6, "x1") cond3 = InCondition(x2, x6, ["x1", "x2", "x3"]) cs1.add_condition(cond3) cs1.add_condition(cond2)
# estimating:lightgbm:n_estimator, Type: Constant, Value: 100:int # preprocessing:0num->final:__choice__, Type: Categorical, Choices: {scale.standardize}, Default: scale.standardize # preprocessing:0num->final:scale.standardize:placeholder, Type: Constant, Value: placeholder # Conditions: # estimating:lightgbm:n_estimator | estimating:__choice__ == 'lightgbm' # preprocessing:0num->final:scale.standardize:placeholder | preprocessing:0num->final:__choice__ == 'scale.standardize' # scale.standardize standardize_cs = ConfigurationSpace() standardize_cs.add_hyperparameter(Constant("copy", "True:bool")) # scale.normalize normalize_cs = ConfigurationSpace() normalize_cs.add_hyperparameter(Constant("copy", "True:bool")) # scale scale_cs = ConfigurationSpace() scale_choice = CategoricalHyperparameter( '__choice__', ["scale.standardize", "scale.normalize"]) scale_cs.add_hyperparameter(scale_choice) scale_cs.add_configuration_space("scale.standardize", standardize_cs, parent_hyperparameter={ "parent": scale_choice, "value": "scale.standardize" }) scale_cs.add_configuration_space("scale.normalize", normalize_cs, parent_hyperparameter={ "parent": scale_choice, "value": "scale.normalize" }) # reduce.pca
import time import numpy as np from ConfigSpace import ConfigurationSpace, CategoricalHyperparameter, \ ForbiddenEqualsClause from ConfigSpace.io import pcs from ConfigSpace.util import get_one_exchange_neighbourhood #with open('/home/feurerm/projects/ConfigSpace/test/test_searchspaces/auto' # '-sklearn_2017_04.pcs') as fh: # cs = pcs.read(fh) cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter("hp1", [0, 1, 2, 3, 4, 5])) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 3)) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 5)) times = [] for i in range(20): start_time = time.time() configs = cs.sample_configuration(500000) end_time = time.time() times.append(end_time - start_time) print("all times:", times) print('Sampling 500000 configurations took on average:', np.mean(times))
def get_hyperparameter_search_space(dataset_properties=None): max_num_layers = 7 # Maximum number of layers coded # Hacky way to condition layers params based on the number of layers # 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer layer_choices = [ chr(i) for i in range(ord('c'), ord('b') + max_num_layers) ] batch_size = UniformIntegerHyperparameter("batch_size", 32, 4096, log=True, default=32) number_epochs = UniformIntegerHyperparameter("number_epochs", 2, 80, default=5) num_layers = CategoricalHyperparameter("num_layers", choices=layer_choices, default='c') lr = UniformFloatHyperparameter("learning_rate", 1e-6, 1.0, log=True, default=0.01) l2 = UniformFloatHyperparameter("lambda2", 1e-7, 1e-2, log=True, default=1e-4) dropout_output = UniformFloatHyperparameter("dropout_output", 0.0, 0.99, default=0.5) # Define basic hyperparameters and define the config space # basic means that are independent from the number of layers cs = ConfigurationSpace() cs.add_hyperparameter(number_epochs) cs.add_hyperparameter(batch_size) cs.add_hyperparameter(num_layers) cs.add_hyperparameter(lr) cs.add_hyperparameter(l2) cs.add_hyperparameter(dropout_output) # Define parameters with different child parameters and conditions solver_choices = [ "adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov", "smorm3s" ] solver = CategoricalHyperparameter(name="solver", choices=solver_choices, default="smorm3s") beta1 = UniformFloatHyperparameter("beta1", 1e-4, 0.1, log=True, default=0.1) beta2 = UniformFloatHyperparameter("beta2", 1e-4, 0.1, log=True, default=0.01) rho = UniformFloatHyperparameter("rho", 0.05, 0.99, log=True, default=0.95) momentum = UniformFloatHyperparameter("momentum", 0.3, 0.999, default=0.9) # TODO: Add policy based on this sklearn sgd policy_choices = ['fixed', 'inv', 'exp', 'step'] lr_policy = CategoricalHyperparameter(name="lr_policy", choices=policy_choices, default='fixed') gamma = UniformFloatHyperparameter(name="gamma", lower=1e-3, upper=1e-1, default=1e-2) power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5) epoch_step = UniformIntegerHyperparameter("epoch_step", 2, 20, default=5) cs.add_hyperparameter(solver) cs.add_hyperparameter(beta1) cs.add_hyperparameter(beta2) cs.add_hyperparameter(momentum) cs.add_hyperparameter(rho) cs.add_hyperparameter(lr_policy) cs.add_hyperparameter(gamma) cs.add_hyperparameter(power) cs.add_hyperparameter(epoch_step) # Define parameters that are needed it for each layer output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh'] activations_choices = [ 'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear' ] weight_choices = [ 'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform', 'he_normal', 'he_uniform', 'ortogonal', 'sparse' ] # Iterate over parameters that are used in each layer for i in range(1, max_num_layers): layer_units = UniformIntegerHyperparameter("num_units_layer_" + str(i), 64, 4096, log=True, default=128) cs.add_hyperparameter(layer_units) layer_dropout = UniformFloatHyperparameter("dropout_layer_" + str(i), 0.0, 0.99, default=0.5) cs.add_hyperparameter(layer_dropout) weight_initialization = CategoricalHyperparameter( 'weight_init_' + str(i), choices=weight_choices, default='he_normal') cs.add_hyperparameter(weight_initialization) layer_std = UniformFloatHyperparameter("std_layer_" + str(i), 1e-6, 0.1, log=True, default=0.005) cs.add_hyperparameter(layer_std) layer_activation = CategoricalHyperparameter( "activation_layer_" + str(i), choices=activations_choices, default="relu") cs.add_hyperparameter(layer_activation) layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' + str(i), 0.01, 0.99, default=0.3) cs.add_hyperparameter(layer_leakiness) layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' + str(i), 0.5, 1.0, default=2. / 3.) cs.add_hyperparameter(layer_tanh_alpha) layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' + str(i), 1.1, 3.0, log=True, default=1.7159) cs.add_hyperparameter(layer_tanh_beta) # TODO: Could be in a function in a new module for i in range(2, max_num_layers): # Condition layers parameter on layer choice layer_unit_param = cs.get_hyperparameter("num_units_layer_" + str(i)) layer_cond = InCondition(child=layer_unit_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition dropout parameter on layer choice layer_dropout_param = cs.get_hyperparameter("dropout_layer_" + str(i)) layer_cond = InCondition(child=layer_dropout_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition weight initialization on layer choice layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i)) layer_cond = InCondition(child=layer_weight_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition std parameter on weight layer initialization choice layer_std_param = cs.get_hyperparameter("std_layer_" + str(i)) weight_cond = EqualsCondition(child=layer_std_param, parent=layer_weight_param, value='normal') cs.add_condition(weight_cond) # Condition activation parameter on layer choice layer_activation_param = cs.get_hyperparameter( "activation_layer_" + str(i)) layer_cond = InCondition(child=layer_activation_param, parent=num_layers, values=[l for l in layer_choices[i - 1:]]) cs.add_condition(layer_cond) # Condition leakiness on activation choice layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_leakiness_param, parent=layer_activation_param, value='leaky') cs.add_condition(activation_cond) # Condition tanh on activation choice layer_tanh_alpha_param = cs.get_hyperparameter( "tanh_alpha_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_alpha_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" + str(i)) activation_cond = EqualsCondition(child=layer_tanh_beta_param, parent=layer_activation_param, value='scaledTanh') cs.add_condition(activation_cond) # Conditioning on solver momentum_depends_on_solver = InCondition( momentum, solver, values=["momentum", "nesterov"]) beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam") beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam") rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta") cs.add_condition(momentum_depends_on_solver) cs.add_condition(beta1_depends_on_solver) cs.add_condition(beta2_depends_on_solver) cs.add_condition(rho_depends_on_solver) # Conditioning on learning rate policy lr_policy_depends_on_solver = InCondition( lr_policy, solver, ["adadelta", "adagrad", "sgd", "momentum", "nesterov"]) gamma_depends_on_policy = InCondition(child=gamma, parent=lr_policy, values=["inv", "exp", "step"]) power_depends_on_policy = EqualsCondition(power, lr_policy, "inv") epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy, "step") cs.add_condition(lr_policy_depends_on_solver) cs.add_condition(gamma_depends_on_policy) cs.add_condition(power_depends_on_policy) cs.add_condition(epoch_step_depends_on_policy) return cs
def test_deactivate_inactive_hyperparameters(self): diamond = ConfigurationSpace() head = CategoricalHyperparameter('head', [0, 1]) left = CategoricalHyperparameter('left', [0, 1]) right = CategoricalHyperparameter('right', [0, 1]) bottom = CategoricalHyperparameter('bottom', [0, 1]) diamond.add_hyperparameters([head, left, right, bottom]) diamond.add_condition(EqualsCondition(left, head, 0)) diamond.add_condition(EqualsCondition(right, head, 0)) diamond.add_condition( AndConjunction(EqualsCondition(bottom, left, 0), EqualsCondition(bottom, right, 0))) c = deactivate_inactive_hyperparameters( { 'head': 0, 'left': 0, 'right': 0, 'bottom': 0 }, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters( { 'head': 1, 'left': 0, 'right': 0, 'bottom': 0 }, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters( { 'head': 0, 'left': 1, 'right': 0, 'bottom': 0 }, diamond) diamond._check_configuration_rigorous(c) diamond = ConfigurationSpace() head = CategoricalHyperparameter('head', [0, 1]) left = CategoricalHyperparameter('left', [0, 1]) right = CategoricalHyperparameter('right', [0, 1]) bottom = CategoricalHyperparameter('bottom', [0, 1]) diamond.add_hyperparameters([head, left, right, bottom]) diamond.add_condition(EqualsCondition(left, head, 0)) diamond.add_condition(EqualsCondition(right, head, 0)) diamond.add_condition( OrConjunction(EqualsCondition(bottom, left, 0), EqualsCondition(bottom, right, 0))) c = deactivate_inactive_hyperparameters( { 'head': 0, 'left': 0, 'right': 0, 'bottom': 0 }, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters( { 'head': 1, 'left': 1, 'right': 0, 'bottom': 0 }, diamond) diamond._check_configuration_rigorous(c) c = deactivate_inactive_hyperparameters( { 'head': 0, 'left': 1, 'right': 0, 'bottom': 0 }, diamond) diamond._check_configuration_rigorous(c) plain = ConfigurationSpace() a = UniformIntegerHyperparameter('a', 0, 10) b = UniformIntegerHyperparameter('b', 0, 10) plain.add_hyperparameters([a, b]) c = deactivate_inactive_hyperparameters({'a': 5, 'b': 6}, plain) plain.check_configuration(c)
def test_random_neighborhood_cat(self): hp = CategoricalHyperparameter('a', [5, 6, 7, 8]) self._test_random_neigbor(hp)
def test_random_neighbor_cat(self): hp = CategoricalHyperparameter('a', [5, 6, 7, 8]) all_neighbors = self._test_get_one_exchange_neighbourhood(hp) all_neighbors = [neighbor for neighbor in all_neighbors] self.assertEqual(len(all_neighbors), 300) # 3 (neighbors) * 100 (samples)
def setUp(self): cs = ConfigurationSpace() cs.add_hyperparameter(CategoricalHyperparameter("parent", [0, 1])) cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10)) cs.add_hyperparameter(UniformIntegerHyperparameter("friend", 0, 5)) self.cs = cs
def test_setitem(self): ''' Checks overriding a sampled configuration ''' pcs = ConfigurationSpace() pcs.add_hyperparameter( UniformIntegerHyperparameter('x0', 1, 5, default_value=1)) x1 = pcs.add_hyperparameter( CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab')) # Condition x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2])) pcs.add_condition(EqualsCondition(x2, x1, 'ab')) # Forbidden x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2])) pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2)) conf = pcs.get_default_configuration() # failed because it's a invalid configuration with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"): conf['x0'] = 0 # failed because the variable didn't exists with self.assertRaisesRegex( KeyError, "Hyperparameter 'x_0' does not exist in this configuration space.", ): conf['x_0'] = 1 # failed because forbidden clause is violated with self.assertRaisesRegex( ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2", ): conf['x3'] = 2 self.assertEqual(conf['x3'], 1) # successful operation 1 x0_old = conf['x0'] if x0_old == 1: conf['x0'] = 2 else: conf['x0'] = 1 x0_new = conf['x0'] self.assertNotEqual(x0_old, x0_new) pcs._check_configuration_rigorous(conf) self.assertEqual(conf['x2'], 1) # successful operation 2 x1_old = conf['x1'] if x1_old == 'ab': conf['x1'] = 'cd' else: conf['x1'] = 'ab' x1_new = conf['x1'] self.assertNotEqual(x1_old, x1_new) pcs._check_configuration_rigorous(conf) self.assertRaises(KeyError, conf.__getitem__, 'x2')
def get_categorical_param(name: str): return CategoricalHyperparameter(name, choices=["a", "b", "c"])