def setUp(self): def get_uniform_param(name: str): return UniformFloatHyperparameter(name, 0, 1) def get_constant_param(name: str): return Constant(name, 0.) def get_categorical_param(name: str): return CategoricalHyperparameter(name, choices=["a", "b", "c"]) def get_ordinal_param(name: str): return OrdinalHyperparameter(name, [8, 6, 4, 2]) get_params = [ get_uniform_param, get_constant_param, get_categorical_param, get_ordinal_param ] self.cs = ConfigurationSpace() for j, get_param in enumerate(get_params): param_name = f"x{j}" self.cs.add_hyperparameter(get_param(param_name)) param_constrained = CategoricalHyperparameter("constrained", choices=["a", "b", "c"]) self.cs.add_hyperparameter(param_constrained) self.cs.add_forbidden_clause( ForbiddenEqualsClause(param_constrained, "b")) for i in range(5): self.cs.add_hyperparameter( UniformFloatHyperparameter('x%d' % (i + len(get_params)), 0, 1))
def test_illegal_default_configuration(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("loss", ["l1", "l2"], default_value='l1') hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value='l1') cs.add_hyperparameter(hp1) cs.add_hyperparameter(hp2) forb1 = ForbiddenEqualsClause(hp1, "l1") forb2 = ForbiddenEqualsClause(hp2, "l1") forb3 = ForbiddenAndConjunction(forb1, forb2) # cs.add_forbidden_clause(forb3) self.assertRaisesRegexp( ValueError, "Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && " "Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def test_check_configuration2(self): # Test that hyperparameters which are not active must not be set and # that evaluating forbidden clauses does not choke on missing # hyperparameters cs = ConfigurationSpace() classifier = CategoricalHyperparameter( "classifier", ["k_nearest_neighbors", "extra_trees"]) metric = CategoricalHyperparameter("metric", ["minkowski", "other"]) p = CategoricalHyperparameter("k_nearest_neighbors:p", [1, 2]) metric_depends_on_classifier = EqualsCondition(metric, classifier, "k_nearest_neighbors") p_depends_on_metric = EqualsCondition(p, metric, "minkowski") cs.add_hyperparameter(metric) cs.add_hyperparameter(p) cs.add_hyperparameter(classifier) cs.add_condition(metric_depends_on_classifier) cs.add_condition(p_depends_on_metric) forbidden = ForbiddenEqualsClause(metric, "other") cs.add_forbidden_clause(forbidden) configuration = Configuration(cs, dict(classifier="extra_trees")) # check backward compatibility with checking configurations instead of vectors cs.check_configuration(configuration)
def test_setitem(self): ''' Checks overriding a sampled configuration ''' pcs = ConfigurationSpace() pcs.add_hyperparameter(UniformIntegerHyperparameter('x0', 1, 5, default_value=1)) x1 = pcs.add_hyperparameter( CategoricalHyperparameter('x1', ['ab', 'bc', 'cd', 'de'], default_value='ab') ) # Condition x2 = pcs.add_hyperparameter(CategoricalHyperparameter('x2', [1, 2])) pcs.add_condition(EqualsCondition(x2, x1, 'ab')) # Forbidden x3 = pcs.add_hyperparameter(CategoricalHyperparameter('x3', [1, 2])) pcs.add_forbidden_clause(ForbiddenEqualsClause(x3, 2)) conf = pcs.get_default_configuration() # failed because it's a invalid configuration with self.assertRaisesRegex(ValueError, "Illegal value '0' for hyperparameter x0"): conf['x0'] = 0 # failed because the variable didn't exists with self.assertRaisesRegex( KeyError, "Hyperparameter 'x_0' does not exist in this configuration space.", ): conf['x_0'] = 1 # failed because forbidden clause is violated with self.assertRaisesRegex( ForbiddenValueError, "Given vector violates forbidden clause Forbidden: x3 == 2", ): conf['x3'] = 2 self.assertEqual(conf['x3'], 1) # successful operation 1 x0_old = conf['x0'] if x0_old == 1: conf['x0'] = 2 else: conf['x0'] = 1 x0_new = conf['x0'] self.assertNotEqual(x0_old, x0_new) pcs._check_configuration_rigorous(conf) self.assertEqual(conf['x2'], 1) # successful operation 2 x1_old = conf['x1'] if x1_old == 'ab': conf['x1'] = 'cd' else: conf['x1'] = 'ab' x1_new = conf['x1'] self.assertNotEqual(x1_old, x1_new) pcs._check_configuration_rigorous(conf) self.assertRaises(KeyError, conf.__getitem__, 'x2')
class AgglomerativeClustering(object, metaclass=Metaclass): # static variables _name = "AgglomerativeClustering" _model = cluster.AgglomerativeClustering _params = [ UniformIntegerHyperparameter("n_clusters", 1, 80, default_value=10), CategoricalHyperparameter( "linkage", ['ward', 'complete', 'average', 'single'], default_value='complete'), CategoricalHyperparameter( "affinity", ['euclidean', 'cityblock', 'l2', 'l1', 'manhattan', 'cosine'], default_value='euclidean') # 'ward' has been included now # 'precomputed' has been excluded from "affinity" s possible values because it requires # a precomputed distance matrix as input from user ] _params_names = set([p.name for p in _params]) _conditions = [] _forbidden_clauses = [ ForbiddenAndConjunction( ForbiddenEqualsClause(_params[1], "ward"), ForbiddenInClause( _params[2], ['cosine', 'cityblock', 'l2', 'l1', 'manhattan'])) ]
def set_probabilities_in_cs(self, cs: ConfigurationSpace, relied2models: Dict[str, List[str]], relied2AllModels: Dict[str, List[str]], all_models: List[str], **kwargs): estimator = cs.get_hyperparameter(f"{PHASE2}:__choice__") probabilities = [] model2prob = {} L = 0 for rely_model in relied2models: cur_models = relied2models[rely_model] L += len(cur_models) for model in cur_models: model2prob[model] = kwargs[rely_model] / len(cur_models) p_rest = (1 - sum(model2prob.values())) / (len(all_models) - L) for model in estimator.choices: probabilities.append(model2prob.get(model, p_rest)) estimator.probabilities = probabilities default_estimator_choice = None for models in relied2models.values(): if models: default_estimator_choice = models[0] estimator.default_value = default_estimator_choice for rely_model, path in RelyModels.info: forbid_eq_value = path[-1] path = path[:-1] forbid_eq_key = ":".join(path + ["__choice__"]) forbid_eq_key_hp = cs.get_hyperparameter(forbid_eq_key) forbid_in_key = f"{PHASE2}:__choice__" hit = relied2AllModels.get(rely_model) if not hit: choices = list(forbid_eq_key_hp.choices) choices.remove(forbid_eq_value) forbid_eq_key_hp.choices = tuple(choices) forbid_eq_key_hp.default_value = choices[0] forbid_eq_key_hp.probabilities = [1 / len(choices) ] * len(choices) # fixme 最后我放弃了在这上面进行修改,在hdl部分就做了预处理 continue forbid_in_value = list(set(all_models) - set(hit)) # 只选择了boost模型 if not forbid_in_value: continue choices = forbid_eq_key_hp.choices probabilities = [] p: float = kwargs[rely_model] p_rest = (1 - p) * (len(choices) - 1) for choice in choices: if choice == forbid_eq_value: probabilities.append(p) else: probabilities.append(p_rest) forbid_eq_key_hp.probabilities = probabilities cs.add_forbidden_clause( ForbiddenAndConjunction( ForbiddenEqualsClause(forbid_eq_key_hp, forbid_eq_value), ForbiddenInClause(cs.get_hyperparameter(forbid_in_key), forbid_in_value), ))
def test_add_forbidden_illegal(self): cs = ConfigurationSpace() hp = CategoricalHyperparameter("input1", [0, 1]) forb = ForbiddenEqualsClause(hp, 1) self.assertRaisesRegex( ValueError, "Cannot add clause '%s'" % forb, cs.add_forbidden_clause, forb, ) forb2 = ForbiddenEqualsClause(hp, 0) self.assertRaisesRegex( ValueError, "Cannot add clause '%s'" % forb, cs.add_forbidden_clauses, [forb, forb2], )
def test_check_forbidden_with_sampled_vector_configuration(self): cs = ConfigurationSpace() metric = CategoricalHyperparameter("metric", ["minkowski", "other"]) cs.add_hyperparameter(metric) forbidden = ForbiddenEqualsClause(metric, "other") cs.add_forbidden_clause(forbidden) configuration = Configuration(cs, vector=np.ones(1, dtype=float)) self.assertRaisesRegex(ValueError, "violates forbidden clause", cs._check_forbidden, configuration.get_array())
def get_configspace(self, optimizer='smac'): if optimizer == 'smac': cs = ConfigurationSpace() penalty = CategoricalHyperparameter( "penalty", ["l1", "l2"], default_value="l2") loss = CategoricalHyperparameter( "loss", ["hinge", "squared_hinge"], default_value="squared_hinge") dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True') # This is set ad-hoc tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) C = UniformFloatHyperparameter( "C", 0.03125, 32768, log=True, default_value=1.0) multi_class = Constant("multi_class", "ovr") # These are set ad-hoc fit_intercept = Constant("fit_intercept", "True") intercept_scaling = Constant("intercept_scaling", 1) cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class, fit_intercept, intercept_scaling]) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge") ) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge") ) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "True"), ForbiddenEqualsClause(penalty, "l1") ) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) return cs elif optimizer == 'tpe': from hyperopt import hp space = {'penalty': hp.choice('liblinear_combination', [{'penalty': "l1", 'loss': "squared_hinge", 'dual': "False"}, {'penalty': "l2", 'loss': "hinge", 'dual': "True"}, {'penalty': "l2", 'loss': "squared_hinge", 'dual': "True"}, {'penalty': "l2", 'loss': "squared_hinge", 'dual': "False"}]), 'loss': None, 'dual': None, 'tol': hp.loguniform('liblinear_tol', np.log(1e-5), np.log(1e-1)), 'C': hp.loguniform('liblinear_C', np.log(0.03125), np.log(32768)), 'multi_class': hp.choice('liblinear_multi_class', ["ovr"]), 'fit_intercept': hp.choice('liblinear_fit_intercept', ["True"]), 'intercept_scaling': hp.choice('liblinear_intercept_scaling', [1])} return space else: raise ValueError('Unknown optimizer %s when getting configspace' % optimizer)
def get_hyperparameter_search_space(**kwargs): cs = ConfigurationSpace() penalty = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value="l2") loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default_value="squared_hinge") dual = CategoricalHyperparameter("dual", [True, False], default_value=True) tol = UniformFloatHyperparameter("tol", 1e-5, 120., default_value=1e-4) C = UniformFloatHyperparameter("C", 1e-7, 100., default_value=1.) multi_class = CategoricalHyperparameter("multi_class", ["ovr", "crammer_singer"], default_value="ovr") fit_intercept = CategoricalHyperparameter("fit_intercept", [True, False], default_value=True) intercept_scaling = UniformFloatHyperparameter("intercept_scaling", 0., 1., default_value=1.) max_iter = UniformIntegerHyperparameter("max_iter", 100, 2000, default_value=1000) cs.add_hyperparameters( [C, penalty, loss, dual, tol, multi_class, fit_intercept, intercept_scaling, max_iter]) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge") ) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, False), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge") ) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, False), ForbiddenEqualsClause(penalty, "l1") ) constant_penalty_and_loss2 = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, True), ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "squared_hinge") ) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) cs.add_forbidden_clause(constant_penalty_and_loss2) return cs
def test_add_forbidden_clause(self): cs = ConfigurationSpace() hp1 = CategoricalHyperparameter("input1", [0, 1]) cs.add_hyperparameter(hp1) forb = ForbiddenEqualsClause(hp1, 1) # TODO add checking whether a forbidden clause makes sense at all cs.add_forbidden_clause(forb) # TODO add something to properly retrieve the forbidden clauses self.assertEqual(str(cs), "Configuration space object:\n " "Hyperparameters:\n input1, " "Type: Categorical, Choices: {0, 1}, " "Default: 0\n" " Forbidden Clauses:\n" " Forbidden: input1 == 1\n")
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace): assert isinstance(value, list) for item in value: assert isinstance(item, dict) clauses = [] for k, v in item.items(): if isinstance(v, list) and len(v) == 1: v = v[0] if isinstance(v, list): clauses.append( ForbiddenInClause(store[k], list(map(smac_hdl._encode, v)))) else: clauses.append( ForbiddenEqualsClause(store[k], smac_hdl._encode(v))) cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def string2forbidden_base(base_forbid_desc: str, hp_dict: dict): pattern_equal = r'[\s(]*Forbidden:\s(.*?)\s==\s(.*)' pattern_in = r'[\s(]*Forbidden:\s(.*?)\sin\s(.*)?}' matchobj_equal = re.match(pattern_equal, base_forbid_desc) matchobj_in = re.match(pattern_in, base_forbid_desc) if matchobj_equal: forbid_name = matchobj_equal.group(1) target_value = matchobj_equal.group(2)[1:-1] forbid = ForbiddenEqualsClause(hp_dict[forbid_name], target_value) elif matchobj_in: forbid_name = matchobj_in.group(1) choice_str = matchobj_in.group(2).split(',') choices = [choice[2:-1] for choice in choice_str] forbid = ForbiddenInClause(hp_dict[forbid_name], choices) else: raise ValueError("Unsupported forbidden type in config_space!") return forbid
def test_add_configuration_space(self): cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter("input1", [0, 1])) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) hp2 = cs.add_hyperparameter(UniformIntegerHyperparameter("child", 0, 10)) cs.add_condition(EqualsCondition(hp2, hp1, 0)) cs2 = ConfigurationSpace() cs2.add_configuration_space('prefix', cs, delimiter='__') self.assertEqual(str(cs2), '''Configuration space object: Hyperparameters: prefix__child, Type: UniformInteger, Range: [0, 10], Default: 5 prefix__input1, Type: Categorical, Choices: {0, 1}, Default: 0 Conditions: prefix__child | prefix__input1 == 0 Forbidden Clauses: Forbidden: prefix__input1 == 1 ''')
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace): assert isinstance(value, list) for item in value: assert isinstance(item, dict) clauses = [] for name, forbidden_values in item.items(): if isinstance(forbidden_values, list) and len(forbidden_values) == 1: forbidden_values = forbidden_values[0] if isinstance(forbidden_values, list): clauses.append( ForbiddenInClause( store[name], list(map(hp_def._encode, forbidden_values)))) else: clauses.append( ForbiddenEqualsClause( store[name], hp_def._encode(forbidden_values))) cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace, leader_model): assert isinstance(value, list) for item in value: assert isinstance(item, dict) clauses = [] for name, forbidden_values in item.items(): true_name = add_leader_model(name, leader_model, SERIES_CONNECT_LEADER_TOKEN) if isinstance(forbidden_values, list) and len(forbidden_values) == 1: forbidden_values = forbidden_values[0] if isinstance(forbidden_values, list): clauses.append( ForbiddenInClause( store[true_name], list(map(smac_hdl._encode, forbidden_values)))) else: clauses.append( ForbiddenEqualsClause( store[true_name], smac_hdl._encode(forbidden_values))) cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def get_hyperparameter_search_space(): cs = ConfigurationSpace() penalty = CategoricalHyperparameter( "penalty", ["l1", "l2"], default_value="l2") loss = CategoricalHyperparameter( "loss", ["hinge", "squared_hinge"], default_value="squared_hinge") dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True') # This is set ad-hoc tol = UniformFloatHyperparameter( "tol", 1e-5, 1e-1, default_value=1e-4, log=True) C = UniformFloatHyperparameter( "C", 0.03125, 32768, log=True, default_value=1.0) multi_class = UnParametrizedHyperparameter("multi_class", "ovr") # These are set ad-hoc fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True") intercept_scaling = UnParametrizedHyperparameter("intercept_scaling", 1) cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class, fit_intercept, intercept_scaling]) penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(penalty, "l1"), ForbiddenEqualsClause(loss, "hinge") ) constant_penalty_and_loss = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "False"), ForbiddenEqualsClause(penalty, "l2"), ForbiddenEqualsClause(loss, "hinge") ) penalty_and_dual = ForbiddenAndConjunction( ForbiddenEqualsClause(dual, "True"), ForbiddenEqualsClause(penalty, "l1") ) cs.add_forbidden_clause(penalty_and_loss) cs.add_forbidden_clause(constant_penalty_and_loss) cs.add_forbidden_clause(penalty_and_dual) return cs
import os import sys sys.path.append(os.getcwd()) cs1 = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformIntegerHyperparameter("x2", 1, 15, q=2.0, log=True) x3 = CategoricalHyperparameter("x3", ['x1', 'x2', 'x3']) x4 = CategoricalHyperparameter("x9", ['x1', 'x2', 'x3']) x5 = CategoricalHyperparameter("x5", ['x1', 'x2', 'x3']) x6 = CategoricalHyperparameter("x6", ['x1', 'x2', 'x3']) x7 = CategoricalHyperparameter("x7", ['x1', 'x3', 'x2'], default_value='x2') cs1.add_hyperparameters([x1, x2, x3, x7, x6, x5, x4]) cond1 = ForbiddenAndConjunction(ForbiddenEqualsClause(x3, "x1"), ForbiddenEqualsClause(x4, "x2"), ForbiddenEqualsClause(x5, "x3")) cs1.add_forbidden_clause(cond1) cond2 = EqualsCondition(x1, x5, "x1") cond7 = EqualsCondition(x2, x6, "x1") cond3 = InCondition(x2, x6, ["x1", "x2", "x3"]) cs1.add_condition(cond3) cs1.add_condition(cond2) cond4 = ForbiddenEqualsClause(x4, 'x3') cond5 = ForbiddenInClause(x7, ['x1', 'x3']) cs1.add_forbidden_clause(cond5) cs1.add_forbidden_clause(cond4)
import numpy as np from ConfigSpace import ConfigurationSpace, CategoricalHyperparameter, \ ForbiddenEqualsClause from ConfigSpace.io import pcs from ConfigSpace.util import get_one_exchange_neighbourhood #with open('/home/feurerm/projects/ConfigSpace/test/test_searchspaces/auto' # '-sklearn_2017_04.pcs') as fh: # cs = pcs.read(fh) cs = ConfigurationSpace() hp1 = cs.add_hyperparameter(CategoricalHyperparameter("hp1", [0, 1, 2, 3, 4, 5])) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 1)) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 3)) cs.add_forbidden_clause(ForbiddenEqualsClause(hp1, 5)) times = [] for i in range(20): start_time = time.time() configs = cs.sample_configuration(500000) end_time = time.time() times.append(end_time - start_time) print("all times:", times) print('Sampling 500000 configurations took on average:', np.mean(times)) times = []
import sys sys.path.append(os.getcwd()) cs1 = ConfigurationSpace() x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0) x2 = UniformIntegerHyperparameter("x2", 1, 15, q=2.0, log=True) x3 = CategoricalHyperparameter("x3", ['x1', 'x2', 'x3']) x4 = CategoricalHyperparameter("x9", ['x1', 'x2', 'x3']) x5 = CategoricalHyperparameter("x5", ['x1', 'x2', 'x3']) x6 = CategoricalHyperparameter("x6", ['x1', 'x2', 'x3']) x7 = CategoricalHyperparameter("x7", ['x1', 'x3', 'x2'], default_value='x2') cs1.add_hyperparameters([x1, x2, x3, x7, x6, x5, x4]) cond1 = ForbiddenAndConjunction( ForbiddenEqualsClause(x3, "x1"), ForbiddenEqualsClause(x4, "x2"), ForbiddenEqualsClause(x5, "x3") ) cs1.add_forbidden_clause(cond1) cond2 = EqualsCondition(x1, x5, "x1") cond7 = EqualsCondition(x2, x6, "x1") cond3 = InCondition(x2, x6, ["x1", "x2", "x3"]) cs1.add_condition(cond3) cs1.add_condition(cond2) cond4 = ForbiddenEqualsClause(x4, 'x3') cond5 = ForbiddenInClause(x7, ['x1', 'x3']) cs1.add_forbidden_clause(cond5) cs1.add_forbidden_clause(cond4)