def get_configspace(self, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2"], default_value="l2")
loss = CategoricalHyperparameter(
"loss", ["hinge", "squared_hinge"], default_value="squared_hinge")
dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True')
# This is set ad-hoc
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
C = UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default_value=1.0)
multi_class = Constant("multi_class", "ovr")
# These are set ad-hoc
fit_intercept = Constant("fit_intercept", "True")
intercept_scaling = Constant("intercept_scaling", 1)
cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class,
fit_intercept, intercept_scaling])
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge")
)
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge")
)
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "True"),
ForbiddenEqualsClause(penalty, "l1")
)
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'penalty': hp.choice('liblinear_combination',
[{'penalty': "l1", 'loss': "squared_hinge", 'dual': "False"},
{'penalty': "l2", 'loss': "hinge", 'dual': "True"},
{'penalty': "l2", 'loss': "squared_hinge", 'dual': "True"},
{'penalty': "l2", 'loss': "squared_hinge", 'dual': "False"}]),
'loss': None,
'dual': None,
'tol': hp.loguniform('liblinear_tol', np.log(1e-5), np.log(1e-1)),
'C': hp.loguniform('liblinear_C', np.log(0.03125), np.log(32768)),
'multi_class': hp.choice('liblinear_multi_class', ["ovr"]),
'fit_intercept': hp.choice('liblinear_fit_intercept', ["True"]),
'intercept_scaling': hp.choice('liblinear_intercept_scaling', [1])}
return space
else:
raise ValueError('Unknown optimizer %s when getting configspace' % optimizer)
class AgglomerativeClustering(object, metaclass=Metaclass):
# static variables
_name = "AgglomerativeClustering"
_model = cluster.AgglomerativeClustering
_params = [
UniformIntegerHyperparameter("n_clusters", 1, 80,
default_value=10),
CategoricalHyperparameter(
"linkage", ['ward', 'complete', 'average', 'single'],
default_value='complete'),
CategoricalHyperparameter(
"affinity",
['euclidean', 'cityblock', 'l2', 'l1', 'manhattan', 'cosine'],
default_value='euclidean')
# 'ward' has been included now
# 'precomputed' has been excluded from "affinity" s possible values because it requires
# a precomputed distance matrix as input from user
]
_params_names = set([p.name for p in _params])
_conditions = []
_forbidden_clauses = [
ForbiddenAndConjunction(
ForbiddenEqualsClause(_params[1], "ward"),
ForbiddenInClause(
_params[2],
['cosine', 'cityblock', 'l2', 'l1', 'manhattan']))
]
def string2forbidden(forbid_desc: str, hp_dict: dict):
def string2forbidden_base(base_forbid_desc: str, hp_dict: dict):
pattern_equal = r'[\s(]*Forbidden:\s(.*?)\s==\s(.*)'
pattern_in = r'[\s(]*Forbidden:\s(.*?)\sin\s(.*)?}'
matchobj_equal = re.match(pattern_equal, base_forbid_desc)
matchobj_in = re.match(pattern_in, base_forbid_desc)
if matchobj_equal:
forbid_name = matchobj_equal.group(1)
target_value = matchobj_equal.group(2)[1:-1]
forbid = ForbiddenEqualsClause(hp_dict[forbid_name], target_value)
elif matchobj_in:
forbid_name = matchobj_in.group(1)
choice_str = matchobj_in.group(2).split(',')
choices = [choice[2:-1] for choice in choice_str]
forbid = ForbiddenInClause(hp_dict[forbid_name], choices)
else:
raise ValueError("Unsupported forbidden type in config_space!")
return forbid
forbidden_strlist = forbid_desc.split('&&')
if len(forbidden_strlist) == 1:
return string2forbidden_base(forbid_desc, hp_dict)
else:
forbiddden_list = [string2forbidden_base(split_forbidden[:-1], hp_dict) for split_forbidden in
forbidden_strlist]
return ForbiddenAndConjunction(*forbiddden_list)
def set_probabilities_in_cs(self, cs: ConfigurationSpace,
relied2models: Dict[str, List[str]],
relied2AllModels: Dict[str, List[str]],
all_models: List[str], **kwargs):
estimator = cs.get_hyperparameter(f"{PHASE2}:__choice__")
probabilities = []
model2prob = {}
L = 0
for rely_model in relied2models:
cur_models = relied2models[rely_model]
L += len(cur_models)
for model in cur_models:
model2prob[model] = kwargs[rely_model] / len(cur_models)
p_rest = (1 - sum(model2prob.values())) / (len(all_models) - L)
for model in estimator.choices:
probabilities.append(model2prob.get(model, p_rest))
estimator.probabilities = probabilities
default_estimator_choice = None
for models in relied2models.values():
if models:
default_estimator_choice = models[0]
estimator.default_value = default_estimator_choice
for rely_model, path in RelyModels.info:
forbid_eq_value = path[-1]
path = path[:-1]
forbid_eq_key = ":".join(path + ["__choice__"])
forbid_eq_key_hp = cs.get_hyperparameter(forbid_eq_key)
forbid_in_key = f"{PHASE2}:__choice__"
hit = relied2AllModels.get(rely_model)
if not hit:
choices = list(forbid_eq_key_hp.choices)
choices.remove(forbid_eq_value)
forbid_eq_key_hp.choices = tuple(choices)
forbid_eq_key_hp.default_value = choices[0]
forbid_eq_key_hp.probabilities = [1 / len(choices)
] * len(choices)
# fixme 最后我放弃了在这上面进行修改，在hdl部分就做了预处理
continue
forbid_in_value = list(set(all_models) - set(hit))
# 只选择了boost模型
if not forbid_in_value:
continue
choices = forbid_eq_key_hp.choices
probabilities = []
p: float = kwargs[rely_model]
p_rest = (1 - p) * (len(choices) - 1)
for choice in choices:
if choice == forbid_eq_value:
probabilities.append(p)
else:
probabilities.append(p_rest)
forbid_eq_key_hp.probabilities = probabilities
cs.add_forbidden_clause(
ForbiddenAndConjunction(
ForbiddenEqualsClause(forbid_eq_key_hp, forbid_eq_value),
ForbiddenInClause(cs.get_hyperparameter(forbid_in_key),
forbid_in_value),
))
def test_illegal_default_configuration(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("loss", ["l1", "l2"], default_value='l1')
hp2 = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value='l1')
cs.add_hyperparameter(hp1)
cs.add_hyperparameter(hp2)
forb1 = ForbiddenEqualsClause(hp1, "l1")
forb2 = ForbiddenEqualsClause(hp2, "l1")
forb3 = ForbiddenAndConjunction(forb1, forb2)
# cs.add_forbidden_clause(forb3)
self.assertRaisesRegexp(ValueError, "Given vector violates forbidden clause \(Forbidden: loss == \'l1\' && "
"Forbidden: penalty == \'l1\'\)", cs.add_forbidden_clause, forb3)
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter("penalty", ["l1", "l2"], default_value="l2")
loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"], default_value="squared_hinge")
dual = CategoricalHyperparameter("dual", [True, False], default_value=True)
tol = UniformFloatHyperparameter("tol", 1e-5, 120., default_value=1e-4)
C = UniformFloatHyperparameter("C", 1e-7, 100., default_value=1.)
multi_class = CategoricalHyperparameter("multi_class", ["ovr", "crammer_singer"], default_value="ovr")
fit_intercept = CategoricalHyperparameter("fit_intercept", [True, False], default_value=True)
intercept_scaling = UniformFloatHyperparameter("intercept_scaling", 0., 1., default_value=1.)
max_iter = UniformIntegerHyperparameter("max_iter", 100, 2000, default_value=1000)
cs.add_hyperparameters(
[C, penalty, loss, dual, tol, multi_class, fit_intercept, intercept_scaling, max_iter])
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge")
)
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, False),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge")
)
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, False),
ForbiddenEqualsClause(penalty, "l1")
)
constant_penalty_and_loss2 = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, True),
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "squared_hinge")
)
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
cs.add_forbidden_clause(constant_penalty_and_loss2)
return cs
def get_hyperparameter_search_space():
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2"], default_value="l2")
loss = CategoricalHyperparameter(
"loss", ["hinge", "squared_hinge"], default_value="squared_hinge")
dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True')
# This is set ad-hoc
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
C = UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default_value=1.0)
multi_class = UnParametrizedHyperparameter("multi_class", "ovr")
# These are set ad-hoc
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
intercept_scaling = UnParametrizedHyperparameter("intercept_scaling", 1)
cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class,
fit_intercept, intercept_scaling])
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge")
)
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge")
)
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "True"),
ForbiddenEqualsClause(penalty, "l1")
)
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
return cs
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace):
assert isinstance(value, list)
for item in value:
assert isinstance(item, dict)
clauses = []
for k, v in item.items():
if isinstance(v, list) and len(v) == 1:
v = v[0]
if isinstance(v, list):
clauses.append(
ForbiddenInClause(store[k],
list(map(smac_hdl._encode, v))))
else:
clauses.append(
ForbiddenEqualsClause(store[k], smac_hdl._encode(v)))
cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter("penalty", ["l1", "l2", "elasticnet", "none"], default_value='l2')
solver = CategoricalHyperparameter("solver", ["newton-cg", "lbfgs", "liblinear", "sag", "saga"],
default_value="lbfgs")
dual = CategoricalHyperparameter("dual", choices=[True, False], default_value=False)
tol = UniformFloatHyperparameter("tol", lower=1e-7, upper=100., default_value=1.0e-4, log=True)
C = UniformFloatHyperparameter("C", lower=1e-7, upper=100., default_value=1.0, log=True)
fit_intercept = CategoricalHyperparameter("fit_intercept", choices=[True, False], default_value=True)
intercept_scaling = UniformFloatHyperparameter("intercept_scaling", lower=0.0001, upper=2.0, default_value=1.0,
log=True)
max_iter = UniformIntegerHyperparameter("max_iter", lower=50, upper=10000, default_value=100)
multi_class = CategoricalHyperparameter("multi_class", ["ovr", "multinomial", "auto"], default_value="auto")
l1_ratio = UniformFloatHyperparameter("l1_ratio", lower=0., upper=1., default_value=0.1)
l1_ratio_condition = InCondition(l1_ratio, penalty, ["elasticnet"])
dual_condition = AndConjunction(InCondition(dual, penalty, ["l2"]), InCondition(dual, solver, ["liblinear"]))
cs.add_hyperparameters([penalty, solver, dual, tol, C, fit_intercept, intercept_scaling, max_iter, multi_class,
l1_ratio])
penaltyAndLbfgs = ForbiddenAndConjunction(
ForbiddenEqualsClause(solver, "lbfgs"),
ForbiddenInClause(penalty, ["l1", "elasticnet"])
)
penaltyAndNewton = ForbiddenAndConjunction(
ForbiddenEqualsClause(solver, "newton-cg"),
ForbiddenInClause(penalty, ["l1", "elasticnet"])
)
penaltyAndSag = ForbiddenAndConjunction(
ForbiddenEqualsClause(solver, "sag"),
ForbiddenInClause(penalty, ["l1", "elasticnet"])
)
penaltyAndSaga = ForbiddenAndConjunction(
ForbiddenInClause(penalty, ["elasticnet"]),
ForbiddenInClause(solver, ["newton-cg", "lbfgs", "sag"])
)
penaltyAndSagaa = ForbiddenAndConjunction(
ForbiddenInClause(penalty, ["elasticnet", "none"]),
ForbiddenInClause(solver, ["liblinear"])
)
penaltyAndSagaaa = ForbiddenAndConjunction(
ForbiddenInClause(multi_class, ["multinomial"]),
ForbiddenInClause(solver, ["liblinear"])
)
cs.add_forbidden_clause(penaltyAndLbfgs)
cs.add_forbidden_clause(penaltyAndNewton)
cs.add_forbidden_clause(penaltyAndSag)
cs.add_forbidden_clause(penaltyAndSagaa)
cs.add_forbidden_clause(penaltyAndSaga)
cs.add_forbidden_clause(penaltyAndSagaaa)
cs.add_condition(l1_ratio_condition)
cs.add_condition(dual_condition)
return cs
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace):
assert isinstance(value, list)
for item in value:
assert isinstance(item, dict)
clauses = []
for name, forbidden_values in item.items():
if isinstance(forbidden_values,
list) and len(forbidden_values) == 1:
forbidden_values = forbidden_values[0]
if isinstance(forbidden_values, list):
clauses.append(
ForbiddenInClause(
store[name],
list(map(hp_def._encode, forbidden_values))))
else:
clauses.append(
ForbiddenEqualsClause(
store[name], hp_def._encode(forbidden_values)))
cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
def __forbidden(self, value: List, store: Dict, cs: ConfigurationSpace,
leader_model):
assert isinstance(value, list)
for item in value:
assert isinstance(item, dict)
clauses = []
for name, forbidden_values in item.items():
true_name = add_leader_model(name, leader_model,
SERIES_CONNECT_LEADER_TOKEN)
if isinstance(forbidden_values,
list) and len(forbidden_values) == 1:
forbidden_values = forbidden_values[0]
if isinstance(forbidden_values, list):
clauses.append(
ForbiddenInClause(
store[true_name],
list(map(smac_hdl._encode, forbidden_values))))
else:
clauses.append(
ForbiddenEqualsClause(
store[true_name],
smac_hdl._encode(forbidden_values)))
cs.add_forbidden_clause(ForbiddenAndConjunction(*clauses))
sys.path.append(os.getcwd())
cs1 = ConfigurationSpace()
x1 = UniformFloatHyperparameter("x1", -5, 10, default_value=0)
x2 = UniformIntegerHyperparameter("x2", 1, 15, q=2.0, log=True)
x3 = CategoricalHyperparameter("x3", ['x1', 'x2', 'x3'])
x4 = CategoricalHyperparameter("x9", ['x1', 'x2', 'x3'])
x5 = CategoricalHyperparameter("x5", ['x1', 'x2', 'x3'])
x6 = CategoricalHyperparameter("x6", ['x1', 'x2', 'x3'])
x7 = CategoricalHyperparameter("x7", ['x1', 'x3', 'x2'], default_value='x2')
cs1.add_hyperparameters([x1, x2, x3, x7, x6, x5, x4])
cond1 = ForbiddenAndConjunction(
ForbiddenEqualsClause(x3, "x1"),
ForbiddenEqualsClause(x4, "x2"),
ForbiddenEqualsClause(x5, "x3")
)
cs1.add_forbidden_clause(cond1)
cond2 = EqualsCondition(x1, x5, "x1")
cond7 = EqualsCondition(x2, x6, "x1")
cond3 = InCondition(x2, x6, ["x1", "x2", "x3"])
cs1.add_condition(cond3)
cs1.add_condition(cond2)
cond4 = ForbiddenEqualsClause(x4, 'x3')
cond5 = ForbiddenInClause(x7, ['x1', 'x3'])
cs1.add_forbidden_clause(cond5)
cs1.add_forbidden_clause(cond4)