def test_condition_from_cryptominisat(self):
parent = CategoricalHyperparameter('blkrest', ['0', '1'], default_value='1')
child = UniformIntegerHyperparameter('blkrestlen', 2000, 10000,
log=True)
condition = EqualsCondition(child, parent, '1')
self.assertFalse(condition.evaluate(dict(blkrest='0')))
self.assertTrue(condition.evaluate(dict(blkrest='1')))
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter("n_components",
10,
2000,
default_value=100)
kernel = CategoricalHyperparameter(
'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter(
"gamma",
3.0517578125e-05,
8,
log=True,
default_value=1.0,
)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions(
[degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
n_components = UniformIntegerHyperparameter(
"n_components", 50, 5000, default_value=100, log=True)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(**kwargs):
n_components_factor = UniformFloatHyperparameter("n_components_factor",
0.,
1.,
default_value=1.)
kernel = CategoricalHyperparameter(
'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=1.0)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1., 1., default_value=0.)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[n_components_factor, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions(
[degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_cs():
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 10000)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def test_get_parents(self):
# Necessary because we couldn't call cs.get_parents for
# clasp-sat-params-nat.pcs
counter = UniformIntegerHyperparameter('bump', 10, 4096, log=True)
_1_S_countercond = CategoricalHyperparameter('cony', ['yes', 'no'])
_1_0_restarts = CategoricalHyperparameter('restarts', ['F', 'L', 'D',
'x', '+', 'no'],
default_value='x')
condition = EqualsCondition(counter, _1_S_countercond, 'yes')
# All conditions inherit get_parents from abstractcondition
self.assertEqual([_1_S_countercond], condition.get_parents())
condition2 = InCondition(counter, _1_0_restarts, ['F', 'D', 'L', 'x', '+'])
# All conjunctions inherit get_parents from abstractconjunction
conjunction = AndConjunction(condition, condition2)
self.assertEqual([_1_S_countercond, _1_0_restarts], conjunction.get_parents())
def condition_specification(child_name, condition, configuration_space):
# specifies the condition type
child = configuration_space.get_hyperparameter(child_name)
parent_name = condition[0]
parent = configuration_space.get_hyperparameter(parent_name)
operation = condition[1]
if operation == 'in':
restrictions = condition[3:-1:2]
for i in range(len(restrictions)):
if isinstance(parent, FloatHyperparameter):
restrictions[i] = float(restrictions[i])
elif isinstance(parent, IntegerHyperparameter):
restrictions[i] = int(restrictions[i])
if len(restrictions) == 1:
condition = EqualsCondition(child, parent, restrictions[0])
else:
condition = InCondition(child, parent, values=restrictions)
return condition
else:
restrictions = condition[2]
if isinstance(parent, FloatHyperparameter):
restrictions = float(restrictions)
elif isinstance(parent, IntegerHyperparameter):
restrictions = int(restrictions)
if operation == '==':
condition = EqualsCondition(child, parent, restrictions)
elif operation == '!=':
condition = NotEqualsCondition(child, parent, restrictions)
else:
if isinstance(parent, FloatHyperparameter):
restrictions = float(restrictions)
elif isinstance(parent, IntegerHyperparameter):
restrictions = int(restrictions)
elif isinstance(parent, OrdinalHyperparameter):
pass
else:
raise ValueError('The parent of a conditional hyperparameter '
'must be either a float, int or ordinal '
'hyperparameter, but is %s.' % type(parent))
if operation == '<':
condition = LessThanCondition(child, parent, restrictions)
elif operation == '>':
condition = GreaterThanCondition(child, parent, restrictions)
return condition
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default_value="log")
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1, log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True,
default_value=1e-4)
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=1e-4, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1,
default_value=0.5)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
tol, epsilon, learning_rate, eta0, power_t,
average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
def addSearchSpaceGrid(hp: SearchSpaceGrid, disc: int,
parent_disc: Hyperparameter,
cs: ConfigurationSpace) -> None:
smac = SearchSpaceGridtoSMAC(hp, disc)
for hyp in smac:
cs.add_hyperparameter(hyp)
cs.add_condition(
EqualsCondition(child=hyp, parent=parent_disc, value=disc))
def test_get_parents(self):
# Necessary because we couldn't call cs.get_parents for
# clasp-sat-params-nat.pcs
counter = UniformIntegerHyperparameter('bump', 10, 4096, log=True)
_1_S_countercond = CategoricalHyperparameter('cony', ['yes', 'no'])
_1_0_restarts = CategoricalHyperparameter(
'restarts', ['F', 'L', 'D', 'x', '+', 'no'], default='x')
condition = EqualsCondition(counter, _1_S_countercond, 'yes')
# All conditions inherit get_parents from abstractcondition
self.assertEqual([_1_S_countercond], condition.get_parents())
condition2 = InCondition(counter, _1_0_restarts,
['F', 'D', 'L', 'x', '+'])
# All conjunctions inherit get_parents from abstractconjunction
conjunction = AndConjunction(condition, condition2)
self.assertEqual([_1_S_countercond, _1_0_restarts],
conjunction.get_parents())
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss", ["least_squares"],
default_value="least_squares")
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=1,
default_value=0.1,
log=True)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf",
lower=1,
upper=200,
default_value=20,
log=True)
max_depth = UnParametrizedHyperparameter(name="max_depth",
value="None")
max_leaf_nodes = UniformIntegerHyperparameter(name="max_leaf_nodes",
lower=3,
upper=2047,
default_value=31,
log=True)
max_bins = Constant("max_bins", 255)
l2_regularization = UniformFloatHyperparameter(
name="l2_regularization",
lower=1E-10,
upper=1,
default_value=1E-10,
log=True)
early_stop = CategoricalHyperparameter(
name="early_stop",
choices=["off", "valid", "train"],
default_value="off")
tol = UnParametrizedHyperparameter(name="tol", value=1e-7)
scoring = UnParametrizedHyperparameter(name="scoring", value="loss")
n_iter_no_change = UniformIntegerHyperparameter(
name="n_iter_no_change", lower=1, upper=20, default_value=10)
validation_fraction = UniformFloatHyperparameter(
name="validation_fraction",
lower=0.01,
upper=0.4,
default_value=0.1)
cs.add_hyperparameters([
loss, learning_rate, min_samples_leaf, max_depth, max_leaf_nodes,
max_bins, l2_regularization, early_stop, tol, scoring,
n_iter_no_change, validation_fraction
])
n_iter_no_change_cond = InCondition(n_iter_no_change, early_stop,
["valid", "train"])
validation_fraction_cond = EqualsCondition(validation_fraction,
early_stop, "valid")
cs.add_conditions([n_iter_no_change_cond, validation_fraction_cond])
return cs
def _construct_eq_condition(
condition: Dict,
cs: ConfigurationSpace,
) -> EqualsCondition:
return EqualsCondition(
child=cs.get_hyperparameter(condition['child']),
parent=cs.get_hyperparameter(condition['parent']),
value=condition['value'],
)
def add_optional_hyperparameter(self: ConfigurationSpace,
hyperparam: Hyperparameter,
is_enabled_default=False):
enable_param = CategoricalHyperparameter(hyperparam.name + '__enable',
[True, False],
default_value=is_enabled_default)
self.add_hyperparameters([enable_param, hyperparam])
self.add_condition(EqualsCondition(hyperparam, enable_param, True))
return hyperparam
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = cs.add_hyperparameter(CategoricalHyperparameter("loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default="log"))
penalty = cs.add_hyperparameter(CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default="l2"))
alpha = cs.add_hyperparameter(UniformFloatHyperparameter(
"alpha", 10e-7, 1e-1, log=True, default=0.0001))
l1_ratio = cs.add_hyperparameter(UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1, log=True, default=0.15))
fit_intercept = cs.add_hyperparameter(UnParametrizedHyperparameter(
"fit_intercept", "True"))
n_iter = cs.add_hyperparameter(UniformIntegerHyperparameter(
"n_iter", 5, 1000, log=True, default=20))
epsilon = cs.add_hyperparameter(UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default=1e-4, log=True))
learning_rate = cs.add_hyperparameter(CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default="optimal"))
eta0 = cs.add_hyperparameter(UniformFloatHyperparameter(
"eta0", 10**-7, 0.1, default=0.01))
power_t = cs.add_hyperparameter(UniformFloatHyperparameter(
"power_t", 1e-5, 1, default=0.25))
average = cs.add_hyperparameter(CategoricalHyperparameter(
"average", ["False", "True"], default="False"))
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
# eta0 seems to be always active according to the source code; when
# learning_rate is set to optimial, eta0 is the starting value:
# https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx
#eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling")
#eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant")
#eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant)
power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling")
cs.add_condition(elasticnet)
cs.add_condition(epsilon_condition)
cs.add_condition(power_t_condition)
return cs
def test_all_components_have_the_same_child(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
cond1 = EqualsCondition(hp1, hp2, 1)
cond2 = EqualsCondition(hp1, hp3, 1)
cond3 = EqualsCondition(hp1, hp4, 1)
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
AndConjunction(cond1, cond2, cond3)
AndConjunction(cond4, cond5)
self.assertRaisesRegexp(
ValueError, "All Conjunctions and Conditions must have "
"the same child.", AndConjunction, cond1, cond4)
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
use_minimum_fraction = CategoricalHyperparameter(
"use_minimum_fraction", ["True", "False"], default_value="True")
minimum_fraction = UniformFloatHyperparameter(
"minimum_fraction", lower=.0001, upper=0.5, default_value=0.01, log=True)
cs.add_hyperparameters([use_minimum_fraction, minimum_fraction])
cs.add_condition(EqualsCondition(minimum_fraction,
use_minimum_fraction, 'True'))
return cs
def get_libsvm_svc_default_search_space():
classif_prefix = "classifier:libsvm_svc:"
model_type = CategoricalHyperparameter('classifier:__choice__',
['libsvm_svc'])
imputation = CategoricalHyperparameter('imputation:strategy',
['mean', 'median', 'most_frequent'])
C = UniformFloatHyperparameter(classif_prefix + "C",
0.03125,
32768,
log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name=classif_prefix + "kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter(classif_prefix + "degree",
2,
5,
default_value=3)
gamma = UniformFloatHyperparameter(classif_prefix + "gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(classif_prefix + "coef0",
-1,
1,
default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(classif_prefix + "shrinking",
["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter(classif_prefix + "tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter(classif_prefix + "max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameters([
model_type, imputation, C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter
])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='tpe'):
method = CategoricalHyperparameter("method", ['average', 'weighted'], default_value='weighted')
alpha = UniformFloatHyperparameter("alpha", 1e-5, 1e-3, log=True, default_value=1e-4)
cs = ConfigurationSpace()
cs.add_hyperparameters([method, alpha])
alpha_cond = EqualsCondition(alpha, method, 'weighted')
cs.add_conditions([alpha_cond])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default_value="None")
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5)
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True)
cs.add_hyperparameters([shrinkage, shrinkage_factor, tol])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
def test_and_conjunction(self):
self.assertRaises(TypeError, AndConjunction, "String1", "String2")
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("And", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
# Only one condition in an AndConjunction!
self.assertRaises(ValueError, AndConjunction, cond1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = AndConjunction(cond1, cond2)
andconj1_ = AndConjunction(cond1, cond2)
self.assertEqual(andconj1, andconj1_)
# Test setting vector idx
hyperparameter_idx = {
hp1.name: 0,
hp2.name: 1,
hp3.name: 2,
hp4.name: 3
}
andconj1.set_vector_idx(hyperparameter_idx)
self.assertEqual(andconj1.get_parents_vector(), [0, 1])
self.assertEqual(andconj1.get_children_vector(), [3, 3])
andconj2 = AndConjunction(cond2, cond3)
self.assertNotEqual(andconj1, andconj2)
andconj3 = AndConjunction(cond1, cond2, cond3)
self.assertEqual(
"(And | input1 == 1 && And | input2 == 1 && And | "
"input3 == 1)", str(andconj3))
# Test __eq__
self.assertNotEqual(andconj1, andconj3)
self.assertNotEqual(andconj1, "String")
def get_cs():
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default_value="None")
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5)
n_components = UniformIntegerHyperparameter('n_components', 1, 250, default_value=10)
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4, log=True)
cs.add_hyperparameters([shrinkage, shrinkage_factor, n_components, tol])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
def load_primitive_configspace(configspace, primitive_name):
default_configspace = load_default_configspace(primitive_name)
all_hyperparameters = load_hyperparameters(primitive_name)
default_hyperparameters = set(
default_configspace.get_hyperparameter_names())
casted_hyperparameters = []
union_conditions = []
for hp_name in all_hyperparameters:
new_hp_name = primitive_name + '|' + hp_name
if not isinstance(all_hyperparameters[hp_name], Union):
new_hp = default_configspace.get_hyperparameter(new_hp_name) if new_hp_name in default_hyperparameters \
else None
casted_hp = cast_hyperparameter(all_hyperparameters[hp_name],
new_hp_name)
if casted_hp is not None:
if new_hp is not None and casted_hp.is_legal(
new_hp.default_value):
casted_hyperparameters.append(new_hp)
else:
casted_hyperparameters.append(casted_hp)
elif isinstance(all_hyperparameters[hp_name], Union):
hyperparameter_class = all_hyperparameters[hp_name]
add_case = False
case_choices = []
cases_param_space = []
for case, case_class in hyperparameter_class.configuration.items():
new_hp = None
if new_hp_name + '|' + case in default_hyperparameters:
new_hp = default_configspace.get_hyperparameter(
new_hp_name + '|' + case)
casted_hp = cast_hyperparameter(case_class,
new_hp_name + '|' + case)
if casted_hp is not None:
add_case = True
case_choices.append(case)
if new_hp is not None and casted_hp.is_legal(
new_hp.default_value):
casted_hyperparameters.append(new_hp)
cases_param_space.append(new_hp)
else:
casted_hyperparameters.append(casted_hp)
cases_param_space.append(casted_hp)
if add_case:
case_hyperparameter = CategoricalHyperparameter(
name=new_hp_name + '|case',
choices=case_choices,
default_value=case_choices[0])
for i in range(len(case_choices)):
union_conditions.append(
EqualsCondition(cases_param_space[i],
case_hyperparameter, case_choices[i]))
casted_hyperparameters.append(case_hyperparameter)
def test_write_equals_condition_numerical(self):
expected = "temp '--temp ' i (1, 2)\nls '--ls ' c {sa,ca,ny}| temp==2\n"
temp = UniformIntegerHyperparameter("temp", 1, 2)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(ls)
c1 = EqualsCondition(ls, temp, 2)
cs.add_condition(c1)
value = irace.write(cs)
self.assertEqual(expected, value)
def test_write_equals_condition_categorical(self):
expected = "ls '--ls ' c {sa,ca,ny}\ntemp '--temp ' r (0.500000, 1.000000)| ls==sa\n"
temp = UniformFloatHyperparameter("temp", 0.5, 1)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(ls)
c1 = EqualsCondition(temp, ls, 'sa')
cs.add_condition(c1)
value = irace.write(cs)
self.assertEqual(expected, value)
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
shrinkage = cs.add_hyperparameter(CategoricalHyperparameter(
"shrinkage", ["None", "auto", "manual"], default="None"))
shrinkage_factor = cs.add_hyperparameter(UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5))
n_components = cs.add_hyperparameter(UniformIntegerHyperparameter(
'n_components', 1, 250, default=10))
tol = cs.add_hyperparameter(UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default=1e-4, log=True))
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
def add_component_deepcopy(config_space, prefix, component_space):
# We have to retrieve the configuration space every time because
# we change the objects it returns. If we reused it, we could not
# retrieve the conditions further down
# TODO implement copy for hyperparameters and forbidden and
# conditions!
component = config_space.get_hyperparameter("__choice__")
for parameter in component_space.get_hyperparameters():
new_parameter = copy.deepcopy(parameter)
new_parameter.name = "%s:%s" % (prefix,
new_parameter.name)
config_space.add_hyperparameter(new_parameter)
# We must only add a condition if the hyperparameter is not
# conditional on something else
if len(component_space.get_parents_of(parameter)) == 0:
condition = EqualsCondition(new_parameter, component,
prefix)
config_space.add_condition(condition)
space_copy = copy.deepcopy(component_space)
for condition in space_copy.get_conditions():
dlcs = condition.get_descendant_literal_conditions()
for dlc in dlcs:
if not dlc.child.name.startswith(prefix):
dlc.child.name = "%s:%s" % (
prefix, dlc.child.name)
if not dlc.parent.name.startswith(prefix):
dlc.parent.name = "%s:%s" % (
prefix, dlc.parent.name)
config_space.add_condition(condition)
space_copy = copy.deepcopy(component_space)
for forbidden_clause in space_copy.forbidden_clauses:
dlcs = forbidden_clause.get_descendant_literal_clauses()
for dlc in dlcs:
if not dlc.hyperparameter.name.startswith(prefix):
dlc.hyperparameter.name = "%s:%s" % (prefix,
dlc.hyperparameter.name)
config_space.add_forbidden_clause(forbidden_clause)
return config_space
def test_nested_conjunctions(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = EqualsCondition(hp6, hp2, 1)
cond3 = EqualsCondition(hp6, hp3, 1)
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
# TODO: this does not look nice, And should depend on a large
# conjunction, there should not be many ANDs inside this string!
self.assertEqual("(((AND | input1 == 1 && AND | input2 == 1) || AND | "
"input3 == 1) && AND | input4 == 1 && AND | input5 "
"== 1)", str(conj3))
def test_impute_inactive_hyperparameters(self):
cs = smac.configspace.ConfigurationSpace()
a = cs.add_hyperparameter(
CategoricalHyperparameter('a', [0, 1], default_value=0))
b = cs.add_hyperparameter(
CategoricalHyperparameter('b', [0, 1], default_value=1))
cs.add_condition(EqualsCondition(b, a, 1))
cs.seed(1)
configs = cs.sample_configuration(size=100)
config_array = smac.configspace.convert_configurations_to_array(
configs)
for line in config_array:
if line[0] == 0:
self.assertEqual(line[1], 1)
def test_equals_condition(self):
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
cond_ = EqualsCondition(hp2, hp1, 0)
# Test vector value:
self.assertEqual(cond.vector_value, hp1._inverse_transform(0))
self.assertEqual(cond.vector_value, cond_.vector_value)
# Test invalid conditions:
self.assertRaises(TypeError, EqualsCondition, hp2, "parent", 0)
self.assertRaises(TypeError, EqualsCondition, "child", hp1, 0)
self.assertRaises(ValueError, EqualsCondition, hp1, hp1, 0)
self.assertEqual(cond, cond_)
cond_reverse = EqualsCondition(hp1, hp2, 0)
self.assertNotEqual(cond, cond_reverse)
self.assertNotEqual(cond, dict())
self.assertEqual("child | parent == 0", str(cond))
def __condition(self, item: Dict, store: Dict):
child = item["_child"]
child = store[child]
parent = item["_parent"]
parent = store[parent]
value = (item["_values"])
if (isinstance(value, list) and len(value) == 1):
value = value[0]
if isinstance(value, list):
cond = InCondition(child, parent, list(map(smac_hdl._encode,
value)))
else:
cond = EqualsCondition(child, parent, smac_hdl._encode(value))
return cond
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 2000)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
elif optimizer == 'tpe':
coef0 = hp.uniform("libsvm_coef0", -1, 1)
space = {'C': hp.loguniform('libsvm_C', np.log(0.03125), np.log(32768)),
'gamma': hp.loguniform('libsvm_gamma', np.log(3.0517578125e-5), np.log(8)),
'shrinking': hp.choice('libsvm_shrinking', ["True", "False"]),
'tol': hp.loguniform('libsvm_tol', np.log(1e-5), np.log(1e-1)),
'max_iter': hp.choice('libsvm_max_iter', [2000]),
'kernel': hp.choice('libsvm_kernel',
[("poly", {'degree': hp.randint('libsvm_degree', 4) + 2, 'coef0': coef0}),
("rbf", {}),
("sigmoid", {'coef0': coef0})])}
init_trial = {'C': 1,
'gamma': 0.1,
'shrinking': "True",
'tol': 1e-3,
'max_iter': 2000,
'kernel': ("rbf", {})}
return space
def test_or_conjunction(self):
self.assertRaises(TypeError, AndConjunction, "String1", "String2")
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("Or", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
self.assertRaises(ValueError, OrConjunction, cond1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = OrConjunction(cond1, cond2)
andconj1_ = OrConjunction(cond1, cond2)
self.assertEqual(andconj1, andconj1_)
andconj2 = OrConjunction(cond2, cond3)
self.assertNotEqual(andconj1, andconj2)
andconj3 = OrConjunction(cond1, cond2, cond3)
self.assertEqual("(Or | input1 == 1 || Or | input2 == 1 || Or | "
"input3 == 1)", str(andconj3))
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
algorithm = CategoricalHyperparameter('algorithm',
['parallel', 'deflation'], 'parallel')
whiten = CategoricalHyperparameter('whiten',
['False', 'True'], 'False')
fun = CategoricalHyperparameter(
'fun', ['logcosh', 'exp', 'cube'], 'logcosh')
cs.add_hyperparameters([n_components, algorithm, whiten, fun])
cs.add_condition(EqualsCondition(n_components, whiten, "True"))
return cs
def get_hyperparameter_search_space(cls, dataset_properties=None,
default=None,
include=None,
exclude=None):
cs = ConfigurationSpace()
# Compile a list of legal preprocessors for this problem
available_preprocessors = cls.get_available_components(
data_prop=dataset_properties,
include=include, exclude=exclude)
if len(available_preprocessors) == 0:
raise ValueError(
"No rescaling algorithm found.")
if default is None:
defaults = ['min/max', 'standardize', 'none', 'normalize']
for default_ in defaults:
if default_ in available_preprocessors:
default = default_
break
preprocessor = CategoricalHyperparameter('__choice__',
list(
available_preprocessors.keys()),
default=default)
cs.add_hyperparameter(preprocessor)
for name in available_preprocessors:
preprocessor_configuration_space = available_preprocessors[name]. \
get_hyperparameter_search_space(dataset_properties)
for parameter in preprocessor_configuration_space.get_hyperparameters():
new_parameter = copy.deepcopy(parameter)
new_parameter.name = "%s:%s" % (name, new_parameter.name)
cs.add_hyperparameter(new_parameter)
# We must only add a condition if the hyperparameter is not
# conditional on something else
if len(preprocessor_configuration_space.
get_parents_of(parameter)) == 0:
condition = EqualsCondition(new_parameter, preprocessor,
name)
cs.add_condition(condition)
for condition in available_preprocessors[name]. \
get_hyperparameter_search_space(
dataset_properties).get_conditions():
if not isinstance(condition, AbstractConjunction):
dlcs = [condition]
else:
dlcs = condition.get_descendent_literal_conditions()
for dlc in dlcs:
if not dlc.child.name.startswith(name):
dlc.child.name = "%s:%s" % (name, dlc.child.name)
if not dlc.parent.name.startswith(name):
dlc.parent.name = "%s:%s" % (name, dlc.parent.name)
cs.add_condition(condition)
for forbidden_clause in available_preprocessors[name]. \
get_hyperparameter_search_space(
dataset_properties).forbidden_clauses:
dlcs = forbidden_clause.get_descendant_literal_clauses()
for dlc in dlcs:
if not dlc.hyperparameter.name.startswith(name):
dlc.hyperparameter.name = "%s:%s" % (name,
dlc.hyperparameter.name)
cs.add_forbidden_clause(forbidden_clause)
return cs
def get_hyperparameter_search_space(cls, dataset_properties,
default=None,
include=None,
exclude=None):
if include is not None and exclude is not None:
raise ValueError("The argument include and exclude cannot be used together.")
cs = ConfigurationSpace()
# Compile a list of all estimator objects for this problem
available_estimators = cls.get_available_components(
data_prop=dataset_properties,
include=include,
exclude=exclude)
if len(available_estimators) == 0:
raise ValueError("No regressors found")
if default is None:
defaults = ['random_forest', 'support_vector_regression'] + \
list(available_estimators.keys())
for default_ in defaults:
if default_ in available_estimators:
if include is not None and default_ not in include:
continue
if exclude is not None and default_ in exclude:
continue
default = default_
break
estimator = CategoricalHyperparameter('__choice__',
list(available_estimators.keys()),
default=default)
cs.add_hyperparameter(estimator)
for estimator_name in available_estimators.keys():
# We have to retrieve the configuration space every time because
# we change the objects it returns. If we reused it, we could not
# retrieve the conditions further down
# TODO implement copy for hyperparameters and forbidden and
# conditions!
estimator_configuration_space = available_estimators[
estimator_name]. \
get_hyperparameter_search_space(dataset_properties)
for parameter in estimator_configuration_space.get_hyperparameters():
new_parameter = copy.deepcopy(parameter)
new_parameter.name = "%s:%s" % (
estimator_name, new_parameter.name)
cs.add_hyperparameter(new_parameter)
# We must only add a condition if the hyperparameter is not
# conditional on something else
if len(estimator_configuration_space.
get_parents_of(parameter)) == 0:
condition = EqualsCondition(new_parameter, estimator,
estimator_name)
cs.add_condition(condition)
for condition in available_estimators[estimator_name]. \
get_hyperparameter_search_space(
dataset_properties).get_conditions():
dlcs = condition.get_descendant_literal_conditions()
for dlc in dlcs:
if not dlc.child.name.startswith(estimator_name):
dlc.child.name = "%s:%s" % (
estimator_name, dlc.child.name)
if not dlc.parent.name.startswith(estimator_name):
dlc.parent.name = "%s:%s" % (
estimator_name, dlc.parent.name)
cs.add_condition(condition)
for forbidden_clause in available_estimators[estimator_name]. \
get_hyperparameter_search_space(
dataset_properties).forbidden_clauses:
dlcs = forbidden_clause.get_descendant_literal_clauses()
for dlc in dlcs:
if not dlc.hyperparameter.name.startswith(estimator_name):
dlc.hyperparameter.name = "%s:%s" % (estimator_name,
dlc.hyperparameter.name)
cs.add_forbidden_clause(forbidden_clause)
return cs