def test_impute_inactive_hyperparameters(self):
cs = smac.configspace.ConfigurationSpace()
a = cs.add_hyperparameter(
CategoricalHyperparameter('a', [0, 1], default=0))
b = cs.add_hyperparameter(
CategoricalHyperparameter('b', [0, 1], default=1))
cs.add_condition(EqualsCondition(b, a, 1))
cs.seed(1)
configs = cs.sample_configuration(size=100)
config_array = smac.configspace.convert_configurations_to_array(
configs)
for line in config_array:
if line[0] == 0:
self.assertEqual(line[1], 1)
def test_nested_conjunctions(self):
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = CategoricalHyperparameter("input4", [0, 1])
hp5 = CategoricalHyperparameter("input5", [0, 1])
hp6 = Constant("AND", "True")
cond1 = EqualsCondition(hp6, hp1, 1)
cond2 = EqualsCondition(hp6, hp2, 1)
cond3 = EqualsCondition(hp6, hp3, 1)
cond4 = EqualsCondition(hp6, hp4, 1)
cond5 = EqualsCondition(hp6, hp5, 1)
conj1 = AndConjunction(cond1, cond2)
conj2 = OrConjunction(conj1, cond3)
conj3 = AndConjunction(conj2, cond4, cond5)
# TODO: this does not look nice, And should depend on a large
# conjunction, there should not be many ANDs inside this string!
self.assertEqual("(((AND | input1 == 1 && AND | input2 == 1) || AND | "
"input3 == 1) && AND | input4 == 1 && AND | input5 "
"== 1)", str(conj3))
def test_equals_condition(self):
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
cond = EqualsCondition(hp2, hp1, 0)
cond_ = EqualsCondition(hp2, hp1, 0)
# Test vector value:
self.assertEqual(cond.vector_value, hp1._inverse_transform(0))
self.assertEqual(cond.vector_value, cond_.vector_value)
# Test invalid conditions:
self.assertRaises(TypeError, EqualsCondition, hp2, "parent", 0)
self.assertRaises(TypeError, EqualsCondition, "child", hp1, 0)
self.assertRaises(ValueError, EqualsCondition, hp1, hp1, 0)
self.assertEqual(cond, cond_)
cond_reverse = EqualsCondition(hp1, hp2, 0)
self.assertNotEqual(cond, cond_reverse)
self.assertNotEqual(cond, dict())
self.assertEqual("child | parent == 0", str(cond))
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 2000)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
elif optimizer == 'tpe':
coef0 = hp.uniform("libsvm_coef0", -1, 1)
space = {'C': hp.loguniform('libsvm_C', np.log(0.03125), np.log(32768)),
'gamma': hp.loguniform('libsvm_gamma', np.log(3.0517578125e-5), np.log(8)),
'shrinking': hp.choice('libsvm_shrinking', ["True", "False"]),
'tol': hp.loguniform('libsvm_tol', np.log(1e-5), np.log(1e-1)),
'max_iter': hp.choice('libsvm_max_iter', [2000]),
'kernel': hp.choice('libsvm_kernel',
[("poly", {'degree': hp.randint('libsvm_degree', 4) + 2, 'coef0': coef0}),
("rbf", {}),
("sigmoid", {'coef0': coef0})])}
init_trial = {'C': 1,
'gamma': 0.1,
'shrinking': "True",
'tol': 1e-3,
'max_iter': 2000,
'kernel': ("rbf", {})}
return space
def test_or_conjunction(self):
self.assertRaises(TypeError, AndConjunction, "String1", "String2")
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("Or", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
self.assertRaises(ValueError, OrConjunction, cond1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = OrConjunction(cond1, cond2)
andconj1_ = OrConjunction(cond1, cond2)
self.assertEqual(andconj1, andconj1_)
# Test setting vector idx
hyperparameter_idx = {
hp1.name: 0,
hp2.name: 1,
hp3.name: 2,
hp4.name: 3
}
andconj1.set_vector_idx(hyperparameter_idx)
self.assertEqual(andconj1.get_parents_vector(), [0, 1])
self.assertEqual(andconj1.get_children_vector(), [3, 3])
andconj2 = OrConjunction(cond2, cond3)
self.assertNotEqual(andconj1, andconj2)
andconj3 = OrConjunction(cond1, cond2, cond3)
self.assertEqual(
"(Or | input1 == 1 || Or | input2 == 1 || Or | "
"input3 == 1)", str(andconj3))
def test_or_conjunction(self):
self.assertRaises(TypeError, AndConjunction, "String1", "String2")
hp1 = CategoricalHyperparameter("input1", [0, 1])
hp2 = CategoricalHyperparameter("input2", [0, 1])
hp3 = CategoricalHyperparameter("input3", [0, 1])
hp4 = Constant("Or", "True")
cond1 = EqualsCondition(hp4, hp1, 1)
self.assertRaises(ValueError, OrConjunction, cond1)
cond2 = EqualsCondition(hp4, hp2, 1)
cond3 = EqualsCondition(hp4, hp3, 1)
andconj1 = OrConjunction(cond1, cond2)
andconj1_ = OrConjunction(cond1, cond2)
self.assertEqual(andconj1, andconj1_)
andconj2 = OrConjunction(cond2, cond3)
self.assertNotEqual(andconj1, andconj2)
andconj3 = OrConjunction(cond1, cond2, cond3)
self.assertEqual("(Or | input1 == 1 || Or | input2 == 1 || Or | "
"input3 == 1)", str(andconj3))
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
use_minimum_fraction = cs.add_hyperparameter(
CategoricalHyperparameter("use_minimum_fraction",
["True", "False"],
default="True"))
minimum_fraction = cs.add_hyperparameter(
UniformFloatHyperparameter("minimum_fraction",
lower=.0001,
upper=0.5,
default=0.01,
log=True))
cs.add_condition(
EqualsCondition(minimum_fraction, use_minimum_fraction, 'True'))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
algorithm = CategoricalHyperparameter('algorithm',
['parallel', 'deflation'], 'parallel')
whiten = CategoricalHyperparameter('whiten',
['False', 'True'], 'False')
fun = CategoricalHyperparameter(
'fun', ['logcosh', 'exp', 'cube'], 'logcosh')
cs.add_hyperparameters([n_components, algorithm, whiten, fun])
cs.add_condition(EqualsCondition(n_components, whiten, "True"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
method = CategoricalHyperparameter("method", ['average', 'weighted'],
default_value='weighted')
alpha = UniformFloatHyperparameter("alpha",
1e-5,
1e-3,
log=True,
default_value=1e-4)
cs = ConfigurationSpace()
cs.add_hyperparameters([method, alpha])
alpha_cond = EqualsCondition(alpha, method, 'weighted')
cs.add_conditions([alpha_cond])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
kernel = CategoricalHyperparameter('kernel',
['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=1.0)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def test_get_parents(self):
# Necessary because we couldn't call cs.get_parents for
# clasp-sat-params-nat.pcs
counter = UniformIntegerHyperparameter('bump', 10, 4096, log=True)
_1_S_countercond = CategoricalHyperparameter('cony', ['yes', 'no'])
_1_0_restarts = CategoricalHyperparameter(
'restarts', ['F', 'L', 'D', 'x', '+', 'no'], default='x')
condition = EqualsCondition(counter, _1_S_countercond, 'yes')
# All conditions inherit get_parents from abstractcondition
self.assertEqual([_1_S_countercond], condition.get_parents())
condition2 = InCondition(counter, _1_0_restarts,
['F', 'D', 'L', 'x', '+'])
# All conjunctions inherit get_parents from abstractconjunction
conjunction = AndConjunction(condition, condition2)
self.assertEqual([_1_S_countercond, _1_0_restarts],
conjunction.get_parents())
def get_hyperparameter_search_space(dataset_properties=None):
C = UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default="rbf")
degree = UniformIntegerHyperparameter("degree", 1, 5, default=3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default="True")
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
#max_iter = UnParametrizedHyperparameter("max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameter(C)
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(shrinking)
cs.add_hyperparameter(tol)
#cs.add_hyperparameter(max_iter)
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
n_components_factor = UniformFloatHyperparameter("n_components_factor",
0.,
1.,
default_value=1.)
algorithm = CategoricalHyperparameter("algorithm",
["parallel", "deflation"],
default_value="parallel")
whiten = CategoricalHyperparameter("whiten", [True, False],
default_value=True)
fun = CategoricalHyperparameter("fun", ["logcosh", "exp", "cube"],
default_value="logcosh")
cs.add_hyperparameters([n_components_factor, algorithm, whiten, fun])
cs.add_condition(EqualsCondition(n_components_factor, whiten, True))
return cs
def test_write_OrConjunction_condition(self):
import numpy as np
expected = "lp '--lp ' c {mi,bo}\ntemp '--temp ' r (2.000000, 5.000000)\nls '--ls ' c {sa,ca,ny}| temp==3.0 || lp %in% c(bo)\n"
temp = UniformFloatHyperparameter("temp", np.exp(2), np.exp(5), log=True)
ls = CategoricalHyperparameter("ls", ["sa", "ca", "ny"], "sa")
lp = CategoricalHyperparameter("lp", ["mi", "bo"], "bo")
cs = ConfigurationSpace()
cs.add_hyperparameter(temp)
cs.add_hyperparameter(lp)
cs.add_hyperparameter(ls)
c1 = EqualsCondition(ls, temp, np.exp(3))
c2 = InCondition(ls, lp, ['bo'])
c3 = OrConjunction(c1, c2)
cs.add_condition(c3)
value = irace.write(cs)
self.assertEqual(expected, value)
def get_hyperparameter_search_space(dataset_properties=None):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0)
n_components = UniformIntegerHyperparameter("n_components",
50,
10000,
default=100,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(n_components)
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
cs.add_condition(gamma_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
shrinkage = cs.add_hyperparameter(
CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"],
default="None"))
shrinkage_factor = cs.add_hyperparameter(
UniformFloatHyperparameter("shrinkage_factor", 0., 1., 0.5))
n_components = cs.add_hyperparameter(
UniformIntegerHyperparameter('n_components', 1, 250, default=10))
tol = cs.add_hyperparameter(
UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True))
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage,
"manual"))
return cs
def get_cs():
epsilon = CategoricalHyperparameter("epsilon",
[1e-4, 1e-3, 1e-2, 1e-1, 1],
default_value=1e-4)
C = UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 2000)
cs = ConfigurationSpace()
cs.add_hyperparameters([
epsilon, C, kernel, degree, gamma, coef0, shrinking, tol, max_iter
])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter("shrinkage",
["None", "auto", "manual"],
default_value="None")
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5)
n_components = UniformIntegerHyperparameter('n_components',
1,
250,
default_value=1)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-4,
log=True)
cs.add_hyperparameters(
[shrinkage, shrinkage_factor, n_components, tol])
cs.add_condition(
EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'n_components':
hp.randint('lda_n_components', 250) + 1,
'tol':
hp.loguniform('lda_tol', np.log(1e-5), np.log(1e-1)),
'shrinkage':
hp.choice('lda_shrinkage', [
"None", "auto",
("manual", {
'shrinkage_factor':
hp.uniform('lda_shrinkage_factor', 0, 1)
})
])
}
init_trial = {'n_components': 10, 'tol': 1e-4, 'shrinkage': "None"}
return space
def get_cs():
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter("shrinkage",
["None", "auto", "manual"],
default_value="None")
shrinkage_factor = UniformFloatHyperparameter("shrinkage_factor", 0., 1.,
0.5)
n_components = UniformIntegerHyperparameter('n_components',
1,
250,
default_value=10)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-4,
log=True)
cs.add_hyperparameters([shrinkage, shrinkage_factor, n_components, tol])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter(
"loss", ["least_squares"], default_value="least_squares")
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True)
max_iter = UniformIntegerHyperparameter(
"max_iter", 32, 512, default_value=100)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=200, default_value=20, log=True)
max_depth = UnParametrizedHyperparameter(
name="max_depth", value="None")
max_leaf_nodes = UniformIntegerHyperparameter(
name="max_leaf_nodes", lower=3, upper=2047, default_value=31, log=True)
max_bins = Constant("max_bins", 255)
l2_regularization = UniformFloatHyperparameter(
name="l2_regularization", lower=1E-10, upper=1, default_value=1E-10, log=True)
early_stop = CategoricalHyperparameter(
name="early_stop", choices=["off", "train", "valid"], default_value="off")
tol = UnParametrizedHyperparameter(
name="tol", value=1e-7)
scoring = UnParametrizedHyperparameter(
name="scoring", value="loss")
n_iter_no_change = UniformIntegerHyperparameter(
name="n_iter_no_change", lower=1, upper=20, default_value=10)
validation_fraction = UniformFloatHyperparameter(
name="validation_fraction", lower=0.01, upper=0.4, default_value=0.1)
cs.add_hyperparameters([loss, learning_rate, max_iter, min_samples_leaf,
max_depth, max_leaf_nodes, max_bins, l2_regularization,
early_stop, tol, scoring, n_iter_no_change,
validation_fraction])
n_iter_no_change_cond = InCondition(
n_iter_no_change, early_stop, ["valid", "train"])
validation_fraction_cond = EqualsCondition(
validation_fraction, early_stop, "valid")
cs.add_conditions([n_iter_no_change_cond, validation_fraction_cond])
return cs
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
C = UniformFloatHyperparameter("C", 1e-6, 25., default_value=1.)
fit_intercept = CategoricalHyperparameter("fit_intercept",
[True, False],
default_value=True)
max_iter = UniformIntegerHyperparameter("max_iter",
5,
1000,
default_value=1000)
tol = UniformFloatHyperparameter("tol", 1e-7, 1., default_value=1e-3)
early_stopping = CategoricalHyperparameter("early_stopping",
[True, False], False)
validation_fraction = UniformFloatHyperparameter("validation_fraction",
0.,
1.,
default_value=0.1)
n_iter_no_change = UniformIntegerHyperparameter("n_iter_no_change",
1,
1000,
default_value=5)
shuffle = CategoricalHyperparameter("shuffle", [True, False], True)
loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"],
default_value="hinge")
average = UniformIntegerHyperparameter("average",
1,
100,
default_value=1)
cs.add_hyperparameters([
C, fit_intercept, max_iter, tol, early_stopping,
validation_fraction, n_iter_no_change, shuffle, loss, average
])
validation_fraction_condition = EqualsCondition(
validation_fraction, early_stopping, True)
cs.add_condition(validation_fraction_condition)
return cs
def get_hyperparameter_search_space(**kwargs):
C = UniformFloatHyperparameter("C",
0.03125,
100,
log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter("shrinking", [True, False],
default_value=True)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[C, kernel, degree, gamma, coef0, shrinking, tol])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
hidden_size = UniformIntegerHyperparameter("hidden_size", 100, 500, default_value=200)
activation = CategoricalHyperparameter("activation", ["identity", "logistic", "tanh", "relu"],
default_value="relu")
solver = CategoricalHyperparameter("solver", ["sgd", "adam"], default_value="adam")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1., log=True, default_value=0.0001)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["adaptive", "invscaling", "constant"],
default_value="constant")
learning_rate_init = UniformFloatHyperparameter(
"learning_rate_init", 1e-4, 3e-1, default_value=0.001, log=True)
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True,
default_value=1e-4)
momentum = UniformFloatHyperparameter("momentum", 0.6, 1, q=0.05, default_value=0.9)
nesterovs_momentum = CategoricalHyperparameter("nesterovs_momentum", [True, False], default_value=True)
beta1 = UniformFloatHyperparameter("beta1", 0.6, 1, default_value=0.9)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True,
default_value=0.5)
cs.add_hyperparameters(
[hidden_size, activation, solver, alpha, learning_rate, learning_rate_init, tol, momentum,
nesterovs_momentum, beta1,
power_t])
learning_rate_condition = EqualsCondition(learning_rate, solver, "sgd")
momentum_condition = EqualsCondition(momentum, solver, "sgd")
nesterovs_momentum_condition = EqualsCondition(nesterovs_momentum, solver, "sgd")
beta1_condition = EqualsCondition(beta1, solver, "adam")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([learning_rate_condition, momentum_condition,
nesterovs_momentum_condition, beta1_condition, power_t_condition])
return cs
elif optimizer == 'tpe':
space = {'hidden_size': hp.randint("mlp_hidden_size", 400) + 100,
'activation': hp.choice('mlp_activation', ["identity", "logistic", "tanh", "relu"]),
'solver': hp.choice('mlp_solver',
[("sgd", {'learning_rate': hp.choice('mlp_learning_rate',
[("adaptive", {}),
("constant", {}),
("invscaling", {
'power_t': hp.uniform('mlp_power_t',
1e-5, 1)})]),
'momentum': hp.uniform('mlp_momentum', 0.6, 1),
'nesterovs_momentum': hp.choice('mlp_nesterovs_momentum',
[True, False])}),
("adam", {'beta1': hp.uniform('mlp_beta1', 0.6, 1)})]),
'alpha': hp.loguniform('mlp_alpha', np.log(1e-7), np.log(1e-1)),
'learning_rate_init': hp.loguniform('mlp_learning_rate_init', np.log(1e-6), np.log(1e-1)),
'tol': hp.loguniform('mlp_tol', np.log(1e-5), np.log(1e-1))}
return space
def get_benchmark_configspace(benchmark_id):
if benchmark_id == 'fcnet':
cs = ConfigurationSpace()
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-4,
1e-2,
default_value=1e-3,
q=2e-4)
momentum = UniformFloatHyperparameter("momentum",
0.,
.5,
default_value=0.,
q=.1)
lr_decay = UniformFloatHyperparameter("lr_decay",
.7,
.99,
default_value=9e-1,
q=3e-2)
n_layer1 = UniformIntegerHyperparameter("n_layer1",
32,
256,
default_value=96,
q=8)
n_layer2 = UniformIntegerHyperparameter("n_layer2",
64,
256,
default_value=128,
q=8)
batch_size = UniformIntegerHyperparameter("batch_size",
32,
128,
default_value=64,
q=8)
dropout1 = UniformFloatHyperparameter("kb_1",
.3,
.9,
default_value=.5,
q=.1)
dropout2 = UniformFloatHyperparameter("kb_2",
.3,
.9,
default_value=.5,
q=.1)
kernel_regularizer = UniformFloatHyperparameter("k_reg",
1e-9,
1e-4,
default_value=1e-6,
q=5e-7,
log=True)
cs.add_hyperparameters([
learning_rate, momentum, lr_decay, n_layer1, n_layer2, batch_size,
dropout1, dropout2, kernel_regularizer
])
elif benchmark_id in ['covtype', 'higgs']:
cs = ConfigurationSpace()
# n_estimators = UniformFloatHyperparameter("n_estimators", 100, 600, default_value=200, q=10)
eta = UniformFloatHyperparameter("eta",
0.01,
0.9,
default_value=0.3,
q=0.01)
min_child_weight = UniformFloatHyperparameter("min_child_weight",
0,
10,
default_value=1,
q=0.1)
max_depth = UniformIntegerHyperparameter("max_depth",
1,
12,
default_value=6)
subsample = UniformFloatHyperparameter("subsample",
0.1,
1,
default_value=1,
q=0.1)
gamma = UniformFloatHyperparameter("gamma",
0,
10,
default_value=0,
q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.1,
1,
default_value=1.,
q=0.1)
alpha = UniformFloatHyperparameter("alpha",
0,
10,
default_value=0.,
q=0.1)
_lambda = UniformFloatHyperparameter("lambda",
1,
10,
default_value=1,
q=0.1)
cs.add_hyperparameters([
eta, min_child_weight, max_depth, subsample, gamma,
colsample_bytree, alpha, _lambda
])
elif benchmark_id in ['covtype_svm', 'mnist_svm']:
C = UniformFloatHyperparameter("C",
1e-3,
1e5,
log=True,
default_value=1.0)
kernel = CategoricalHyperparameter("kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma",
1e-5,
10,
log=True,
default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 10000)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[C, kernel, degree, gamma, coef0, tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
elif benchmark_id == 'cifar':
cs = ConfigurationSpace()
# padding_size = CategoricalHyperparameter('padding_size', [1, 2, 3], default_value=2)
# batch_size = CategoricalHyperparameter('train_batch_size', [256])
batch_size = UniformIntegerHyperparameter("train_batch_size",
32,
256,
default_value=64,
q=8)
init_lr = UniformFloatHyperparameter('init_lr',
lower=1e-3,
upper=0.3,
default_value=0.1,
log=True)
# lr_decay_factor = UniformFloatHyperparameter('lr_decay_factor', lower=0.01, upper=0.2, default_value=0.1,
# log=True)
lr_decay_factor = UnParametrizedHyperparameter('lr_decay_factor', 0.1)
weight_decay = UniformFloatHyperparameter('weight_decay',
lower=1e-5,
upper=1e-2,
default_value=0.0002,
log=True)
momentum = UniformFloatHyperparameter("momentum",
0.5,
.99,
default_value=0.9)
nesterov = CategoricalHyperparameter('nesterov', ['True', 'False'],
default_value='True')
cs.add_hyperparameters([
nesterov, batch_size, init_lr, lr_decay_factor, weight_decay,
momentum
])
elif benchmark_id == 'convnet':
cs = ConfigurationSpace()
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-5,
5e-2,
default_value=1e-4,
q=3e-5,
log=True)
batch_size = UniformIntegerHyperparameter("batch_size",
16,
128,
q=16,
default_value=32)
momentum = UniformFloatHyperparameter("momentum",
0.,
.5,
default_value=0.,
q=.1)
lr_decay = UniformFloatHyperparameter("lr_decay",
.7,
.99,
default_value=9e-1,
q=3e-2)
dropout_value = UniformFloatHyperparameter("dropout",
.1,
.7,
default_value=.5,
q=.1)
cs.add_hyperparameters(
[learning_rate, batch_size, momentum, lr_decay, dropout_value])
num_pooling_layer = UniformIntegerHyperparameter("n_pooling_layer",
2,
3,
default_value=2)
num_conv_layer1 = UniformIntegerHyperparameter("n_conv_layer1",
16,
64,
default_value=32,
q=2)
num_conv_layer2 = UniformIntegerHyperparameter("n_conv_layer2",
32,
96,
default_value=64,
q=2)
num_conv_layer3 = UniformIntegerHyperparameter("n_conv_layer3",
32,
96,
default_value=64,
q=2)
num_fully_layer = UniformIntegerHyperparameter("n_fully_unit",
128,
512,
default_value=256,
q=64)
cs.add_hyperparameters([
num_pooling_layer, num_conv_layer1, num_conv_layer2,
num_conv_layer3, num_fully_layer
])
for i in [1, 2, 3]:
kernel_init_stddev = UniformFloatHyperparameter(
"kernel_init_stddev%d" % i,
1e-3,
5e-2,
default_value=1e-2,
q=2e-3)
kernel_regularizer = UniformFloatHyperparameter(
"kernel_regularizer%d" % i,
1e-9,
1e-4,
default_value=1e-6,
q=5e-7,
log=True)
cs.add_hyperparameters([kernel_init_stddev, kernel_regularizer])
if i == 3:
k_init_cond = InCondition(child=kernel_init_stddev,
parent=num_pooling_layer,
values=[3])
k_reg_cond = InCondition(child=kernel_regularizer,
parent=num_pooling_layer,
values=[3])
cs.add_conditions([k_init_cond, k_reg_cond])
return cs
elif 'sys' in benchmark_id:
from mfes.evaluate_function.sys.combined_evaluator import get_combined_cs
from solnml.datasets.utils import load_data
tmp_node = load_data('balloon',
data_dir='../soln-ml/',
task_type=0,
datanode_returned=True)
cs = get_combined_cs(tmp_node)
return cs
else:
raise ValueError('Invalid benchmark id: %s!' % benchmark_id)
return cs
def test_read_new_configuration_space_complex_conditionals(self):
classi = OrdinalHyperparameter(
"classi",
[
"random_forest", "extra_trees", "k_nearest_neighbors",
"something"
],
)
knn_weights = CategoricalHyperparameter("knn_weights",
["uniform", "distance"])
weather = OrdinalHyperparameter(
"weather", ["sunny", "rainy", "cloudy", "snowing"])
temperature = CategoricalHyperparameter("temperature", ["high", "low"])
rain = CategoricalHyperparameter("rain", ["yes", "no"])
gloves = OrdinalHyperparameter("gloves",
["none", "yarn", "leather", "gortex"])
heur1 = CategoricalHyperparameter("heur1", ["off", "on"])
heur2 = CategoricalHyperparameter("heur2", ["off", "on"])
heur_order = CategoricalHyperparameter("heur_order",
["heur1then2", "heur2then1"])
gloves_condition = OrConjunction(
EqualsCondition(gloves, rain, "yes"),
EqualsCondition(gloves, temperature, "low"))
heur_condition = AndConjunction(
EqualsCondition(heur_order, heur1, "on"),
EqualsCondition(heur_order, heur2, "on"))
and_conjunction = AndConjunction(
NotEqualsCondition(knn_weights, classi, "extra_trees"),
EqualsCondition(knn_weights, classi, "random_forest"))
Cl_condition = OrConjunction(
EqualsCondition(knn_weights, classi, "k_nearest_neighbors"),
and_conjunction, EqualsCondition(knn_weights, classi, "something"))
and1 = AndConjunction(EqualsCondition(temperature, weather, "rainy"),
EqualsCondition(temperature, weather, "cloudy"))
and2 = AndConjunction(
EqualsCondition(temperature, weather, "sunny"),
NotEqualsCondition(temperature, weather, "snowing"))
another_condition = OrConjunction(and1, and2)
complex_conditional_space = ConfigurationSpace()
complex_conditional_space.add_hyperparameter(classi)
complex_conditional_space.add_hyperparameter(knn_weights)
complex_conditional_space.add_hyperparameter(weather)
complex_conditional_space.add_hyperparameter(temperature)
complex_conditional_space.add_hyperparameter(rain)
complex_conditional_space.add_hyperparameter(gloves)
complex_conditional_space.add_hyperparameter(heur1)
complex_conditional_space.add_hyperparameter(heur2)
complex_conditional_space.add_hyperparameter(heur_order)
complex_conditional_space.add_condition(gloves_condition)
complex_conditional_space.add_condition(heur_condition)
complex_conditional_space.add_condition(Cl_condition)
complex_conditional_space.add_condition(another_condition)
complex_cs = list()
complex_cs.append(
"classi ordinal {random_forest,extra_trees,k_nearest_neighbors, something} "
"[random_forest]")
complex_cs.append(
"knn_weights categorical {uniform, distance} [uniform]")
complex_cs.append(
"weather ordinal {sunny, rainy, cloudy, snowing} [sunny]")
complex_cs.append("temperature categorical {high, low} [high]")
complex_cs.append("rain categorical { yes, no } [yes]")
complex_cs.append(
"gloves ordinal { none, yarn, leather, gortex } [none]")
complex_cs.append("heur1 categorical { off, on } [off]")
complex_cs.append("heur2 categorical { off, on } [off]")
complex_cs.append(
"heur_order categorical { heur1then2, heur2then1 } [heur1then2]")
complex_cs.append("gloves | rain == yes || temperature == low")
complex_cs.append("heur_order | heur1 == on && heur2 == on")
complex_cs.append(
"knn_weights | classi == k_nearest_neighbors || "
"classi != extra_trees && classi == random_forest || classi == something"
)
complex_cs.append(
"temperature | weather == rainy && weather == cloudy || "
"weather == sunny && weather != snowing")
cs_new = pcs_new.read(complex_cs)
self.assertEqual(cs_new, complex_conditional_space)
import unittest
from ConfigSpace.configuration_space import ConfigurationSpace
import ConfigSpace.read_and_write.pcs as pcs
import ConfigSpace.read_and_write.pcs_new as pcs_new
from ConfigSpace.hyperparameters import CategoricalHyperparameter, \
UniformIntegerHyperparameter, UniformFloatHyperparameter, OrdinalHyperparameter
from ConfigSpace.conditions import EqualsCondition, InCondition, \
AndConjunction, OrConjunction, NotEqualsCondition, \
GreaterThanCondition
from ConfigSpace.forbidden import ForbiddenInClause, ForbiddenAndConjunction
# More complex search space
classifier = CategoricalHyperparameter("classifier", ["svm", "nn"])
kernel = CategoricalHyperparameter("kernel", ["rbf", "poly", "sigmoid"])
kernel_condition = EqualsCondition(kernel, classifier, "svm")
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True)
C_condition = EqualsCondition(C, classifier, "svm")
gamma = UniformFloatHyperparameter("gamma", 0.000030518, 8, log=True)
gamma_condition = EqualsCondition(gamma, kernel, "rbf")
degree = UniformIntegerHyperparameter("degree", 1, 5)
degree_condition = InCondition(degree, kernel, ["poly", "sigmoid"])
neurons = UniformIntegerHyperparameter("neurons", 16, 1024)
neurons_condition = EqualsCondition(neurons, classifier, "nn")
lr = UniformFloatHyperparameter("lr", 0.0001, 1.0)
lr_condition = EqualsCondition(lr, classifier, "nn")
preprocessing = CategoricalHyperparameter("preprocessing", ["None", "pca"])
conditional_space = ConfigurationSpace()
conditional_space.add_hyperparameter(preprocessing)
conditional_space.add_hyperparameter(classifier)
conditional_space.add_hyperparameter(kernel)
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default_value="log")
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1, log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True,
default_value=1e-4)
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=1e-4, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True,
default_value=0.5)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
tol, epsilon, learning_rate, eta0, power_t,
average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
elif optimizer == 'tpe':
eta0 = hp.loguniform('sgd_eta0', np.log(1e-7), np.log(1e-1))
space = {
'loss': hp.choice('sgd_loss', [
("modified_huber", {'epsilon': hp.loguniform('sgd_epsilon', np.log(1e-5), np.log(1e-1))}),
("hinge", {}),
("log", {}),
("squared_hinge", {}),
("perceptron", {})]),
'penalty': hp.choice('sgd_penalty',
[("elasticnet",
{'l1_ratio': hp.loguniform('sgd_l1_ratio', np.log(1e-9), np.log(1))}),
("l1", None),
("l2", None)]),
'alpha': hp.loguniform('sgd_alpha', np.log(1e-7), np.log(1e-1)),
'fit_intercept': hp.choice('sgd_fit_intercept', ["True"]),
'tol': hp.loguniform('sgd_tol', np.log(1e-5), np.log(1e-1)),
'learning_rate': hp.choice('sgd_learning_rate', [("optimal", {}),
("invscaling",
{'power_t': hp.loguniform('sgd_power_t', np.log(1e-5),
np.log(1)),
'eta0': eta0}),
("constant", {'eta0': eta0})]),
'average': hp.choice('sgd_average', ["True", "False"])}
init_trial = {'loss': ("log", {}),
'penalty': ("l2", {}),
'alpha': 1e-4,
'fit_intercept': "True",
'tol': 1e-4,
'learning_rate': ("invscaling", {'power_t': 0.5, 'eta0': 0.01}),
'average': "False"}
return space
def get_hyperparameter_search_space(cls,
dataset_properties,
default=None,
include=None,
exclude=None):
cs = ConfigurationSpace()
# Compile a list of legal preprocessors for this problem
available_preprocessors = cls.get_available_components(
data_prop=dataset_properties, include=include, exclude=exclude)
if len(available_preprocessors) == 0:
raise ValueError(
"No preprocessors found, please add NoPreprocessing")
if default is None:
defaults = [
'no_preprocessing', 'select_percentile', 'pca', 'truncatedSVD'
]
for default_ in defaults:
if default_ in available_preprocessors:
default = default_
break
preprocessor = CategoricalHyperparameter(
'__choice__',
list(available_preprocessors.keys()),
default=default)
cs.add_hyperparameter(preprocessor)
for name in available_preprocessors:
preprocessor_configuration_space = available_preprocessors[name]. \
get_hyperparameter_search_space(dataset_properties)
for parameter in preprocessor_configuration_space.get_hyperparameters(
):
new_parameter = copy.deepcopy(parameter)
new_parameter.name = "%s:%s" % (name, new_parameter.name)
cs.add_hyperparameter(new_parameter)
# We must only add a condition if the hyperparameter is not
# conditional on something else
if len(
preprocessor_configuration_space.get_parents_of(
parameter)) == 0:
condition = EqualsCondition(new_parameter, preprocessor,
name)
cs.add_condition(condition)
for condition in available_preprocessors[name]. \
get_hyperparameter_search_space(
dataset_properties).get_conditions():
if not isinstance(condition, AbstractConjunction):
dlcs = [condition]
else:
dlcs = condition.get_descendent_literal_conditions()
for dlc in dlcs:
if not dlc.child.name.startswith(name):
dlc.child.name = "%s:%s" % (name, dlc.child.name)
if not dlc.parent.name.startswith(name):
dlc.parent.name = "%s:%s" % (name, dlc.parent.name)
cs.add_condition(condition)
for forbidden_clause in available_preprocessors[name]. \
get_hyperparameter_search_space(
dataset_properties).forbidden_clauses:
dlcs = forbidden_clause.get_descendant_literal_clauses()
for dlc in dlcs:
if not dlc.hyperparameter.name.startswith(name):
dlc.hyperparameter.name = "%s:%s" % (
name, dlc.hyperparameter.name)
cs.add_forbidden_clause(forbidden_clause)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
Window_size = UniformIntegerHyperparameter(name="Window_size",
lower=5,
upper=50,
default_value=20)
tsfresh_feature = CategoricalHyperparameter(name="tsfresh_feature",
choices=["True", "False"],
default_value="True")
Difference = CategoricalHyperparameter(name="Difference",
choices=["True", "False"],
default_value="True")
loss = CategoricalHyperparameter("loss", [
"squared_loss", "huber", "epsilon_insensitive",
"squared_epsilon_insensitive"
],
default_value="squared_loss")
penalty = CategoricalHyperparameter("penalty",
["l1", "l2", "elasticnet"],
default_value="l2")
alpha = UniformFloatHyperparameter("alpha",
1e-7,
1e-1,
log=True,
default_value=0.0001)
l1_ratio = UniformFloatHyperparameter("l1_ratio",
1e-9,
1.,
log=True,
default_value=0.15)
tol = UniformFloatHyperparameter("tol",
1e-4,
1e-1,
default_value=1e-3,
log=True)
epsilon = UniformFloatHyperparameter("epsilon",
1e-5,
1e-1,
default_value=0.1,
log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter("eta0",
1e-7,
1e-1,
default_value=0.01)
power_t = UniformFloatHyperparameter("power_t",
1e-5,
1,
default_value=0.25)
average = CategoricalHyperparameter("average", ["False", "True"],
default_value="False")
# un parametrized parameter
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
max_iter = UnParametrizedHyperparameter("max_iter", 1000)
cs.add_hyperparameters([
loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon,
learning_rate, eta0, Difference, power_t, average, Window_size,
tsfresh_feature, max_iter
])
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = InCondition(
epsilon, loss,
["huber", "epsilon_insensitive", "squared_epsilon_insensitive"])
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
"""Get the configuratin space used for hyperparameter searching."""
cs = ConfigurationSpace()
conditions = []
base_estimator = CategoricalHyperparameter(
name="base_estimator",
choices=["rf", "lr", "svc"],
default_value="rf",
)
n_features_to_select = UniformIntegerHyperparameter(
name="n_features_to_select", lower=1, upper=30, default_value=10)
step = UniformFloatHyperparameter(name="step",
lower=0.01,
upper=0.25,
default_value=0.1)
# RF
rf_n_estimators = UniformIntegerHyperparameter(name="rf_n_estimators",
lower=1,
upper=1000,
default_value=100)
rf_criterion = CategoricalHyperparameter(
name="rf_criterion",
choices=["gini", "entropy"],
default_value="gini",
)
rf_max_depth = UniformIntegerHyperparameter(name="rf_max_depth",
lower=1,
upper=100,
default_value=10)
rf_class_weight = CategoricalHyperparameter(
name="rf_class_weight",
choices=["balanced", "balanced_subsample"],
default_value="balanced_subsample",
)
conditions.append(
EqualsCondition(rf_n_estimators, base_estimator, "rf"))
conditions.append(EqualsCondition(rf_criterion, base_estimator, "rf"))
conditions.append(EqualsCondition(rf_max_depth, base_estimator, "rf"))
conditions.append(
EqualsCondition(rf_class_weight, base_estimator, "rf"))
# LR
lr_C = UniformFloatHyperparameter(name="lr_C",
lower=1.0e-3,
upper=1.0e3,
default_value=1.0,
log=True)
lr_penalty = CategoricalHyperparameter(name="lr_penalty",
choices=["l1", "l2"],
default_value="l2")
lr_fit_intercept = CategoricalHyperparameter(name="lr_fit_intercept",
choices=[True, False],
default_value=True)
lr_intercept_scaling = UniformFloatHyperparameter(
name="lr_intercept_scaling",
lower=1.0,
upper=1.0e3,
default_value=1.0,
)
lr_class_weight = CategoricalHyperparameter(
name="lr_class_weight",
choices=["balanced", "None"],
default_value="balanced",
)
conditions.append(EqualsCondition(lr_C, base_estimator, "lr"))
conditions.append(EqualsCondition(lr_penalty, base_estimator, "lr"))
conditions.append(
EqualsCondition(lr_fit_intercept, base_estimator, "lr"))
conditions.append(
AndConjunction(
EqualsCondition(lr_intercept_scaling, base_estimator, "lr"),
EqualsCondition(lr_intercept_scaling, lr_fit_intercept, True),
))
conditions.append(
EqualsCondition(lr_class_weight, base_estimator, "lr"))
# SVC
svc_C = UniformFloatHyperparameter(name="svc_C",
lower=1.0e-3,
upper=1.0e3,
default_value=1.0,
log=True)
svc_penalty = CategoricalHyperparameter(name="svc_penalty",
choices=["l1", "l2"],
default_value="l2")
svc_class_weight = CategoricalHyperparameter(
name="svc_class_weight",
choices=["balanced", "None"],
default_value="balanced",
)
svc_loss = CategoricalHyperparameter(
name="svc_loss",
choices=["hinge", "squared_hinge"],
default_value="squared_hinge",
)
svc_fit_intercept = CategoricalHyperparameter(name="svc_fit_intercept",
choices=[True, False],
default_value=True)
conditions.append(EqualsCondition(svc_C, base_estimator, "svc"))
conditions.append(EqualsCondition(svc_penalty, base_estimator, "svc"))
conditions.append(
EqualsCondition(svc_class_weight, base_estimator, "svc"))
conditions.append(EqualsCondition(svc_loss, base_estimator, "svc"))
conditions.append(
EqualsCondition(svc_fit_intercept, base_estimator, "svc"))
cs.add_hyperparameters([
base_estimator,
n_features_to_select,
step,
rf_n_estimators,
rf_criterion,
rf_max_depth,
rf_class_weight,
lr_C,
lr_penalty,
lr_fit_intercept,
lr_intercept_scaling,
lr_class_weight,
svc_C,
svc_penalty,
svc_class_weight,
svc_loss,
svc_fit_intercept,
])
for c in conditions:
cs.add_condition(c)
return cs
