def test_uniformfloat_to_integer(self):
f1 = UniformFloatHyperparameter("param", 1, 10, q=0.1, log=True)
with warnings.catch_warnings():
f2 = f1.to_integer()
warnings.simplefilter("ignore")
# TODO is this a useful rounding?
# TODO should there be any rounding, if e.g. lower=0.1
self.assertEqual("param, Type: UniformInteger, Range: [1, 10], "
"Default: 3, on log-scale", str(f2))
def get_hyperparameter_search_space(dataset_properties=None):
C = UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default="rbf")
degree = UniformIntegerHyperparameter("degree", 1, 5, default=3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default="True")
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameter(C)
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(shrinking)
cs.add_hyperparameter(tol)
cs.add_hyperparameter(max_iter)
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
gamma = UniformFloatHyperparameter(
"gamma", 0.3, 2., default=1.0)
n_components = UniformIntegerHyperparameter(
"n_components", 50, 10000, default=100, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(n_components)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
keep_variance = UniformFloatHyperparameter(
"keep_variance", 0.5, 0.9999, default=0.9999)
whiten = CategoricalHyperparameter(
"whiten", ["False", "True"], default="False")
cs = ConfigurationSpace()
cs.add_hyperparameter(keep_variance)
cs.add_hyperparameter(whiten)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = cs.add_hyperparameter(CategoricalHyperparameter("loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default="hinge"))
penalty = cs.add_hyperparameter(CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default="l2"))
alpha = cs.add_hyperparameter(UniformFloatHyperparameter(
"alpha", 10e-7, 1e-1, log=True, default=0.0001))
l1_ratio = cs.add_hyperparameter(UniformFloatHyperparameter(
"l1_ratio", 0, 1, default=0.15))
fit_intercept = cs.add_hyperparameter(UnParametrizedHyperparameter(
"fit_intercept", "True"))
n_iter = cs.add_hyperparameter(UniformIntegerHyperparameter(
"n_iter", 5, 1000, default=20))
epsilon = cs.add_hyperparameter(UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default=1e-4, log=True))
learning_rate = cs.add_hyperparameter(CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default="optimal"))
eta0 = cs.add_hyperparameter(UniformFloatHyperparameter(
"eta0", 10**-7, 0.1, default=0.01))
power_t = cs.add_hyperparameter(UniformFloatHyperparameter(
"power_t", 1e-5, 1, default=0.25))
average = cs.add_hyperparameter(CategoricalHyperparameter(
"average", ["False", "True"], default="False"))
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
# eta0 seems to be always active according to the source code; when
# learning_rate is set to optimial, eta0 is the starting value:
# https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx
#eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling")
#eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant")
#eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant)
power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling")
cs.add_condition(elasticnet)
cs.add_condition(epsilon_condition)
cs.add_condition(power_t_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
alpha = UniformFloatHyperparameter(name="alpha",
lower=0.0001,
upper=10,
default=1.0,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(alpha)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
penalty = cs.add_hyperparameter(
CategoricalHyperparameter("penalty", ["l1", "l2"], default="l2"))
loss = cs.add_hyperparameter(
CategoricalHyperparameter("loss", ["hinge", "squared_hinge"],
default="squared_hinge"))
dual = cs.add_hyperparameter(Constant("dual", "False"))
# This is set ad-hoc
tol = cs.add_hyperparameter(
UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True))
C = cs.add_hyperparameter(
UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default=1.0))
multi_class = cs.add_hyperparameter(Constant("multi_class", "ovr"))
# These are set ad-hoc
fit_intercept = cs.add_hyperparameter(Constant("fit_intercept",
"True"))
intercept_scaling = cs.add_hyperparameter(
Constant("intercept_scaling", 1))
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge"))
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge"))
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l1"))
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
shrinkage = cs.add_hyperparameter(
CategoricalHyperparameter("shrinkage", ["None", "auto", "manual"],
default="None"))
shrinkage_factor = cs.add_hyperparameter(
UniformFloatHyperparameter("shrinkage_factor", 0., 1., 0.5))
n_components = cs.add_hyperparameter(
UniformIntegerHyperparameter('n_components', 1, 250, default=10))
tol = cs.add_hyperparameter(
UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True))
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage,
"manual"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0)
n_components = UniformIntegerHyperparameter("n_components",
50,
10000,
default=100,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(n_components)
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
cs.add_condition(gamma_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
loss = CategoricalHyperparameter(
name="loss", choices=["ls", "lad"],
default='ls') #, "huber", "quantile"], default='ls')
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.0001,
upper=1,
default=0.1,
log=True)
subsample = UniformFloatHyperparameter(name="subsample",
lower=0.01,
upper=1.0,
default=1.0,
log=False)
n_estimators = Constant("n_estimators", 100)
max_features = UniformFloatHyperparameter("max_features",
0.5,
5,
default=1)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default=3)
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default=2, log=False)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default=1, log=False)
cs = ConfigurationSpace()
cs.add_hyperparameter(n_estimators)
cs.add_hyperparameter(loss)
cs.add_hyperparameter(learning_rate)
cs.add_hyperparameter(max_features)
cs.add_hyperparameter(max_depth)
cs.add_hyperparameter(min_samples_split)
cs.add_hyperparameter(min_samples_leaf)
cs.add_hyperparameter(subsample)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
percentile = UniformFloatHyperparameter(
"percentile", lower=1, upper=99, default=50)
score_func = UnParametrizedHyperparameter(
name="score_func", value="f_regression")
cs = ConfigurationSpace()
cs.add_hyperparameter(percentile)
cs.add_hyperparameter(score_func)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = cs.add_hyperparameter(Constant("loss", "deviance"))
learning_rate = cs.add_hyperparameter(
UniformFloatHyperparameter(name="learning_rate",
lower=0.0001,
upper=1,
default=0.1,
log=True))
n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100))
max_depth = cs.add_hyperparameter(
UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default=3))
min_samples_split = cs.add_hyperparameter(
UniformIntegerHyperparameter(name="min_samples_split",
lower=2,
upper=20,
default=2,
log=False))
min_samples_leaf = cs.add_hyperparameter(
UniformIntegerHyperparameter(name="min_samples_leaf",
lower=1,
upper=20,
default=1,
log=False))
min_weight_fraction_leaf = cs.add_hyperparameter(
UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.))
subsample = cs.add_hyperparameter(
UniformFloatHyperparameter(name="subsample",
lower=0.01,
upper=1.0,
default=1.0,
log=False))
max_features = cs.add_hyperparameter(
UniformFloatHyperparameter("max_features", 0.5, 5, default=1))
max_leaf_nodes = cs.add_hyperparameter(
UnParametrizedHyperparameter(name="max_leaf_nodes", value="None"))
return cs
def _lr_policy_configuration_space(cs, policy=None):
if policy == 'inv':
lr_policy = Constant(name='lr_policy', value='inv')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-2,
upper=1.0,
log=True,
default=1e-2)
power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(power)
elif policy == 'exp':
lr_policy = Constant(name='lr_policy', value='exp')
gamma = UniformFloatHyperparameter(name="gamma",
lower=0.7,
upper=1.0,
default=0.79)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
elif policy == 'step':
lr_policy = Constant(name='lr_policy', value='step')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-2,
upper=1.0,
log=True,
default=1e-2)
epoch_step = CategoricalHyperparameter("epoch_step", [6, 8, 12],
default=8)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(epoch_step)
else:
lr_policy = Constant(name='lr_policy', value='fixed')
cs.add_hyperparameter(lr_policy)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConstrainedFeedNet.get_hyperparameter_search_space()
solver = Constant(name='solver', value='momentum')
cs.add_hyperparameter(solver)
momentum = UniformFloatHyperparameter("momentum",
0.3,
0.999,
default=0.9)
cs.add_hyperparameter(momentum)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"],
default="hinge")
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
n_iter = UniformIntegerHyperparameter("n_iter", 5, 1000, default=20)
C = UniformFloatHyperparameter("C", 1e-5, 10, 1, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(loss)
cs.add_hyperparameter(fit_intercept)
cs.add_hyperparameter(n_iter)
cs.add_hyperparameter(C)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
nugget = UniformFloatHyperparameter(name="nugget",
lower=0.0001,
upper=10,
default=0.1,
log=True)
thetaL = UniformFloatHyperparameter(name="thetaL",
lower=1e-6,
upper=1e-3,
default=1e-4,
log=True)
thetaU = UniformFloatHyperparameter(name="thetaU",
lower=0.2,
upper=10,
default=1.0,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(nugget)
cs.add_hyperparameter(thetaL)
cs.add_hyperparameter(thetaU)
return cs
def test_uniformfloat_is_legal(self):
f1 = UniformFloatHyperparameter("param", 0.1, 10, q=0.1, log=True)
self.assertTrue(f1.is_legal(3.0))
self.assertTrue(f1.is_legal(3))
self.assertFalse(f1.is_legal(-0.1))
self.assertFalse(f1.is_legal(10.1))
self.assertFalse(f1.is_legal("AAA"))
self.assertFalse(f1.is_legal(dict()))
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
C = cs.add_hyperparameter(
UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default=1.0))
loss = cs.add_hyperparameter(
CategoricalHyperparameter(
"loss", ["epsilon_insensitive", "squared_epsilon_insensitive"],
default="squared_epsilon_insensitive"))
# Random Guess
epsilon = cs.add_hyperparameter(
UniformFloatHyperparameter(name="epsilon",
lower=0.001,
upper=1,
default=0.1,
log=True))
dual = cs.add_hyperparameter(Constant("dual", "False"))
# These are set ad-hoc
tol = cs.add_hyperparameter(
UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True))
fit_intercept = cs.add_hyperparameter(Constant("fit_intercept",
"True"))
intercept_scaling = cs.add_hyperparameter(
Constant("intercept_scaling", 1))
dual_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(loss, "epsilon_insensitive"))
cs.add_forbidden_clause(dual_and_loss)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# base_estimator = Constant(name="base_estimator", value="None")
n_estimators = cs.add_hyperparameter(UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default=50, log=False))
learning_rate = cs.add_hyperparameter(UniformFloatHyperparameter(
name="learning_rate", lower=0.0001, upper=2, default=0.1, log=True))
loss = cs.add_hyperparameter(CategoricalHyperparameter(
name="loss", choices=["linear", "square", "exponential"],
default="linear"))
max_depth = cs.add_hyperparameter(UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default=1, log=False))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
# Constrain the search space of the DeepFeedNet
# Fixed Architecture for MNIST
batch_size = Constant(name='batch_size', value=963)
# TODO: Decimal library to work around floating point issue
dropout_layer_1 = Constant(name='dropout_layer_1', value=0.39426633933)
dropout_output = Constant(name='dropout_output', value=0.085813712701)
num_layers = Constant(name='num_layers', value='c')
num_units_layer_1 = Constant(name='num_units_layer_1', value=1861)
number_updates = Constant(name='number_updates', value=1105)
std_layer_1 = Constant(name='std_layer_1', value=0.00351015701)
# To Optimize for all cases
l2 = UniformFloatHyperparameter("lambda2",
1e-6,
1e-2,
log=True,
default=1e-3)
lr = UniformFloatHyperparameter("learning_rate",
1e-4,
1e-1,
log=True,
default=1e-2)
cs = ConfigurationSpace()
cs.add_hyperparameter(batch_size)
cs.add_hyperparameter(number_updates)
cs.add_hyperparameter(num_layers)
cs.add_hyperparameter(num_units_layer_1)
cs.add_hyperparameter(dropout_layer_1)
cs.add_hyperparameter(dropout_output)
cs.add_hyperparameter(std_layer_1)
cs.add_hyperparameter(lr)
cs.add_hyperparameter(l2)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100))
criterion = cs.add_hyperparameter(Constant("criterion", "mse"))
max_features = cs.add_hyperparameter(
UniformFloatHyperparameter("max_features", 0.5, 5, default=1))
max_depth = cs.add_hyperparameter(
UnParametrizedHyperparameter(name="max_depth", value="None"))
min_samples_split = cs.add_hyperparameter(
UniformIntegerHyperparameter("min_samples_split", 2, 20,
default=2))
min_samples_leaf = cs.add_hyperparameter(
UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default=1))
# Unparametrized, we use min_samples as regularization
# max_leaf_nodes_or_max_depth = UnParametrizedHyperparameter(
# name="max_leaf_nodes_or_max_depth", value="max_depth")
# CategoricalHyperparameter("max_leaf_nodes_or_max_depth",
# choices=["max_leaf_nodes", "max_depth"], default="max_depth")
# min_weight_fraction_leaf = UniformFloatHyperparameter(
# "min_weight_fraction_leaf", 0.0, 0.1)
# max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes",
# value="None")
bootstrap = cs.add_hyperparameter(
CategoricalHyperparameter("bootstrap", ["True", "False"],
default="False"))
# Conditions
# Not applicable because max_leaf_nodes is no legal value of the parent
#cond_max_leaf_nodes_or_max_depth = \
# EqualsCondition(child=max_leaf_nodes,
# parent=max_leaf_nodes_or_max_depth,
# value="max_leaf_nodes")
#cond2_max_leaf_nodes_or_max_depth = \
# EqualsCondition(child=use_max_depth,
# parent=max_leaf_nodes_or_max_depth,
# value="max_depth")
#cond_max_depth = EqualsCondition(child=max_depth, parent=use_max_depth,
#value="True")
#cs.add_condition(cond_max_leaf_nodes_or_max_depth)
#cs.add_condition(cond2_max_leaf_nodes_or_max_depth)
#cs.add_condition(cond_max_depth)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
use_minimum_fraction = cs.add_hyperparameter(
CategoricalHyperparameter("use_minimum_fraction",
["True", "False"],
default="True"))
minimum_fraction = cs.add_hyperparameter(
UniformFloatHyperparameter("minimum_fraction",
lower=.0001,
upper=0.5,
default=0.01,
log=True))
cs.add_condition(
EqualsCondition(minimum_fraction, use_minimum_fraction, 'True'))
return cs
def test_not_equals_condition_illegal_value(self):
epsilon = UniformFloatHyperparameter("epsilon",
1e-5,
1e-1,
default=1e-4,
log=True)
loss = CategoricalHyperparameter(
"loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default="hinge")
self.assertRaisesRegexp(
ValueError, "Hyperparameter 'epsilon' is "
"conditional on the illegal value 'huber' of "
"its parent hyperparameter 'loss'", NotEqualsCondition, epsilon,
loss, "huber")
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
cs.add_hyperparameter(Constant("n_estimators", 100))
cs.add_hyperparameter(Constant("criterion", "mse"))
cs.add_hyperparameter(UniformFloatHyperparameter(
"max_features", 0.5, 5, default=1))
cs.add_hyperparameter(UnParametrizedHyperparameter("max_depth", "None"))
cs.add_hyperparameter(UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default=2))
cs.add_hyperparameter(UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default=1))
cs.add_hyperparameter(
UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.))
cs.add_hyperparameter(UnParametrizedHyperparameter("max_leaf_nodes", "None"))
cs.add_hyperparameter(CategoricalHyperparameter(
"bootstrap", ["True", "False"], default="True"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
percentile = UniformFloatHyperparameter(
name="percentile", lower=10, upper=90, default=50)
score_func = CategoricalHyperparameter(
name="score_func", choices=["chi2", "f_classif"], default="chi2")
if dataset_properties is not None:
# Chi2 can handle sparse data, so we respect this
if 'sparse' in dataset_properties and dataset_properties['sparse']:
score_func = Constant(
name="score_func", value="chi2")
cs = ConfigurationSpace()
cs.add_hyperparameter(percentile)
cs.add_hyperparameter(score_func)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
criterion = cs.add_hyperparameter(Constant('criterion', 'mse'))
splitter = cs.add_hyperparameter(Constant("splitter", "best"))
max_features = cs.add_hyperparameter(Constant('max_features', 1.0))
max_depth = cs.add_hyperparameter(UniformFloatHyperparameter(
'max_depth', 0., 2., default=0.5))
min_samples_split = cs.add_hyperparameter(UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default=2))
min_samples_leaf = cs.add_hyperparameter(UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default=1))
min_weight_fraction_leaf = cs.add_hyperparameter(
Constant("min_weight_fraction_leaf", 0.0))
max_leaf_nodes = cs.add_hyperparameter(
UnParametrizedHyperparameter("max_leaf_nodes", "None"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# the smoothing parameter is a non-negative float
# I will limit it to 1000 and put it on a logarithmic scale. (SF)
# Please adjust that, if you know a proper range, this is just a guess.
alpha = UniformFloatHyperparameter(name="alpha",
lower=1e-2,
upper=100,
default=1,
log=True)
fit_prior = CategoricalHyperparameter(name="fit_prior",
choices=["True", "False"],
default="True")
cs.add_hyperparameter(alpha)
cs.add_hyperparameter(fit_prior)
return cs
def read(pb_string):
searchspace = dict()
if type(pb_string) == file:
pb_string = pb_string.read()
if not isinstance(pb_string, str):
raise ValueError("Input is not a string or a file")
pb_space = Experiment()
text_format.Merge(pb_string, pb_space)
for para in pb_space.variable:
name = str(para.name)
size = para.size
if size != 1:
raise NotImplementedError(
"%s has size %s, we only convert with size 1" % (name, size))
if para.type == Experiment.ParameterSpec.ENUM:
choices = list()
for ch in para.options:
choices.append(str(ch))
searchspace[name] = CategoricalHyperparameter(name=name,
choices=choices,
conditions=None)
elif para.type == Experiment.ParameterSpec.FLOAT:
searchspace[name] = UniformFloatHyperparameter(name=name,
lower=para.min,
upper=para.max,
base=None,
q=None,
conditions=None)
elif para.type == Experiment.ParameterSpec.INT:
searchspace[name] = UniformIntegerHyperparameter(name=name,
lower=para.min,
upper=para.max,
base=None,
q=None,
conditions=None)
else:
raise NotImplementedError("Don't know that type: %s (%s)" %
(type(para), para.name))
return searchspace
def get_hyperparameter_search_space(dataset_properties=None):
alpha = UniformFloatHyperparameter(name="alpha",
lower=0.01,
upper=0.5,
default=0.1)
score_func = CategoricalHyperparameter(name="score_func",
choices=["chi2", "f_classif"],
default="chi2")
if dataset_properties is not None:
# Chi2 can handle sparse data, so we respect this
if 'sparse' in dataset_properties and dataset_properties['sparse']:
score_func = Constant(name="score_func", value="chi2")
mode = CategoricalHyperparameter('mode', ['fpr', 'fdr', 'fwe'], 'fpr')
cs = ConfigurationSpace()
cs.add_hyperparameter(alpha)
cs.add_hyperparameter(score_func)
cs.add_hyperparameter(mode)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_iter = cs.add_hyperparameter(
UnParametrizedHyperparameter("n_iter", value=300))
tol = cs.add_hyperparameter(
UniformFloatHyperparameter("tol",
10**-5,
10**-1,
default=10**-4,
log=True))
alpha_1 = cs.add_hyperparameter(
UniformFloatHyperparameter(name="alpha_1",
lower=10**-10,
upper=10**-3,
default=10**-6))
alpha_2 = cs.add_hyperparameter(
UniformFloatHyperparameter(name="alpha_2",
log=True,
lower=10**-10,
upper=10**-3,
default=10**-6))
lambda_1 = cs.add_hyperparameter(
UniformFloatHyperparameter(name="lambda_1",
log=True,
lower=10**-10,
upper=10**-3,
default=10**-6))
lambda_2 = cs.add_hyperparameter(
UniformFloatHyperparameter(name="lambda_2",
log=True,
lower=10**-10,
upper=10**-3,
default=10**-6))
threshold_lambda = cs.add_hyperparameter(
UniformFloatHyperparameter(name="threshold_lambda",
log=True,
lower=10**3,
upper=10**5,
default=10**4))
fit_intercept = cs.add_hyperparameter(
UnParametrizedHyperparameter("fit_intercept", "True"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
max_num_layers = 7 # Maximum number of layers coded
# Hacky way to condition layers params based on the number of layers
# 'c'=1, 'd'=2, 'e'=3 ,'f'=4', g ='5', h='6' + output_layer
layer_choices = [
chr(i) for i in range(ord('c'),
ord('b') + max_num_layers)
]
batch_size = UniformIntegerHyperparameter("batch_size",
32,
4096,
log=True,
default=32)
number_epochs = UniformIntegerHyperparameter("number_epochs",
2,
80,
default=5)
num_layers = CategoricalHyperparameter("num_layers",
choices=layer_choices,
default='c')
lr = UniformFloatHyperparameter("learning_rate",
1e-6,
1.0,
log=True,
default=0.01)
l2 = UniformFloatHyperparameter("lambda2",
1e-7,
1e-2,
log=True,
default=1e-4)
dropout_output = UniformFloatHyperparameter("dropout_output",
0.0,
0.99,
default=0.5)
# Define basic hyperparameters and define the config space
# basic means that are independent from the number of layers
cs = ConfigurationSpace()
cs.add_hyperparameter(number_epochs)
cs.add_hyperparameter(batch_size)
cs.add_hyperparameter(num_layers)
cs.add_hyperparameter(lr)
cs.add_hyperparameter(l2)
cs.add_hyperparameter(dropout_output)
# Define parameters with different child parameters and conditions
solver_choices = [
"adam", "adadelta", "adagrad", "sgd", "momentum", "nesterov",
"smorm3s"
]
solver = CategoricalHyperparameter(name="solver",
choices=solver_choices,
default="smorm3s")
beta1 = UniformFloatHyperparameter("beta1",
1e-4,
0.1,
log=True,
default=0.1)
beta2 = UniformFloatHyperparameter("beta2",
1e-4,
0.1,
log=True,
default=0.01)
rho = UniformFloatHyperparameter("rho",
0.05,
0.99,
log=True,
default=0.95)
momentum = UniformFloatHyperparameter("momentum",
0.3,
0.999,
default=0.9)
# TODO: Add policy based on this sklearn sgd
policy_choices = ['fixed', 'inv', 'exp', 'step']
lr_policy = CategoricalHyperparameter(name="lr_policy",
choices=policy_choices,
default='fixed')
gamma = UniformFloatHyperparameter(name="gamma",
lower=1e-3,
upper=1e-1,
default=1e-2)
power = UniformFloatHyperparameter("power", 0.0, 1.0, default=0.5)
epoch_step = UniformIntegerHyperparameter("epoch_step",
2,
20,
default=5)
cs.add_hyperparameter(solver)
cs.add_hyperparameter(beta1)
cs.add_hyperparameter(beta2)
cs.add_hyperparameter(momentum)
cs.add_hyperparameter(rho)
cs.add_hyperparameter(lr_policy)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(power)
cs.add_hyperparameter(epoch_step)
# Define parameters that are needed it for each layer
output_activation_choices = ['softmax', 'sigmoid', 'softplus', 'tanh']
activations_choices = [
'sigmoid', 'tanh', 'scaledTanh', 'elu', 'relu', 'leaky', 'linear'
]
weight_choices = [
'constant', 'normal', 'uniform', 'glorot_normal', 'glorot_uniform',
'he_normal', 'he_uniform', 'ortogonal', 'sparse'
]
# Iterate over parameters that are used in each layer
for i in range(1, max_num_layers):
layer_units = UniformIntegerHyperparameter("num_units_layer_" +
str(i),
64,
4096,
log=True,
default=128)
cs.add_hyperparameter(layer_units)
layer_dropout = UniformFloatHyperparameter("dropout_layer_" +
str(i),
0.0,
0.99,
default=0.5)
cs.add_hyperparameter(layer_dropout)
weight_initialization = CategoricalHyperparameter(
'weight_init_' + str(i),
choices=weight_choices,
default='he_normal')
cs.add_hyperparameter(weight_initialization)
layer_std = UniformFloatHyperparameter("std_layer_" + str(i),
1e-6,
0.1,
log=True,
default=0.005)
cs.add_hyperparameter(layer_std)
layer_activation = CategoricalHyperparameter(
"activation_layer_" + str(i),
choices=activations_choices,
default="relu")
cs.add_hyperparameter(layer_activation)
layer_leakiness = UniformFloatHyperparameter('leakiness_layer_' +
str(i),
0.01,
0.99,
default=0.3)
cs.add_hyperparameter(layer_leakiness)
layer_tanh_alpha = UniformFloatHyperparameter('tanh_alpha_layer_' +
str(i),
0.5,
1.0,
default=2. / 3.)
cs.add_hyperparameter(layer_tanh_alpha)
layer_tanh_beta = UniformFloatHyperparameter('tanh_beta_layer_' +
str(i),
1.1,
3.0,
log=True,
default=1.7159)
cs.add_hyperparameter(layer_tanh_beta)
# TODO: Could be in a function in a new module
for i in range(2, max_num_layers):
# Condition layers parameter on layer choice
layer_unit_param = cs.get_hyperparameter("num_units_layer_" +
str(i))
layer_cond = InCondition(child=layer_unit_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition dropout parameter on layer choice
layer_dropout_param = cs.get_hyperparameter("dropout_layer_" +
str(i))
layer_cond = InCondition(child=layer_dropout_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition weight initialization on layer choice
layer_weight_param = cs.get_hyperparameter("weight_init_" + str(i))
layer_cond = InCondition(child=layer_weight_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition std parameter on weight layer initialization choice
layer_std_param = cs.get_hyperparameter("std_layer_" + str(i))
weight_cond = EqualsCondition(child=layer_std_param,
parent=layer_weight_param,
value='normal')
cs.add_condition(weight_cond)
# Condition activation parameter on layer choice
layer_activation_param = cs.get_hyperparameter(
"activation_layer_" + str(i))
layer_cond = InCondition(child=layer_activation_param,
parent=num_layers,
values=[l for l in layer_choices[i - 1:]])
cs.add_condition(layer_cond)
# Condition leakiness on activation choice
layer_leakiness_param = cs.get_hyperparameter("leakiness_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_leakiness_param,
parent=layer_activation_param,
value='leaky')
cs.add_condition(activation_cond)
# Condition tanh on activation choice
layer_tanh_alpha_param = cs.get_hyperparameter(
"tanh_alpha_layer_" + str(i))
activation_cond = EqualsCondition(child=layer_tanh_alpha_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
layer_tanh_beta_param = cs.get_hyperparameter("tanh_beta_layer_" +
str(i))
activation_cond = EqualsCondition(child=layer_tanh_beta_param,
parent=layer_activation_param,
value='scaledTanh')
cs.add_condition(activation_cond)
# Conditioning on solver
momentum_depends_on_solver = InCondition(
momentum, solver, values=["momentum", "nesterov"])
beta1_depends_on_solver = EqualsCondition(beta1, solver, "adam")
beta2_depends_on_solver = EqualsCondition(beta2, solver, "adam")
rho_depends_on_solver = EqualsCondition(rho, solver, "adadelta")
cs.add_condition(momentum_depends_on_solver)
cs.add_condition(beta1_depends_on_solver)
cs.add_condition(beta2_depends_on_solver)
cs.add_condition(rho_depends_on_solver)
# Conditioning on learning rate policy
lr_policy_depends_on_solver = InCondition(
lr_policy, solver,
["adadelta", "adagrad", "sgd", "momentum", "nesterov"])
gamma_depends_on_policy = InCondition(child=gamma,
parent=lr_policy,
values=["inv", "exp", "step"])
power_depends_on_policy = EqualsCondition(power, lr_policy, "inv")
epoch_step_depends_on_policy = EqualsCondition(epoch_step, lr_policy,
"step")
cs.add_condition(lr_policy_depends_on_solver)
cs.add_condition(gamma_depends_on_policy)
cs.add_condition(power_depends_on_policy)
cs.add_condition(epoch_step_depends_on_policy)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
# Copied from libsvm_c
C = UniformFloatHyperparameter(name="C",
lower=0.03125,
upper=32768,
log=True,
default=1.0)
kernel = CategoricalHyperparameter(
name="kernel",
choices=['linear', 'poly', 'rbf', 'sigmoid'],
default="rbf")
degree = UniformIntegerHyperparameter(name="degree",
lower=1,
upper=5,
default=3)
# Changed the gamma value to 0.0 (is 0.1 for classification)
gamma = UniformFloatHyperparameter(name="gamma",
lower=3.0517578125e-05,
upper=8,
log=True,
default=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(name="coef0",
lower=-1,
upper=1,
default=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(name="shrinking",
choices=["True", "False"],
default="True")
tol = UniformFloatHyperparameter(name="tol",
lower=1e-5,
upper=1e-1,
default=1e-3,
log=True)
max_iter = UnParametrizedHyperparameter("max_iter", -1)
# Random Guess
epsilon = UniformFloatHyperparameter(name="epsilon",
lower=0.001,
upper=1,
default=0.1,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(C)
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(shrinking)
cs.add_hyperparameter(tol)
cs.add_hyperparameter(max_iter)
cs.add_hyperparameter(epsilon)
degree_depends_on_kernel = InCondition(child=degree,
parent=kernel,
values=('poly', 'rbf',
'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma,
parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0,
parent=kernel,
values=('poly', 'sigmoid'))
cs.add_condition(degree_depends_on_kernel)
cs.add_condition(gamma_depends_on_kernel)
cs.add_condition(coef0_depends_on_kernel)
return cs
