def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_iter = UnParametrizedHyperparameter("n_iter", value=300)
tol = UniformFloatHyperparameter("tol", 10 ** -5, 10 ** -1,
default_value=10 ** -3, log=True)
alpha_1 = UniformFloatHyperparameter(name="alpha_1", lower=10 ** -10,
upper=10 ** -3, default_value=10 ** -6)
alpha_2 = UniformFloatHyperparameter(name="alpha_2", log=True,
lower=10 ** -10, upper=10 ** -3,
default_value=10 ** -6)
lambda_1 = UniformFloatHyperparameter(name="lambda_1", log=True,
lower=10 ** -10, upper=10 ** -3,
default_value=10 ** -6)
lambda_2 = UniformFloatHyperparameter(name="lambda_2", log=True,
lower=10 ** -10, upper=10 ** -3,
default_value=10 ** -6)
threshold_lambda = UniformFloatHyperparameter(name="threshold_lambda",
log=True,
lower=10 ** 3,
upper=10 ** 5,
default_value=10 ** 4)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
cs.add_hyperparameters([n_iter, tol, alpha_1, alpha_2, lambda_1,
lambda_2, threshold_lambda, fit_intercept])
return cs
def test_write_forbidden(self):
cs = ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 2)
hp3 = UniformIntegerHyperparameter("child2", 0, 2)
hp4 = UniformIntegerHyperparameter("child3", 0, 2)
hp5 = CategoricalHyperparameter("child4", [4, 5, 6, 7])
cs.add_hyperparameters([hp1, hp2, hp3, hp4, hp5])
forb2 = ForbiddenEqualsClause(hp1, 1)
forb3 = ForbiddenInClause(hp2, range(2, 3))
forb4 = ForbiddenInClause(hp3, range(2, 3))
forb5 = ForbiddenInClause(hp4, range(2, 3))
forb6 = ForbiddenInClause(hp5, [6, 7])
and1 = ForbiddenAndConjunction(forb2, forb3)
and2 = ForbiddenAndConjunction(forb2, forb4)
and3 = ForbiddenAndConjunction(forb2, forb5)
cs.add_forbidden_clauses(
[forb2, forb3, forb4, forb5, forb6, and1, and2, and3])
irace.write(cs) # generates file called forbidden.txt
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
C = UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default_value=1.0)
loss = CategoricalHyperparameter(
"loss", ["epsilon_insensitive", "squared_epsilon_insensitive"],
default_value="squared_epsilon_insensitive")
# Random Guess
epsilon = UniformFloatHyperparameter(
name="epsilon", lower=0.001, upper=1, default_value=0.1, log=True)
dual = Constant("dual", "False")
# These are set ad-hoc
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
fit_intercept =Constant("fit_intercept", "True")
intercept_scaling = Constant("intercept_scaling", 1)
cs.add_hyperparameters([C, loss, epsilon, dual, tol, fit_intercept,
intercept_scaling])
dual_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(loss, "epsilon_insensitive")
)
cs.add_forbidden_clause(dual_and_loss)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = Constant("loss", "deviance")
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True)
n_estimators = UniformIntegerHyperparameter(
"n_estimators", 50, 500, default_value=100)
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default_value=3)
criterion = CategoricalHyperparameter(
'criterion', ['friedman_mse', 'mse', 'mae'],
default_value='mse')
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)
subsample = UniformFloatHyperparameter(
name="subsample", lower=0.01, upper=1.0, default_value=1.0)
max_features = UniformFloatHyperparameter(
"max_features", 0.1, 1.0 , default_value=1)
max_leaf_nodes = UnParametrizedHyperparameter(
name="max_leaf_nodes", value="None")
min_impurity_decrease = UnParametrizedHyperparameter(
name='min_impurity_decrease', value=0.0)
cs.add_hyperparameters([loss, learning_rate, n_estimators, max_depth,
criterion, min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, subsample,
max_features, max_leaf_nodes,
min_impurity_decrease])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter(
"criterion", ["gini", "entropy"], default_value="gini")
# The maximum number of features used in the forest is calculated as m^max_features, where
# m is the total number of features, and max_features is the hyperparameter specified below.
# The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This
# corresponds with Geurts' heuristic.
max_features = UniformFloatHyperparameter(
"max_features", 0., 1., default_value=0.5)
max_depth = UnParametrizedHyperparameter("max_depth", "None")
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_impurity_decrease = UnParametrizedHyperparameter('min_impurity_decrease', 0.0)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="True")
cs.add_hyperparameters([n_estimators, criterion, max_features,
max_depth, min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, max_leaf_nodes,
bootstrap, min_impurity_decrease])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
n_components = UniformIntegerHyperparameter(
"n_components", 50, 10000, default_value=100, log=True)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter(
"criterion", ["gini", "entropy"], default_value="gini")
max_features = UniformFloatHyperparameter("max_features", 0, 1,
default_value=0.5)
max_depth = UnParametrizedHyperparameter(name="max_depth", value="None")
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
'min_weight_fraction_leaf', 0.)
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="False")
cs.add_hyperparameters([n_estimators, criterion, max_features,
max_depth, max_leaf_nodes, min_samples_split,
min_samples_leaf, min_weight_fraction_leaf,
min_impurity_decrease, bootstrap])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter(
"loss", ["ls", "lad", "huber", "quantile"], default_value="ls")
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=1, default_value=0.1, log=True)
n_estimators = UniformIntegerHyperparameter(
"n_estimators", 50, 500, default_value=100)
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default_value=3)
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default_value=2, log=False)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default_value=1, log=False)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
"min_weight_fraction_leaf", 0.)
subsample = UniformFloatHyperparameter(
name="subsample", lower=0.01, upper=1.0, default_value=1.0, log=False)
max_features = UniformFloatHyperparameter(
"max_features", 0.1, 1.0, default_value=1)
max_leaf_nodes = UnParametrizedHyperparameter(
name="max_leaf_nodes", value="None")
min_impurity_decrease = UnParametrizedHyperparameter(
name='min_impurity_decrease', value=0.0)
alpha = UniformFloatHyperparameter(
"alpha", lower=0.75, upper=0.99, default_value=0.9)
cs.add_hyperparameters([loss, learning_rate, n_estimators, max_depth,
min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, subsample, max_features,
max_leaf_nodes, min_impurity_decrease, alpha])
cs.add_condition(InCondition(alpha, loss, ['huber', 'quantile']))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
gamma = UniformFloatHyperparameter(
"gamma", 3.0517578125e-05, 8, default_value=1.0, log=True)
n_components = UniformIntegerHyperparameter(
"n_components", 50, 10000, default_value=100, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([gamma, n_components])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
alpha = UniformFloatHyperparameter(
"alpha", 10 ** -5, 10., log=True, default_value=1.)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1,
default_value=1e-3, log=True)
cs.add_hyperparameters([alpha, fit_intercept, tol])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
use_minimum_fraction = CategoricalHyperparameter(
"use_minimum_fraction", ["True", "False"], default_value="True")
minimum_fraction = UniformFloatHyperparameter(
"minimum_fraction", lower=.0001, upper=0.5, default_value=0.01, log=True)
cs.add_hyperparameters([use_minimum_fraction, minimum_fraction])
cs.add_condition(EqualsCondition(minimum_fraction,
use_minimum_fraction, 'True'))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
percentile = UniformFloatHyperparameter(
"percentile", lower=1, upper=99, default_value=50)
score_func = CategoricalHyperparameter(
name="score_func", choices=["f_regression", "mutual_info"])
cs = ConfigurationSpace()
cs.add_hyperparameters([percentile, score_func])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
q_min = UniformFloatHyperparameter(
'q_min', 0.001, 0.3, default_value=0.25
)
q_max = UniformFloatHyperparameter(
'q_max', 0.7, 0.999, default_value=0.75
)
cs.add_hyperparameters((q_min, q_max))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_neighbors = UniformIntegerHyperparameter(
name="n_neighbors", lower=1, upper=100, log=True, default_value=1)
weights = CategoricalHyperparameter(
name="weights", choices=["uniform", "distance"], default_value="uniform")
p = CategoricalHyperparameter(name="p", choices=[1, 2], default_value=2)
cs.add_hyperparameters([n_neighbors, weights, p])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
alpha = UniformFloatHyperparameter(
name="alpha", lower=1e-14, upper=1.0, default_value=1e-8, log=True)
thetaL = UniformFloatHyperparameter(
name="thetaL", lower=1e-10, upper=1e-3, default_value=1e-6, log=True)
thetaU = UniformFloatHyperparameter(
name="thetaU", lower=1.0, upper=100000, default_value=100000.0, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([alpha, thetaL, thetaU])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# TODO parametrize like the Random Forest as n_quantiles = n_features^param
n_quantiles = UniformIntegerHyperparameter(
'n_quantiles', lower=10, upper=2000, default_value=1000
)
output_distribution = CategoricalHyperparameter(
'output_distribution', ['uniform', 'normal']
)
cs.add_hyperparameters((n_quantiles, output_distribution))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
# More than degree 3 is too expensive!
degree = UniformIntegerHyperparameter("degree", 2, 3, 2)
interaction_only = CategoricalHyperparameter("interaction_only",
["False", "True"], "False")
include_bias = CategoricalHyperparameter("include_bias",
["True", "False"], "True")
cs = ConfigurationSpace()
cs.add_hyperparameters([degree, interaction_only, include_bias])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss",
["squared_loss", "huber", "epsilon_insensitive",
"squared_epsilon_insensitive"],
default_value="squared_loss")
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1., log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter(
"fit_intercept", "True")
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=0.1, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter(
"power_t", 1e-5, 1, default_value=0.25)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
tol, epsilon, learning_rate, eta0,
power_t, average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = InCondition(epsilon, loss,
["huber", "epsilon_insensitive", "squared_epsilon_insensitive"])
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default_value=50, log=False)
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True)
algorithm = CategoricalHyperparameter(
name="algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R")
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters([n_estimators, learning_rate, algorithm, max_depth])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# the smoothing parameter is a non-negative float
# I will limit it to 1000 and put it on a logarithmic scale. (SF)
# Please adjust that, if you know a proper range, this is just a guess.
alpha = UniformFloatHyperparameter(name="alpha", lower=1e-2, upper=100,
default_value=1, log=True)
fit_prior = CategoricalHyperparameter(name="fit_prior",
choices=["True", "False"],
default_value="True")
cs.add_hyperparameters([alpha, fit_prior])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
algorithm = CategoricalHyperparameter('algorithm',
['parallel', 'deflation'], 'parallel')
whiten = CategoricalHyperparameter('whiten',
['False', 'True'], 'False')
fun = CategoricalHyperparameter(
'fun', ['logcosh', 'exp', 'cube'], 'logcosh')
cs.add_hyperparameters([n_components, algorithm, whiten, fun])
cs.add_condition(EqualsCondition(n_components, whiten, "True"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
C = UniformFloatHyperparameter("C", 1e-5, 10, 1.0, log=True)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
loss = CategoricalHyperparameter(
"loss", ["hinge", "squared_hinge"], default_value="hinge"
)
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-4,
log=True)
# Note: Average could also be an Integer if > 1
average = CategoricalHyperparameter('average', ['False', 'True'],
default_value='False')
cs = ConfigurationSpace()
cs.add_hyperparameters([loss, fit_intercept, tol, C, average])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_clusters = UniformIntegerHyperparameter("n_clusters", 2, 400, 25)
affinity = CategoricalHyperparameter(
"affinity", ["euclidean", "manhattan", "cosine"], "euclidean")
linkage = CategoricalHyperparameter(
"linkage", ["ward", "complete", "average"], "ward")
pooling_func = CategoricalHyperparameter(
"pooling_func", ["mean", "median", "max"])
cs.add_hyperparameters([n_clusters, affinity, linkage, pooling_func])
affinity_and_linkage = ForbiddenAndConjunction(
ForbiddenInClause(affinity, ["manhattan", "cosine"]),
ForbiddenEqualsClause(linkage, "ward"))
cs.add_forbidden_clause(affinity_and_linkage)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
kernel = CategoricalHyperparameter('kernel',
['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=1.0)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# base_estimator = Constant(name="base_estimator", value="None")
n_estimators = UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default_value=50,
log=False)
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=2, default_value=0.1,
log=True)
loss = CategoricalHyperparameter(
name="loss", choices=["linear", "square", "exponential"],
default_value="linear")
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters([n_estimators, learning_rate, loss, max_depth])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = Constant("loss", "deviance")
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=1,
default_value=0.1,
log=True)
n_estimators = UniformIntegerHyperparameter("n_estimators",
50,
500,
default_value=100)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default_value=3)
criterion = CategoricalHyperparameter('criterion',
['friedman_mse', 'mse', 'mae'],
default_value='mse')
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
"min_weight_fraction_leaf", 0.)
subsample = UniformFloatHyperparameter(name="subsample",
lower=0.01,
upper=1.0,
default_value=1.0)
max_features = UniformFloatHyperparameter("max_features",
0.1,
1.0,
default_value=1)
max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes",
value="None")
min_impurity_decrease = UnParametrizedHyperparameter(
name='min_impurity_decrease', value=0.0)
cs.add_hyperparameters([
loss, learning_rate, n_estimators, max_depth, criterion,
min_samples_split, min_samples_leaf, min_weight_fraction_leaf,
subsample, max_features, max_leaf_nodes, min_impurity_decrease
])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default_value="log")
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1, log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=1e-4, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1,
default_value=0.5)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
epsilon, learning_rate, eta0, power_t,
average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
Window_size = UniformIntegerHyperparameter(
name="Window_size", lower=5, upper=50, default_value=20)
Difference = CategoricalHyperparameter(
name="Difference", choices=["True", "False"], default_value="True")
tsfresh_feature = CategoricalHyperparameter(
name="tsfresh_feature", choices=["True", "False"], default_value="True")
n_estimators = UniformIntegerHyperparameter("n_estimators", 10, 200, default_value=100)
criterion = CategoricalHyperparameter("criterion",
['mse', 'friedman_mse', 'mae'], default_value='mse')
max_features = UniformFloatHyperparameter(
"max_features", 0.1, 1.0, default_value=1.0)
max_depth = UnParametrizedHyperparameter("max_depth", "None")
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.0)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="True")
cs.add_hyperparameters([n_estimators, criterion, max_features, Difference,
max_depth, min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, max_leaf_nodes,
min_impurity_decrease, bootstrap, tsfresh_feature, Window_size])
return cs
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
if optimizer == 'smac':
alpha = UniformFloatHyperparameter("alpha",
0.01,
32,
log=True,
default_value=1.0)
tol = UniformFloatHyperparameter("tol",
1e-6,
1e-2,
default_value=1e-4,
log=True)
max_iter = UniformFloatHyperparameter("max_iter",
100,
1000,
q=100,
default_value=100)
solver = CategoricalHyperparameter("solver",
choices=["auto", "saga"],
default_value="auto")
cs = ConfigurationSpace()
cs.add_hyperparameters([alpha, tol, max_iter, solver])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'alpha': hp.loguniform('ridge_alpha', np.log(0.01),
np.log(32)),
'tol': hp.loguniform('ridge_tol', np.log(1e-6), np.log(1e-2)),
'max_iter': hp.uniform('ridge_max_iter', 100, 1000),
'solver': hp.choice('ridge_solver', ["auto", "saga"])
}
init_trial = {
'alpha': 1,
'tol': 1e-4,
'max_iter': 100,
'solver': "auto"
}
return space
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
solver = CategoricalHyperparameter(name="solver",
choices=['svd', 'lsqr', 'eigen'],
default_value='svd')
shrinkage = UniformFloatHyperparameter(name="shrinkage",
lower=0.0,
upper=1.0,
default_value=0.5)
n_components = UniformIntegerHyperparameter(name="n_components",
lower=1,
upper=29,
default_value=10)
tol = UniformFloatHyperparameter(name="tol",
lower=0.0001,
upper=1,
default_value=0.0001)
cs.add_hyperparameters([solver, shrinkage, n_components, tol])
return cs
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators",
50,
500,
default_value=50,
log=False)
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=2.,
default_value=1.0,
log=True)
algorithm = CategoricalHyperparameter("algorithm",
["SAMME", "SAMME.R"],
default_value="SAMME.R")
cs.add_hyperparameters([learning_rate, n_estimators, algorithm])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# the smoothing parameter is a non-negative float
# I will limit it to 100 and put it on a logarithmic scale. (SF)
# Please adjust that, if you know a proper range, this is just a guess.
alpha = UniformFloatHyperparameter(name="alpha",
lower=1e-2,
upper=100,
default_value=1,
log=True)
fit_prior = CategoricalHyperparameter(name="fit_prior",
choices=["True", "False"],
default_value="True")
cs.add_hyperparameters([alpha, fit_prior])
return cs
def get_hyperparameter_search_space(
dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None,
use_augmenter: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="use_augmenter",
value_range=(True, False),
default_value=True,
),
sigma_offset: HyperparameterSearchSpace = HyperparameterSearchSpace(hyperparameter="sigma_offset",
value_range=(0.0, 3.0),
default_value=0.3,
),
) -> ConfigurationSpace:
cs = ConfigurationSpace()
use_augmenter = get_hyperparameter(use_augmenter, CategoricalHyperparameter)
sigma_offset = get_hyperparameter(sigma_offset, UniformFloatHyperparameter)
cs.add_hyperparameters([use_augmenter, sigma_offset])
# only add hyperparameters to configuration space if we are using the augmenter
cs.add_condition(CS.EqualsCondition(sigma_offset, use_augmenter, True))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
percentile = UniformFloatHyperparameter(name="percentile",
lower=1,
upper=99,
default_value=50)
score_func = CategoricalHyperparameter(
name="score_func",
choices=["chi2", "f_classif", "mutual_info"],
default_value="chi2")
if dataset_properties is not None:
# Chi2 can handle sparse data, so we respect this
if 'sparse' in dataset_properties and dataset_properties['sparse']:
score_func = Constant(name="score_func", value="chi2")
cs = ConfigurationSpace()
cs.add_hyperparameters([percentile, score_func])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_components = UniformIntegerHyperparameter("n_components",
10,
2000,
default_value=100)
algorithm = CategoricalHyperparameter('algorithm',
['parallel', 'deflation'],
'parallel')
whiten = CategoricalHyperparameter('whiten', ['False', 'True'],
'False')
fun = CategoricalHyperparameter('fun', ['logcosh', 'exp', 'cube'],
'logcosh')
cs.add_hyperparameters([n_components, algorithm, whiten, fun])
cs.add_condition(EqualsCondition(n_components, whiten, "True"))
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
epsilon = CategoricalHyperparameter("epsilon", [1e-4, 1e-3, 1e-2, 1e-1, 1], default_value=1e-4)
loss = CategoricalHyperparameter(
"loss", ["epsilon_insensitive", "squared_epsilon_insensitive"], default_value="epsilon_insensitive")
dual = CategoricalHyperparameter("dual", ['True', 'False'], default_value='True')
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
C = UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default_value=1.0)
fit_intercept = Constant("fit_intercept", "True")
intercept_scaling = Constant("intercept_scaling", 1)
cs.add_hyperparameters([epsilon, loss, dual, tol, C,
fit_intercept, intercept_scaling])
dual_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(loss, "epsilon_insensitive")
)
cs.add_forbidden_clause(dual_and_loss)
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'loss': hp.choice('liblinear_combination', [{'loss': "epsilon_insensitive", 'dual': "True"},
{'loss': "squared_epsilon_insensitive",
'dual': "True"},
{'loss': "squared_epsilon_insensitive",
'dual': "False"}]),
'dual': None,
'tol': hp.loguniform('liblinear_tol', np.log(1e-5), np.log(1e-1)),
'C': hp.loguniform('liblinear_C', np.log(0.03125), np.log(32768)),
'fit_intercept': hp.choice('liblinear_fit_intercept', ["True"]),
'intercept_scaling': hp.choice('liblinear_intercept_scaling', [1])}
init_trial = {'loss': {'loss': "epsilon_insensitive", 'dual': "True"},
'tol': 1e-4,
'C': 1,
'fit_intercept': "True",
'intercept_scaling': 1}
return space
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# n_estimators = Constant("n_estimators", 100)
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=10,
upper=600,
default_value=100)
criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"],
default_value="gini")
# The maximum number of features used in the forest is calculated as m^max_features, where
# m is the total number of features, and max_features is the hyperparameter specified below.
# The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This
# corresponds with Geurts' heuristic.
max_features = UniformFloatHyperparameter("max_features",
0.,
1.,
default_value=0.5)
max_depth = UnParametrizedHyperparameter("max_depth", "None")
# max_depth = UniformIntegerHyperparameter("max_depth", 1, 20, default_value=10)
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
0,
20,
default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
"min_weight_fraction_leaf", 0.)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.0)
bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"],
default_value="True")
cs.add_hyperparameters([
n_estimators, criterion, max_features, max_depth,
min_samples_split, min_samples_leaf, min_weight_fraction_leaf,
max_leaf_nodes, bootstrap, min_impurity_decrease
])
return cs
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter("shrinkage",
["None", "auto", "manual"],
default_value="None")
shrinkage_factor = UniformFloatHyperparameter(
"shrinkage_factor", 0., 1., 0.5)
n_components = UniformIntegerHyperparameter('n_components',
1,
250,
default_value=1)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-4,
log=True)
cs.add_hyperparameters(
[shrinkage, shrinkage_factor, n_components, tol])
cs.add_condition(
EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {
'n_components':
hp.randint('lda_n_components', 250) + 1,
'tol':
hp.loguniform('lda_tol', np.log(1e-5), np.log(1e-1)),
'shrinkage':
hp.choice('lda_shrinkage', [
"None", "auto",
("manual", {
'shrinkage_factor':
hp.uniform('lda_shrinkage_factor', 0, 1)
})
])
}
init_trial = {'n_components': 10, 'tol': 1e-4, 'shrinkage': "None"}
return space
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter("penalty", ["l1", "l2"],
default="l2")
loss = CategoricalHyperparameter("loss", ["hinge", "squared_hinge"],
default="squared_hinge")
dual = Constant("dual", "False")
# This is set ad-hoc
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default=1e-4,
log=True)
C = UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default=1.0)
multi_class = Constant("multi_class", "ovr")
# These are set ad-hoc
fit_intercept = Constant("fit_intercept", "True")
intercept_scaling = Constant("intercept_scaling", 1)
cs.add_hyperparameters([
penalty, loss, dual, tol, C, multi_class, fit_intercept,
intercept_scaling
])
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge"))
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge"))
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l1"))
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
# Copied from libsvm_c
C = UniformFloatHyperparameter(
name="C", lower=0.03125, upper=32768, log=True, default=1.0)
kernel = CategoricalHyperparameter(
name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'],
default="rbf")
degree = UniformIntegerHyperparameter(
name="degree", lower=1, upper=5, default=3)
# Changed the gamma value to 0.0 (is 0.1 for classification)
gamma = UniformFloatHyperparameter(
name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(
name="coef0", lower=-1, upper=1, default=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(
name="shrinking", choices=["True", "False"], default="True")
tol = UniformFloatHyperparameter(
name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True)
max_iter = UnParametrizedHyperparameter("max_iter", -1)
# Random Guess
epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001,
upper=1, default=0.1, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter, epsilon])
degree_depends_on_kernel = InCondition(child=degree, parent=kernel,
values=('poly', 'rbf', 'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel,
values=('poly', 'sigmoid'))
cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel,
coef0_depends_on_kernel])
return cs
def get_cs():
epsilon = CategoricalHyperparameter("epsilon",
[1e-4, 1e-3, 1e-2, 1e-1, 1],
default_value=1e-4)
C = UniformFloatHyperparameter("C",
0.03125,
32768,
log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 2000)
cs = ConfigurationSpace()
cs.add_hyperparameters([
epsilon, C, kernel, degree, gamma, coef0, shrinking, tol, max_iter
])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
def get_cs():
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators",
100,
1000,
default_value=500)
num_leaves = UniformIntegerHyperparameter("num_leaves",
31,
1023,
default_value=31)
learning_rate = UniformFloatHyperparameter("learning_rate",
0.025,
0.3,
default_value=0.1,
log=True)
min_child_weight = UniformIntegerHyperparameter("min_child_weight",
1,
10,
default_value=1)
subsample = UniformFloatHyperparameter("subsample",
0.5,
1,
default_value=1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.5,
1,
default_value=1)
reg_alpha = UniformFloatHyperparameter('reg_alpha',
1e-10,
10,
log=True,
default_value=1e-10)
reg_lambda = UniformFloatHyperparameter("reg_lambda",
1e-10,
10,
log=True,
default_value=1e-10)
cs.add_hyperparameters([
n_estimators, num_leaves, learning_rate, min_child_weight,
subsample, colsample_bytree, reg_alpha, reg_lambda
])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter(
"criterion", ["gini", "entropy"], default_value="gini")
max_features = UniformFloatHyperparameter("max_features", 0, 1,
default_value=0.5, q=0.05)
max_depth = UnParametrizedHyperparameter(name="max_depth", value="None")
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
'min_weight_fraction_leaf', 0.)
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="False")
cs.add_hyperparameters([n_estimators, criterion, max_features,
max_depth, max_leaf_nodes, min_samples_split,
min_samples_leaf, min_weight_fraction_leaf,
min_impurity_decrease, bootstrap])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': 100,
'criterion': hp.choice('ets_criterion', ['gini', 'entropy']),
'max_features': hp.uniform('ets_max_features', 0, 1),
'max_depth': "None",
'max_leaf_nodes': "None",
'min_samples_leaf': hp.randint('ets_samples_leaf', 20) + 1,
'min_samples_split': hp.randint('ets_samples_split', 19) + 2,
'min_weight_fraction_leaf': 0.,
'min_impurity_decrease': 0.,
'bootstrap': hp.choice('ets_bootstrap', ['True', 'False'])}
return space
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default_value=50, log=False)
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=2, default_value=0.1, log=True)
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters([n_estimators, learning_rate, max_depth])
return cs
elif optimizer == 'tpe':
space = {'n_estimators': hp.randint('ab_n_estimators', 451) + 50,
'learning_rate': hp.loguniform('ab_learning_rate', np.log(0.01), np.log(2)),
'max_depth': hp.randint('ab_max_depth', 10) + 1}
init_trial = {'n_estimators': 50, 'learning_rate': 0.1, 'algorithm': "SAMME.R", 'max_depth': 1}
return space
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
alpha = UniformFloatHyperparameter("alpha", 0.01, 32, log=True, default_value=1.0)
tol = UniformFloatHyperparameter("tol", 1e-6, 1e-2, default_value=1e-4,
log=True)
max_iter = UniformFloatHyperparameter("max_iter", 100, 1000, q=100, default_value=100)
cs = ConfigurationSpace()
cs.add_hyperparameters([alpha, tol, max_iter])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'alpha': hp.loguniform('lasso_alpha', np.log(0.01), np.log(32)),
'tol': hp.loguniform('lasso_tol', np.log(1e-6), np.log(1e-2)),
'max_iter': hp.uniform('lasso_max_iter', 100, 1000)}
init_trial = {'alpha': 1, 'tol': 1e-4, 'max_iter': 100}
return space
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
learning_rate = UniformFloatHyperparameter(name="learning_rate", lower=0.01, upper=1, default_value=0.1,
log=True)
min_samples_leaf = UniformFloatHyperparameter("min_samples_leaf_factor", 0.0001, 0.25, default_value=0.0001,
log=True)
max_depth_factor = UniformFloatHyperparameter("max_depth_factor", 1e-5, 2.5, default_value=1.)
max_leaf_nodes_factor = UniformFloatHyperparameter("max_leaf_nodes_factor", 1e-5, 1., default_value=1.)
l2_regularization = UniformFloatHyperparameter(name="l2_regularization", lower=1e-10, upper=1., log=True,
default_value=1e-10)
n_iter_no_change = UniformIntegerHyperparameter(name="n_iter_no_change", lower=1, upper=20, default_value=10)
validation_fraction = UniformFloatHyperparameter(name="validation_fraction", lower=0.01, upper=0.4,
default_value=0.1)
cs.add_hyperparameters(
[learning_rate, min_samples_leaf, max_depth_factor, max_leaf_nodes_factor, l2_regularization,
n_iter_no_change, validation_fraction])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
alpha = UniformFloatHyperparameter(name="alpha",
lower=1e-14,
upper=1.0,
default_value=1e-8,
log=True)
thetaL = UniformFloatHyperparameter(name="thetaL",
lower=1e-10,
upper=1e-3,
default_value=1e-6,
log=True)
thetaU = UniformFloatHyperparameter(name="thetaU",
lower=1.0,
upper=100000,
default_value=100000.0,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([alpha, thetaL, thetaU])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter(
"gamma",
3.0517578125e-05, 8,
log=True,
default_value=1.0,
)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_cs():
cs = ConfigurationSpace()
shrinkage = CategoricalHyperparameter("shrinkage",
["None", "auto", "manual"],
default_value="None")
shrinkage_factor = UniformFloatHyperparameter("shrinkage_factor", 0., 1.,
0.5)
n_components = UniformIntegerHyperparameter('n_components',
1,
250,
default_value=10)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-4,
log=True)
cs.add_hyperparameters([shrinkage, shrinkage_factor, n_components, tol])
cs.add_condition(EqualsCondition(shrinkage_factor, shrinkage, "manual"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
nugget = UniformFloatHyperparameter(name="nugget",
lower=0.0001,
upper=10,
default=0.1,
log=True)
thetaL = UniformFloatHyperparameter(name="thetaL",
lower=1e-6,
upper=1e-3,
default=1e-4,
log=True)
thetaU = UniformFloatHyperparameter(name="thetaU",
lower=0.2,
upper=10,
default=1.0,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([nugget, thetaL, thetaU])
return cs
def get_adaboost_default_search_space(dataset_properties=None):
classif_prefix = 'classifier:adaboost:'
cs = ConfigurationSpace()
model_type = CategoricalHyperparameter('classifier:__choice__', ['adaboost'])
imputation = CategoricalHyperparameter('imputation:strategy', ['mean', 'median', 'most_frequent'])
n_estimators = UniformIntegerHyperparameter(
name=classif_prefix + "n_estimators", lower=50, upper=500, default_value=50, log=False)
learning_rate = UniformFloatHyperparameter(
name=classif_prefix + "learning_rate", lower=0.01, upper=2, default_value=0.1, log=True)
algorithm = CategoricalHyperparameter(
name=classif_prefix + "algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R")
max_depth = UniformIntegerHyperparameter(
name=classif_prefix + "max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters([model_type, imputation, n_estimators, learning_rate, algorithm, max_depth])
return cs
def make_config_space(dataset_shape: tuple) -> ConfigurationSpace:
r"""Build a ConfigurationSpace object encompassing all different autoencoder types and parameters.
The resulting ConfigurationSpace object depends on the given dataset shape.
Adapting it to different datasets requires manual adjustments (see :func:`build_autoencoder_from_existing`).
:param dataset_shape: Shape of the dataset this ConfigurationSpace should be valid for.
:return: A ConfigurationSpace object tied to the given dataset size.
.. note:
Changes to this function invalidate previous optimization results.
"""
input_dim = dataset_shape[-1]
# Build Configuration Space which defines all parameters and their ranges
cs = ConfigurationSpace()
ae_type = CategoricalHyperparameter('ae_type', ['deep', 'deep_ksparse'],
default_value='deep_ksparse')
act_type = CategoricalHyperparameter('act_type',
['relu', 'sigmoid', 'tanh'],
default_value='relu')
epochs = UniformIntegerHyperparameter('epochs', 1, 50, default_value=10)
cs.add_hyperparameters([ae_type, act_type, epochs])
num_layers = UniformIntegerHyperparameter('num_hidden_layers',
1,
5,
default_value=3)
latent_dim = UniformIntegerHyperparameter('latent_dim',
2,
input_dim // 2,
default_value=2)
cs.add_hyperparameters([num_layers, latent_dim])
ksparse_k = UniformIntegerHyperparameter('k', 50, 200, default_value=200)
cs.add_hyperparameter(ksparse_k)
cs.add_condition(
InCondition(child=ksparse_k, parent=ae_type, values=['deep_ksparse']))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
#Parametrized
learning_rate = UniformFloatHyperparameter(
name='learning_rate', lower=0.01, upper=1, default=0.1)
feature_fraction = UniformFloatHyperparameter(
name='feature_fraction', lower=0.5, upper=1.0, default=0.9)
num_leaves = UniformIntegerHyperparameter(
name='num_leaves', lower=5, upper=500, default=31)
min_data_in_leaf = UniformIntegerHyperparameter(
name='min_data_in_leaf', lower=1, upper=30, default=20)
lambda_l1 = UniformFloatHyperparameter(
name='lambda_l1', lower=0.0, upper=10.0, default=0.0)
lambda_l2 = UniformFloatHyperparameter(
name='lambda_l2', lower=0.0, upper=10.0, default=0.0)
max_bin = UniformIntegerHyperparameter(
name='max_bin', lower=50, upper=500, default=255)
#UnParametrized
bagging_fraction = UnParametrizedHyperparameter(
name='bagging_fraction', value=1.0)
bagging_freq = UnParametrizedHyperparameter(
name='bagging_freq', value=5)
max_depth = UnParametrizedHyperparameter(
name='max_depth', value=-1)
min_data_in_bin = UnParametrizedHyperparameter(
name='min_data_in_bin', value=5)
min_gain_to_split = UnParametrizedHyperparameter(
name='min_gain_to_split', value=0)
min_sum_hessian_in_leaf = UnParametrizedHyperparameter(
name='min_sum_hessian_in_leaf', value=1e-3)
cs.add_hyperparameters([learning_rate, feature_fraction, bagging_fraction,
num_leaves, min_data_in_leaf, lambda_l1,
lambda_l2, min_data_in_bin, max_depth, max_bin,
min_sum_hessian_in_leaf, min_gain_to_split])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
C = UniformFloatHyperparameter(
name="C", lower=0.03125, upper=32768, log=True, default_value=1.0)
# Random Guess
epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001,
upper=1, default_value=0.1,
log=True)
kernel = CategoricalHyperparameter(
name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'],
default_value="rbf")
degree = UniformIntegerHyperparameter(
name="degree", lower=2, upper=5, default_value=3)
gamma = UniformFloatHyperparameter(
name="gamma", lower=3.0517578125e-05, upper=8, log=True, default_value=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(
name="coef0", lower=-1, upper=1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(
name="shrinking", choices=["True", "False"], default_value="True")
tol = UniformFloatHyperparameter(
name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True)
max_iter = UnParametrizedHyperparameter("max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter, epsilon])
degree_depends_on_kernel = InCondition(child=degree, parent=kernel,
values=('poly', 'rbf', 'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel,
values=('poly', 'sigmoid'))
cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel,
coef0_depends_on_kernel])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
Window_size = UniformIntegerHyperparameter(name="Window_size",
lower=5,
upper=50,
default_value=20)
Difference = CategoricalHyperparameter(name="Difference",
choices=["True", "False"],
default_value="True")
tsfresh_feature = CategoricalHyperparameter(name="tsfresh_feature",
choices=["True", "False"],
default_value="True")
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=50,
upper=500,
default_value=50,
log=False)
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=2,
default_value=0.1,
log=True)
loss = CategoricalHyperparameter(
name="loss",
choices=["linear", "square", "exponential"],
default_value="linear")
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=1,
upper=10,
default_value=3,
log=False)
cs.add_hyperparameters([
n_estimators, Difference, learning_rate, loss, max_depth,
Window_size, tsfresh_feature
])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default_value=50, log=False)
sampling_strategy = CategoricalHyperparameter(
name="sampling_strategy", choices=["majority", "not minority", "not majority", "all"],
default_value="not minority")
replacement = CategoricalHyperparameter(
"replacement", ["True", "False"], default_value="False")
ab_n_estimators = UniformIntegerHyperparameter(
name="ab_n_estimators", lower=50, upper=500, default_value=50, log=False)
ab_learning_rate = UniformFloatHyperparameter(
name="ab_learning_rate", lower=0.01, upper=2, default_value=0.1, log=True)
ab_algorithm = CategoricalHyperparameter(
name="ab_algorithm", choices=["SAMME.R", "SAMME"], default_value="SAMME.R")
ab_max_depth = UniformIntegerHyperparameter(
name="ab_max_depth", lower=1, upper=10, default_value=1, log=False)
cs.add_hyperparameters([n_estimators, sampling_strategy, replacement, ab_n_estimators,
ab_learning_rate, ab_algorithm, ab_max_depth])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': hp.randint('easy_ensemble_n_estimators', 451) + 50,
'sampling_strategy': hp.choice('easy_ensemble_sampling_strategy',
["majority", "not minority", "not majority", "all"]),
'replacement': hp.choice('easy_ensemble_replacement', ["True", "False"]),
'ab_n_estimators': hp.randint('ab_n_estimators', 451) + 50,
'ab_learning_rate': hp.loguniform('ab_learning_rate', np.log(0.01), np.log(2)),
'ab_algorithm': hp.choice('ab_algorithm', ["SAMME.R", "SAMME"]),
'ab_max_depth': hp.randint('ab_max_depth', 10) + 1}
init_trial = {'n_estimators': 10,
'sampling_strategy': "not minority",
'replacement': "False",
'ab_n_estimators': 50,
'ab_learning_rate': 0.1,
'ab_algorithm': "SAMME.R",
'ab_max_depth': 1}
return space
def get_hyperparameter_search_space(dataset_properties=None):
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=10, upper=100,
default_value=10)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=2, upper=10,
default_value=5)
min_samples_split = UniformIntegerHyperparameter(name="min_samples_split",
lower=2, upper=20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(name="min_samples_leaf",
lower=1, upper=20,
default_value=1)
min_weight_fraction_leaf = Constant('min_weight_fraction_leaf', 1.0)
max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes",
value="None")
bootstrap = CategoricalHyperparameter('bootstrap', ['True', 'False'])
cs = ConfigurationSpace()
cs.add_hyperparameters([n_estimators, max_depth, min_samples_split,
min_samples_leaf, min_weight_fraction_leaf,
max_leaf_nodes, bootstrap])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
criterion = CategoricalHyperparameter(
"criterion", ["gini", "entropy"], default_value="gini")
max_depth = UniformFloatHyperparameter(
'max_depth', 0., 2., default_value=0.5)
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = Constant("min_weight_fraction_leaf", 0.0)
max_features = UnParametrizedHyperparameter('max_features', 1.0)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_impurity_decrease = UnParametrizedHyperparameter('min_impurity_decrease', 0.0)
cs.add_hyperparameters([criterion, max_features, max_depth,
min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, max_leaf_nodes,
min_impurity_decrease])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2"], default_value="l2")
loss = CategoricalHyperparameter(
"loss", ["hinge", "squared_hinge"], default_value="squared_hinge")
dual = Constant("dual", "False")
# This is set ad-hoc
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
C = UniformFloatHyperparameter(
"C", 0.03125, 32768, log=True, default_value=1.0)
multi_class = Constant("multi_class", "ovr")
# These are set ad-hoc
fit_intercept = Constant("fit_intercept", "True")
intercept_scaling = Constant("intercept_scaling", 1)
cs.add_hyperparameters([penalty, loss, dual, tol, C, multi_class,
fit_intercept, intercept_scaling])
penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(penalty, "l1"),
ForbiddenEqualsClause(loss, "hinge")
)
constant_penalty_and_loss = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l2"),
ForbiddenEqualsClause(loss, "hinge")
)
penalty_and_dual = ForbiddenAndConjunction(
ForbiddenEqualsClause(dual, "False"),
ForbiddenEqualsClause(penalty, "l1")
)
cs.add_forbidden_clause(penalty_and_loss)
cs.add_forbidden_clause(constant_penalty_and_loss)
cs.add_forbidden_clause(penalty_and_dual)
return cs
