def get_hyperparameter_search_space(dataset_properties=None):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
n_components = UniformIntegerHyperparameter(
"n_components", 50, 10000, default_value=100, log=True)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(**kwargs):
n_components_factor = UniformFloatHyperparameter("n_components_factor",
0.,
1.,
default_value=1.)
kernel = CategoricalHyperparameter(
'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default_value=1.0)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1., 1., default_value=0.)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[n_components_factor, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions(
[degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
n_components = UniformIntegerHyperparameter(
"n_components", 50, 5000, default_value=100, log=True)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
alpha = UniformFloatHyperparameter(name='alpha',
lower=10**-5,
upper=1,
log=True,
default_value=1.0)
kernel = CategoricalHyperparameter(
name='kernel',
# We restrict ourselves to two possible kernels for this example
choices=['polynomial', 'rbf'],
default_value='polynomial')
gamma = UniformFloatHyperparameter(name='gamma',
lower=0.00001,
upper=1,
default_value=0.1,
log=True)
degree = UniformIntegerHyperparameter(name='degree',
lower=2,
upper=5,
default_value=3)
coef0 = UniformFloatHyperparameter(
name='coef0',
lower=1e-2,
upper=1e2,
log=True,
default_value=1,
)
cs.add_hyperparameters([alpha, kernel, gamma, degree, coef0])
degree_condition = EqualsCondition(degree, kernel, 'polynomial')
coef0_condition = EqualsCondition(coef0, kernel, 'polynomial')
cs.add_conditions([degree_condition, coef0_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = Constant("loss", "auto")
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=1,
default_value=0.1,
log=True)
max_iter = UniformIntegerHyperparameter("max_iter",
32,
512,
default_value=100)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf",
lower=1,
upper=200,
default_value=20,
log=True)
max_depth = UnParametrizedHyperparameter(name="max_depth",
value="None")
max_leaf_nodes = UniformIntegerHyperparameter(name="max_leaf_nodes",
lower=3,
upper=2047,
default_value=31,
log=True)
max_bins = Constant("max_bins", 256)
l2_regularization = UniformFloatHyperparameter(
name="l2_regularization",
lower=1E-10,
upper=1,
default_value=1E-10,
log=True)
early_stop = CategoricalHyperparameter(
name="early_stop",
choices=["off", "train", "valid"],
default_value="off")
tol = UnParametrizedHyperparameter(name="tol", value=1e-7)
scoring = UnParametrizedHyperparameter(name="scoring", value="loss")
n_iter_no_change = UniformIntegerHyperparameter(
name="n_iter_no_change", lower=1, upper=20, default_value=10)
validation_fraction = UniformFloatHyperparameter(
name="validation_fraction",
lower=0.01,
upper=0.4,
default_value=0.1)
cs.add_hyperparameters([
loss, learning_rate, max_iter, min_samples_leaf, max_depth,
max_leaf_nodes, max_bins, l2_regularization, early_stop, tol,
scoring, n_iter_no_change, validation_fraction
])
n_iter_no_change_cond = InCondition(n_iter_no_change, early_stop,
["valid", "train"])
validation_fraction_cond = EqualsCondition(validation_fraction,
early_stop, "valid")
cs.add_conditions([n_iter_no_change_cond, validation_fraction_cond])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter("n_components",
10,
2000,
default_value=100)
kernel = CategoricalHyperparameter(
'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter(
"gamma",
3.0517578125e-05,
8,
log=True,
default_value=0.01,
)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions(
[degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def load_configspace(self, folder):
"""Will try to load the configspace. If it's a pcs-file, backup_cs will be a list containing all possible
combinations of interpretation for Categoricals. If this issue will be fixed, we can drop this procedure."""
cs_fn_json = os.path.join(folder, 'configspace.json')
cs_fn_pcs = os.path.join(folder, 'configspace.pcs')
if os.path.exists(cs_fn_json):
with open(cs_fn_json, 'r') as fh:
cs = pcs_json.read(fh.read())
backup_cs = []
self.logger.debug(
"Detected and loaded \"%s\". No backup-cs necessary",
cs_fn_json)
elif os.path.exists(cs_fn_pcs):
with open(cs_fn_pcs, 'r') as fh:
cs = pcs_new.read(fh.readlines())
# Create alternative interpretations
categoricals = [
hp for hp in cs.get_hyperparameters()
if isinstance(hp, CategoricalHyperparameter)
]
non_categoricals = [
hp for hp in cs.get_hyperparameters()
if not isinstance(hp, CategoricalHyperparameter)
]
def _get_interpretations(choices):
result = []
if set(choices) == {"True", "False"}:
result.append([True, False])
if all([c.isdigit() for c in choices]):
result.append([int(c) for c in choices])
result.append(choices)
return result
choices_per_cat = [
_get_interpretations(hp.choices) for hp in categoricals
]
combinations = itertools.product(*choices_per_cat)
self.logger.debug(combinations)
backup_cs = []
for combi in combinations:
bcs = ConfigurationSpace()
for hp in non_categoricals:
bcs.add_hyperparameter(hp)
for name, choices in zip([hp.name for hp in categoricals],
combi):
bcs.add_hyperparameter(
CategoricalHyperparameter(name, choices))
bcs.add_conditions(cs.get_conditions())
backup_cs.append(bcs)
self.logger.debug("Sampled %d interpretations of \"%s\"",
len(backup_cs), cs_fn_pcs)
self.logger.debug(choices_per_cat)
else:
raise ValueError("Missing pcs-file at '%s.[pcs|json]'!" %
os.path.join(folder, 'configspace'))
return cs, backup_cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter(
"loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default="log")
penalty = CategoricalHyperparameter("penalty",
["l1", "l2", "elasticnet"],
default="l2")
alpha = UniformFloatHyperparameter("alpha",
10e-7,
1e-1,
log=True,
default=0.0001)
l1_ratio = UniformFloatHyperparameter("l1_ratio",
1e-9,
1,
log=True,
default=0.15)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
n_iter = UniformIntegerHyperparameter("n_iter",
5,
1000,
log=True,
default=20)
epsilon = UniformFloatHyperparameter("epsilon",
1e-5,
1e-1,
default=1e-4,
log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default="optimal")
eta0 = UniformFloatHyperparameter("eta0", 10**-7, 0.1, default=0.01)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default=0.25)
average = CategoricalHyperparameter("average", ["False", "True"],
default="False")
cs.add_hyperparameters([
loss, penalty, alpha, l1_ratio, fit_intercept, n_iter, epsilon,
learning_rate, eta0, power_t, average
])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
# eta0 seems to be always active according to the source code; when
# learning_rate is set to optimial, eta0 is the starting value:
# https://github.com/scikit-learn/scikit-learn/blob/0.15.X/sklearn/linear_model/sgd_fast.pyx
#eta0_and_inv = EqualsCondition(eta0, learning_rate, "invscaling")
#eta0_and_constant = EqualsCondition(eta0, learning_rate, "constant")
#eta0_condition = OrConjunction(eta0_and_inv, eta0_and_constant)
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
hidden_layer_depth = UniformIntegerHyperparameter(name="hidden_layer_depth",
lower=1, upper=3, default_value=1)
num_nodes_per_layer = UniformIntegerHyperparameter(name="num_nodes_per_layer",
lower=16, upper=264, default_value=32,
log=True)
activation = CategoricalHyperparameter(name="activation", choices=['tanh', 'relu'],
default_value='tanh')
alpha = UniformFloatHyperparameter(name="alpha", lower=1e-7, upper=1e-1, default_value=1e-4,
log=True)
learning_rate_init = UniformFloatHyperparameter(name="learning_rate_init",
lower=1e-4, upper=0.5, default_value=1e-3,
log=True)
# Not allowing to turn off early stopping
early_stopping = CategoricalHyperparameter(name="early_stopping",
choices=["valid", "train"], # , "off"],
default_value="valid")
# Constants
n_iter_no_change = Constant(name="n_iter_no_change", value=32) # default=10 is too low
validation_fraction = Constant(name="validation_fraction", value=0.1)
tol = UnParametrizedHyperparameter(name="tol", value=1e-4)
solver = Constant(name="solver", value='adam')
# Relying on sklearn defaults for now
batch_size = UnParametrizedHyperparameter(name="batch_size", value="auto")
shuffle = UnParametrizedHyperparameter(name="shuffle", value="True")
beta_1 = UnParametrizedHyperparameter(name="beta_1", value=0.9)
beta_2 = UnParametrizedHyperparameter(name="beta_2", value=0.999)
epsilon = UnParametrizedHyperparameter(name="epsilon", value=1e-8)
# Not used
# solver=["sgd", "lbfgs"] --> not used to keep searchspace simpler
# learning_rate --> only used when using solver=sgd
# power_t --> only used when using solver=sgd & learning_rate=invscaling
# momentum --> only used when solver=sgd
# nesterovs_momentum --> only used when solver=sgd
# max_fun --> only used when solver=lbfgs
# activation=["identity", "logistic"] --> not useful for classification
cs.add_hyperparameters([hidden_layer_depth, num_nodes_per_layer,
activation, alpha,
learning_rate_init, early_stopping,
n_iter_no_change, validation_fraction, tol,
solver, batch_size, shuffle,
beta_1, beta_2, epsilon])
validation_fraction_cond = InCondition(validation_fraction, early_stopping, ["valid"])
cs.add_conditions([validation_fraction_cond])
# We always use early stopping
# n_iter_no_change_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"])
# tol_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"])
# cs.add_conditions([n_iter_no_change_cond, tol_cond])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter(
"loss",
["squared_loss", "huber", "epsilon_insensitive", "squared_epsilon_insensitive"],
default_value="squared_loss",
)
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1., log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter(
"fit_intercept", "True")
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=0.1, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter(
"power_t", 1e-5, 1, default_value=0.25)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
tol, epsilon, learning_rate, eta0,
power_t, average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = InCondition(
epsilon,
loss,
["huber", "epsilon_insensitive", "squared_epsilon_insensitive"],
)
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='tpe'):
method = CategoricalHyperparameter("method", ['average', 'weighted'], default_value='weighted')
alpha = UniformFloatHyperparameter("alpha", 1e-5, 1e-3, log=True, default_value=1e-4)
cs = ConfigurationSpace()
cs.add_hyperparameters([method, alpha])
alpha_cond = EqualsCondition(alpha, method, 'weighted')
cs.add_conditions([alpha_cond])
return cs
def get_aug_hyperparameter_space():
cs = ConfigurationSpace()
aug = CategoricalHyperparameter('aug',
choices=['True', 'False'],
default_value='True')
auto_aug = CategoricalHyperparameter('auto_aug',
choices=['True', 'False'],
default_value='False')
random_flip = CategoricalHyperparameter('random_flip',
choices=['True', 'False'],
default_value='True')
affine = CategoricalHyperparameter('affine',
choices=['True', 'False'],
default_value='True')
jitter = CategoricalHyperparameter('jitter',
choices=['True', 'False'],
default_value='True')
brightness = CategoricalHyperparameter('brightness',
choices=[0.2],
default_value=0.2)
saturation = CategoricalHyperparameter('saturation',
choices=[0.2],
default_value=0.2)
hue = CategoricalHyperparameter('hue', choices=[0.15], default_value=0.15)
degree = CategoricalHyperparameter('degree',
choices=[10, 20, 30],
default_value=10)
shear = CategoricalHyperparameter('shear',
choices=[0.05, 0.1, 0.2],
default_value=0.1)
cs.add_hyperparameters([
aug, random_flip, auto_aug, affine, jitter, brightness, saturation,
hue, degree, shear
])
auto_aug_on_aug = EqualsCondition(auto_aug, aug, 'True')
random_flip_on_auto_aug = EqualsCondition(random_flip, auto_aug, 'False')
affine_on_auto_aug = EqualsCondition(affine, auto_aug, 'False')
jitter_on_auto_aug = EqualsCondition(jitter, auto_aug, 'False')
brightness_on_jitter = EqualsCondition(brightness, jitter, 'True')
saturation_on_jitter = EqualsCondition(saturation, jitter, 'True')
hue_on_jitter = EqualsCondition(hue, jitter, 'True')
degree_on_affine = EqualsCondition(degree, affine, 'True')
shear_on_affine = EqualsCondition(shear, affine, 'True')
cs.add_conditions([
auto_aug_on_aug, random_flip_on_auto_aug, affine_on_auto_aug,
jitter_on_auto_aug, brightness_on_jitter, saturation_on_jitter,
hue_on_jitter, degree_on_affine, shear_on_affine
])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss",
["squared_loss", "huber", "epsilon_insensitive",
"squared_epsilon_insensitive"],
default_value="squared_loss")
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1., log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter(
"fit_intercept", "True")
tol = UniformFloatHyperparameter(
"tol", 1e-5, 1e-1, default_value=1e-4, log=True)
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=0.1, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter(
"power_t", 1e-5, 1, default_value=0.25)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
tol, epsilon, learning_rate, eta0,
power_t, average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = InCondition(epsilon, loss,
["huber", "epsilon_insensitive", "squared_epsilon_insensitive"])
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter(
"n_components", 10, 2000, default_value=100)
kernel = CategoricalHyperparameter('kernel',
['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=1.0)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
# Copied from libsvm_c
C = UniformFloatHyperparameter(
name="C", lower=0.03125, upper=32768, log=True, default=1.0)
kernel = CategoricalHyperparameter(
name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'],
default="rbf")
degree = UniformIntegerHyperparameter(
name="degree", lower=1, upper=5, default=3)
# Changed the gamma value to 0.0 (is 0.1 for classification)
gamma = UniformFloatHyperparameter(
name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(
name="coef0", lower=-1, upper=1, default=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(
name="shrinking", choices=["True", "False"], default="True")
tol = UniformFloatHyperparameter(
name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True)
max_iter = UnParametrizedHyperparameter("max_iter", -1)
# Random Guess
epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001,
upper=1, default=0.1, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter, epsilon])
degree_depends_on_kernel = InCondition(child=degree, parent=kernel,
values=('poly', 'rbf', 'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel,
values=('poly', 'sigmoid'))
cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel,
coef0_depends_on_kernel])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
C = UniformFloatHyperparameter(
name="C", lower=0.03125, upper=32768, log=True, default_value=1.0)
# Random Guess
epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001,
upper=1, default_value=0.1,
log=True)
kernel = CategoricalHyperparameter(
name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'],
default_value="rbf")
degree = UniformIntegerHyperparameter(
name="degree", lower=2, upper=5, default_value=3)
gamma = UniformFloatHyperparameter(
name="gamma", lower=3.0517578125e-05, upper=8, log=True, default_value=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(
name="coef0", lower=-1, upper=1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(
name="shrinking", choices=["True", "False"], default_value="True")
tol = UniformFloatHyperparameter(
name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True)
max_iter = UnParametrizedHyperparameter("max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter, epsilon])
degree_depends_on_kernel = InCondition(child=degree, parent=kernel,
values=('poly', 'rbf', 'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel,
values=('poly', 'sigmoid'))
cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel,
coef0_depends_on_kernel])
return cs
def load_configspace(self, folder):
"""Will try to load the configspace. cs_options will be a list containing all possible
combinations of interpretation for Categoricals. If this issue will be fixed, we can drop this procedure.
Parameters
----------
folder: str
path to folder in which to look for configspace
Returns
-------
cs_options: list[ConfigurationSpace]
list with possible interpretations for config-space-file. Only contains multiple items if file-format is pcs.
"""
cs_options = []
cs_fn_json = os.path.join(folder, 'configspace.json')
cs_fn_pcs = os.path.join(folder, 'configspace.pcs')
if os.path.exists(cs_fn_json):
with open(cs_fn_json, 'r') as fh:
cs_options = [pcs_json.read(fh.read())]
self.logger.debug(
"Detected and loaded \"%s\". No alternative interpretations necessary",
cs_fn_json)
elif os.path.exists(cs_fn_pcs):
with open(cs_fn_pcs, 'r') as fh:
cs = pcs_new.read(fh.readlines())
# Create alternative interpretations
categoricals = [
hp for hp in cs.get_hyperparameters()
if isinstance(hp, CategoricalHyperparameter)
]
non_categoricals = [
hp for hp in cs.get_hyperparameters()
if not isinstance(hp, CategoricalHyperparameter)
]
def _get_interpretations(choices):
""" Generate different interpretations for critical categorical hyperparameters that are not seamlessly
supported by pcs-format."""
result = []
if set(choices) == {"True", "False"}:
result.append([True, False])
if all([c.isdigit() for c in choices]):
result.append([int(c) for c in choices])
result.append(choices)
return result
choices_per_cat = [
_get_interpretations(hp.choices) for hp in categoricals
]
combinations = itertools.product(*choices_per_cat)
self.logger.debug(combinations)
for combi in combinations:
bcs = ConfigurationSpace()
for hp in non_categoricals:
bcs.add_hyperparameter(hp)
for name, choices in zip([hp.name for hp in categoricals],
combi):
bcs.add_hyperparameter(
CategoricalHyperparameter(name, choices))
bcs.add_conditions(cs.get_conditions())
cs_options.append(bcs)
self.logger.debug("Sampled %d interpretations of \"%s\"",
len(cs_options), cs_fn_pcs)
else:
raise ValueError("Missing pcs-file at '%s.[pcs|json]'!" %
os.path.join(folder, 'configspace'))
return cs_options
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
Window_size = UniformIntegerHyperparameter(name="Window_size",
lower=5,
upper=50,
default_value=20)
tsfresh_feature = CategoricalHyperparameter(name="tsfresh_feature",
choices=["True", "False"],
default_value="True")
Difference = CategoricalHyperparameter(name="Difference",
choices=["True", "False"],
default_value="True")
loss = CategoricalHyperparameter("loss", [
"squared_loss", "huber", "epsilon_insensitive",
"squared_epsilon_insensitive"
],
default_value="squared_loss")
penalty = CategoricalHyperparameter("penalty",
["l1", "l2", "elasticnet"],
default_value="l2")
alpha = UniformFloatHyperparameter("alpha",
1e-7,
1e-1,
log=True,
default_value=0.0001)
l1_ratio = UniformFloatHyperparameter("l1_ratio",
1e-9,
1.,
log=True,
default_value=0.15)
tol = UniformFloatHyperparameter("tol",
1e-4,
1e-1,
default_value=1e-3,
log=True)
epsilon = UniformFloatHyperparameter("epsilon",
1e-5,
1e-1,
default_value=0.1,
log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter("eta0",
1e-7,
1e-1,
default_value=0.01)
power_t = UniformFloatHyperparameter("power_t",
1e-5,
1,
default_value=0.25)
average = CategoricalHyperparameter("average", ["False", "True"],
default_value="False")
# un parametrized parameter
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
max_iter = UnParametrizedHyperparameter("max_iter", 1000)
cs.add_hyperparameters([
loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon,
learning_rate, eta0, Difference, power_t, average, Window_size,
tsfresh_feature, max_iter
])
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = InCondition(
epsilon, loss,
["huber", "epsilon_insensitive", "squared_epsilon_insensitive"])
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition])
return cs
default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(name="shrinking",
choices=["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter(name="tol",
lower=1e-5,
upper=1e-1,
default_value=1e-3,
log=True)
max_iter = Constant("max_iter", -1)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[C, kernel, degree, gamma, coef0, shrinking, tol, max_iter, epsilon])
degree_depends_on_kernel = InCondition(child=degree,
parent=kernel,
values=('poly', 'rbf', 'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma,
parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0,
parent=kernel,
values=('poly', 'sigmoid'))
cs.add_conditions([
degree_depends_on_kernel, gamma_depends_on_kernel, coef0_depends_on_kernel
])
json_utils.write_cs_to_json_file(cs, "SVR")
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter("eta0",
1e-7,
1e-1,
default_value=0.01,
log=True)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, default_value=0.5)
average = CategoricalHyperparameter("average", ["False", "True"],
default_value="False")
cs.add_hyperparameters([
loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon, learning_rate,
eta0, power_t, average
])
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = InCondition(
epsilon, loss,
["huber", "epsilon_insensitive", "squared_epsilon_insensitive"])
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling", "constant"])
power_t_condition = EqualsCondition(power_t, learning_rate, "invscaling")
cs.add_conditions(
[elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con])
json_utils.write_cs_to_json_file(cs, "SGDRegressor")
from ConfigSpace.configuration_space import ConfigurationSpace
from ConfigSpace.hyperparameters import UniformFloatHyperparameter, \
UniformIntegerHyperparameter, CategoricalHyperparameter
from ConfigSpace.conditions import InCondition, EqualsCondition
from automl.utl import json_utils
cs = ConfigurationSpace()
kernel = CategoricalHyperparameter('kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
n_components = UniformIntegerHyperparameter(
"n_components", 50, 10000, default_value=100, log=True)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
cs.add_hyperparameters([kernel, degree, gamma, coef0, n_components])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_conditions([degree_depends_on_poly, coef0_condition, gamma_condition])
json_utils.write_cs_to_json_file(cs, "Nystroem")
def get_hyperparameter_search_space(**kwargs):
cs = ConfigurationSpace()
loss = CategoricalHyperparameter(
"loss", ["log", "modified_huber", "squared_hinge", "perceptron"],
default_value="log")
penalty = CategoricalHyperparameter("penalty",
["l2", "l1", "elasticnet"],
default_value="l2")
alpha = UniformFloatHyperparameter("alpha",
1e-7,
1e-1,
default_value=0.0001,
log=True)
l1_ratio = UniformFloatHyperparameter("l1_ratio",
1e-9,
1.,
default_value=0.15,
log=True)
fit_intercept = CategoricalHyperparameter("fit_intercept",
[True, False],
default_value=True)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-4,
log=True)
epsilon = UniformFloatHyperparameter("epsilon",
1e-5,
1e-1,
default_value=1e-4,
log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["constant", "optimal", "invscaling"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter("eta0",
1e-7,
1e-1,
default_value=0.01,
log=True)
power_t = UniformFloatHyperparameter("power_t",
1e-5,
1,
default_value=0.5)
average = CategoricalHyperparameter("average", [True, False],
default_value=False)
cs.add_hyperparameters([
loss, penalty, alpha, l1_ratio, fit_intercept, tol, epsilon,
learning_rate, eta0, power_t, average
])
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate,
["invscaling", "constant"])
cs.add_conditions([
elasticnet, epsilon_condition, power_t_condition, eta0_in_inv_con
])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
loss = CategoricalHyperparameter("loss",
["hinge", "log", "modified_huber", "squared_hinge", "perceptron"],
default_value="log")
penalty = CategoricalHyperparameter(
"penalty", ["l1", "l2", "elasticnet"], default_value="l2")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1e-1, log=True, default_value=0.0001)
l1_ratio = UniformFloatHyperparameter(
"l1_ratio", 1e-9, 1, log=True, default_value=0.15)
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True,
default_value=1e-4)
epsilon = UniformFloatHyperparameter(
"epsilon", 1e-5, 1e-1, default_value=1e-4, log=True)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["optimal", "invscaling", "constant"],
default_value="invscaling")
eta0 = UniformFloatHyperparameter(
"eta0", 1e-7, 1e-1, default_value=0.01, log=True)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True,
default_value=0.5)
average = CategoricalHyperparameter(
"average", ["False", "True"], default_value="False")
cs.add_hyperparameters([loss, penalty, alpha, l1_ratio, fit_intercept,
tol, epsilon, learning_rate, eta0, power_t,
average])
# TODO add passive/aggressive here, although not properly documented?
elasticnet = EqualsCondition(l1_ratio, penalty, "elasticnet")
epsilon_condition = EqualsCondition(epsilon, loss, "modified_huber")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
# eta0 is only relevant if learning_rate!='optimal' according to code
# https://github.com/scikit-learn/scikit-learn/blob/0.19.X/sklearn/
# linear_model/sgd_fast.pyx#L603
eta0_in_inv_con = InCondition(eta0, learning_rate, ["invscaling",
"constant"])
cs.add_conditions([elasticnet, epsilon_condition, power_t_condition,
eta0_in_inv_con])
return cs
elif optimizer == 'tpe':
eta0 = hp.loguniform('sgd_eta0', np.log(1e-7), np.log(1e-1))
space = {
'loss': hp.choice('sgd_loss', [
("modified_huber", {'epsilon': hp.loguniform('sgd_epsilon', np.log(1e-5), np.log(1e-1))}),
("hinge", {}),
("log", {}),
("squared_hinge", {}),
("perceptron", {})]),
'penalty': hp.choice('sgd_penalty',
[("elasticnet",
{'l1_ratio': hp.loguniform('sgd_l1_ratio', np.log(1e-9), np.log(1))}),
("l1", None),
("l2", None)]),
'alpha': hp.loguniform('sgd_alpha', np.log(1e-7), np.log(1e-1)),
'fit_intercept': hp.choice('sgd_fit_intercept', ["True"]),
'tol': hp.loguniform('sgd_tol', np.log(1e-5), np.log(1e-1)),
'learning_rate': hp.choice('sgd_learning_rate', [("optimal", {}),
("invscaling",
{'power_t': hp.loguniform('sgd_power_t', np.log(1e-5),
np.log(1)),
'eta0': eta0}),
("constant", {'eta0': eta0})]),
'average': hp.choice('sgd_average', ["True", "False"])}
init_trial = {'loss': ("log", {}),
'penalty': ("l2", {}),
'alpha': 1e-4,
'fit_intercept': "True",
'tol': 1e-4,
'learning_rate': ("invscaling", {'power_t': 0.5, 'eta0': 0.01}),
'average': "False"}
return space
batch_size = Constant(name="batch_size", value="auto")
shuffle = Constant(name="shuffle", value="True")
beta_1 = Constant(name="beta_1", value=0.9)
beta_2 = Constant(name="beta_2", value=0.999)
epsilon = Constant(name="epsilon", value=1e-8)
# Not used
# solver=["sgd", "lbfgs"] --> not used to keep searchspace simpler
# learning_rate --> only used when using solver=sgd
# power_t --> only used when using solver=sgd & learning_rate=invscaling
# momentum --> only used when solver=sgd
# nesterovs_momentum --> only used when solver=sgd
# max_fun --> only used when solver=lbfgs
# activation=["identity", "logistic"] --> not useful for classification
cs.add_hyperparameters([
hidden_layer_sizes, activation, alpha, learning_rate_init, early_stopping,
n_iter_no_change, validation_fraction, tol, solver, batch_size, shuffle,
beta_1, beta_2, epsilon
])
validation_fraction_cond = InCondition(validation_fraction, early_stopping,
["True"])
cs.add_conditions([validation_fraction_cond])
# We always use early stopping
# n_iter_no_change_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"])
# tol_cond = InCondition(n_iter_no_change, early_stopping, ["valid", "train"])
# cs.add_conditions([n_iter_no_change_cond, tol_cond])
cs.add_conditions([validation_fraction_cond])
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
# Parameterized Hyperparameters
max_depth = UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=20, default_value=3
)
learning_rate = UniformFloatHyperparameter(
name="learning_rate", lower=0.001, upper=1, default_value=0.1,
log=True,
)
n_estimators = Constant("n_estimators", 512)
booster = CategoricalHyperparameter(
"booster", ["gbtree", "dart"]
)
subsample = UniformFloatHyperparameter(
name="subsample", lower=0.01, upper=1.0, default_value=1.0, log=False
)
min_child_weight = UniformIntegerHyperparameter(
name="min_child_weight", lower=0,
upper=20, default_value=1, log=False
)
colsample_bytree = UniformFloatHyperparameter(
name="colsample_bytree", lower=0.1, upper=1.0, default_value=1,
)
colsample_bylevel = UniformFloatHyperparameter(
name="colsample_bylevel", lower=0.1, upper=1.0, default_value=1,
)
reg_alpha = UniformFloatHyperparameter(
name="reg_alpha", lower=1e-10, upper=1e-1, log=True,
default_value=1e-10)
reg_lambda = UniformFloatHyperparameter(
name="reg_lambda", lower=1e-10, upper=1e-1, log=True,
default_value=1e-10)
# DART Hyperparameters
sample_type = CategoricalHyperparameter(
'sample_type', ['uniform', 'weighted'], default_value='uniform',
)
normalize_type = CategoricalHyperparameter(
'normalize_type', ['tree', 'forest'], default_value='tree',
)
rate_drop = UniformFloatHyperparameter(
'rate_drop', 1e-10, 1-(1e-10), default_value=0.5,
)
# Unparameterized Hyperparameters
# https://xgboost.readthedocs.io/en/latest//parameter.html
# minimum loss reduction required to make a further partition on a
# leaf node of the tree
gamma = UnParametrizedHyperparameter(
name="gamma", value=0)
# absolute regularization (in contrast to eta), comparable to
# gradient clipping in deep learning - according to the internet this
# is most important for unbalanced data
max_delta_step = UnParametrizedHyperparameter(
name="max_delta_step", value=0)
base_score = UnParametrizedHyperparameter(
name="base_score", value=0.5)
scale_pos_weight = UnParametrizedHyperparameter(
name="scale_pos_weight", value=1)
cs.add_hyperparameters([
# Active
max_depth, learning_rate, n_estimators, booster,
subsample, colsample_bytree, colsample_bylevel,
reg_alpha, reg_lambda,
# DART
sample_type, normalize_type, rate_drop,
# Inactive
min_child_weight, max_delta_step, gamma,
base_score, scale_pos_weight
])
sample_type_condition = EqualsCondition(
sample_type, booster, 'dart',
)
normalize_type_condition = EqualsCondition(
normalize_type, booster, 'dart',
)
rate_drop_condition = EqualsCondition(
rate_drop, booster, 'dart',
)
cs.add_conditions([
sample_type_condition, normalize_type_condition,
rate_drop_condition,
])
return cs
def get_benchmark_configspace(benchmark_id):
if benchmark_id == 'fcnet':
cs = ConfigurationSpace()
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-4,
1e-2,
default_value=1e-3,
q=2e-4)
momentum = UniformFloatHyperparameter("momentum",
0.,
.5,
default_value=0.,
q=.1)
lr_decay = UniformFloatHyperparameter("lr_decay",
.7,
.99,
default_value=9e-1,
q=3e-2)
n_layer1 = UniformIntegerHyperparameter("n_layer1",
32,
256,
default_value=96,
q=8)
n_layer2 = UniformIntegerHyperparameter("n_layer2",
64,
256,
default_value=128,
q=8)
batch_size = UniformIntegerHyperparameter("batch_size",
32,
128,
default_value=64,
q=8)
dropout1 = UniformFloatHyperparameter("kb_1",
.3,
.9,
default_value=.5,
q=.1)
dropout2 = UniformFloatHyperparameter("kb_2",
.3,
.9,
default_value=.5,
q=.1)
kernel_regularizer = UniformFloatHyperparameter("k_reg",
1e-9,
1e-4,
default_value=1e-6,
q=5e-7,
log=True)
cs.add_hyperparameters([
learning_rate, momentum, lr_decay, n_layer1, n_layer2, batch_size,
dropout1, dropout2, kernel_regularizer
])
elif benchmark_id in ['covtype', 'higgs']:
cs = ConfigurationSpace()
# n_estimators = UniformFloatHyperparameter("n_estimators", 100, 600, default_value=200, q=10)
eta = UniformFloatHyperparameter("eta",
0.01,
0.9,
default_value=0.3,
q=0.01)
min_child_weight = UniformFloatHyperparameter("min_child_weight",
0,
10,
default_value=1,
q=0.1)
max_depth = UniformIntegerHyperparameter("max_depth",
1,
12,
default_value=6)
subsample = UniformFloatHyperparameter("subsample",
0.1,
1,
default_value=1,
q=0.1)
gamma = UniformFloatHyperparameter("gamma",
0,
10,
default_value=0,
q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.1,
1,
default_value=1.,
q=0.1)
alpha = UniformFloatHyperparameter("alpha",
0,
10,
default_value=0.,
q=0.1)
_lambda = UniformFloatHyperparameter("lambda",
1,
10,
default_value=1,
q=0.1)
cs.add_hyperparameters([
eta, min_child_weight, max_depth, subsample, gamma,
colsample_bytree, alpha, _lambda
])
elif benchmark_id in ['covtype_svm', 'mnist_svm']:
C = UniformFloatHyperparameter("C",
1e-3,
1e5,
log=True,
default_value=1.0)
kernel = CategoricalHyperparameter("kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma",
1e-5,
10,
log=True,
default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
tol = UniformFloatHyperparameter("tol",
1e-5,
1e-1,
default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 10000)
cs = ConfigurationSpace()
cs.add_hyperparameters(
[C, kernel, degree, gamma, coef0, tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
elif benchmark_id == 'cifar':
cs = ConfigurationSpace()
# padding_size = CategoricalHyperparameter('padding_size', [1, 2, 3], default_value=2)
# batch_size = CategoricalHyperparameter('train_batch_size', [256])
batch_size = UniformIntegerHyperparameter("train_batch_size",
32,
256,
default_value=64,
q=8)
init_lr = UniformFloatHyperparameter('init_lr',
lower=1e-3,
upper=0.3,
default_value=0.1,
log=True)
# lr_decay_factor = UniformFloatHyperparameter('lr_decay_factor', lower=0.01, upper=0.2, default_value=0.1,
# log=True)
lr_decay_factor = UnParametrizedHyperparameter('lr_decay_factor', 0.1)
weight_decay = UniformFloatHyperparameter('weight_decay',
lower=1e-5,
upper=1e-2,
default_value=0.0002,
log=True)
momentum = UniformFloatHyperparameter("momentum",
0.5,
.99,
default_value=0.9)
nesterov = CategoricalHyperparameter('nesterov', ['True', 'False'],
default_value='True')
cs.add_hyperparameters([
nesterov, batch_size, init_lr, lr_decay_factor, weight_decay,
momentum
])
elif benchmark_id == 'convnet':
cs = ConfigurationSpace()
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-5,
5e-2,
default_value=1e-4,
q=3e-5,
log=True)
batch_size = UniformIntegerHyperparameter("batch_size",
16,
128,
q=16,
default_value=32)
momentum = UniformFloatHyperparameter("momentum",
0.,
.5,
default_value=0.,
q=.1)
lr_decay = UniformFloatHyperparameter("lr_decay",
.7,
.99,
default_value=9e-1,
q=3e-2)
dropout_value = UniformFloatHyperparameter("dropout",
.1,
.7,
default_value=.5,
q=.1)
cs.add_hyperparameters(
[learning_rate, batch_size, momentum, lr_decay, dropout_value])
num_pooling_layer = UniformIntegerHyperparameter("n_pooling_layer",
2,
3,
default_value=2)
num_conv_layer1 = UniformIntegerHyperparameter("n_conv_layer1",
16,
64,
default_value=32,
q=2)
num_conv_layer2 = UniformIntegerHyperparameter("n_conv_layer2",
32,
96,
default_value=64,
q=2)
num_conv_layer3 = UniformIntegerHyperparameter("n_conv_layer3",
32,
96,
default_value=64,
q=2)
num_fully_layer = UniformIntegerHyperparameter("n_fully_unit",
128,
512,
default_value=256,
q=64)
cs.add_hyperparameters([
num_pooling_layer, num_conv_layer1, num_conv_layer2,
num_conv_layer3, num_fully_layer
])
for i in [1, 2, 3]:
kernel_init_stddev = UniformFloatHyperparameter(
"kernel_init_stddev%d" % i,
1e-3,
5e-2,
default_value=1e-2,
q=2e-3)
kernel_regularizer = UniformFloatHyperparameter(
"kernel_regularizer%d" % i,
1e-9,
1e-4,
default_value=1e-6,
q=5e-7,
log=True)
cs.add_hyperparameters([kernel_init_stddev, kernel_regularizer])
if i == 3:
k_init_cond = InCondition(child=kernel_init_stddev,
parent=num_pooling_layer,
values=[3])
k_reg_cond = InCondition(child=kernel_regularizer,
parent=num_pooling_layer,
values=[3])
cs.add_conditions([k_init_cond, k_reg_cond])
return cs
elif 'sys' in benchmark_id:
from mfes.evaluate_function.sys.combined_evaluator import get_combined_cs
from solnml.datasets.utils import load_data
tmp_node = load_data('balloon',
data_dir='../soln-ml/',
task_type=0,
datanode_returned=True)
cs = get_combined_cs(tmp_node)
return cs
else:
raise ValueError('Invalid benchmark id: %s!' % benchmark_id)
return cs
def get_hyperparameter_search_space(
dataset_properties: Optional[Dict[str,
BaseDatasetPropertiesType]] = None,
n_components: HyperparameterSearchSpace = HyperparameterSearchSpace(
hyperparameter='n_components',
value_range=(0.5, 0.9),
default_value=0.5,
),
kernel: HyperparameterSearchSpace = HyperparameterSearchSpace(
hyperparameter='kernel',
value_range=('poly', 'rbf', 'sigmoid', 'cosine'),
default_value='rbf',
),
gamma: HyperparameterSearchSpace = HyperparameterSearchSpace(
hyperparameter='gamma',
value_range=(3.0517578125e-05, 8),
default_value=0.01,
log=True),
degree: HyperparameterSearchSpace = HyperparameterSearchSpace(
hyperparameter='degree',
value_range=(2, 5),
default_value=3,
log=True),
coef0: HyperparameterSearchSpace = HyperparameterSearchSpace(
hyperparameter='coef0',
value_range=(-1, 1),
default_value=0,
)
) -> ConfigurationSpace:
cs = ConfigurationSpace()
if dataset_properties is not None:
n_features = len(
dataset_properties['numerical_columns']) if isinstance(
dataset_properties['numerical_columns'], List) else 0
if n_features == 1:
log = False
else:
log = n_components.log
n_components = HyperparameterSearchSpace(
hyperparameter='n_components',
value_range=(floor(
float(n_components.value_range[0]) * n_features),
ceil(
float(n_components.value_range[1]) *
n_features)),
default_value=ceil(
float(n_components.default_value) * n_features),
log=log)
else:
n_components = HyperparameterSearchSpace(
hyperparameter='n_components',
value_range=(10, 2000),
default_value=100,
log=n_components.log)
add_hyperparameter(cs, n_components, UniformIntegerHyperparameter)
kernel_hp = get_hyperparameter(kernel, CategoricalHyperparameter)
gamma = get_hyperparameter(gamma, UniformFloatHyperparameter)
coef0 = get_hyperparameter(coef0, UniformFloatHyperparameter)
cs.add_hyperparameters([kernel_hp, gamma, coef0])
if "poly" in kernel_hp.choices:
degree = get_hyperparameter(degree, UniformIntegerHyperparameter)
cs.add_hyperparameters([degree])
degree_depends_on_poly = EqualsCondition(degree, kernel_hp, "poly")
cs.add_conditions([degree_depends_on_poly])
kernels = []
if "sigmoid" in kernel_hp.choices:
kernels.append("sigmoid")
if "poly" in kernel_hp.choices:
kernels.append("poly")
coef0_condition = InCondition(coef0, kernel_hp, kernels)
kernels = []
if "rbf" in kernel_hp.choices:
kernels.append("rbf")
if "poly" in kernel_hp.choices:
kernels.append("poly")
gamma_condition = InCondition(gamma, kernel_hp, kernels)
cs.add_conditions([coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
hidden_size = UniformIntegerHyperparameter("hidden_size", 100, 500, default_value=200)
activation = CategoricalHyperparameter("activation", ["identity", "logistic", "tanh", "relu"],
default_value="relu")
solver = CategoricalHyperparameter("solver", ["sgd", "adam"], default_value="adam")
alpha = UniformFloatHyperparameter(
"alpha", 1e-7, 1., log=True, default_value=0.0001)
learning_rate = CategoricalHyperparameter(
"learning_rate", ["adaptive", "invscaling", "constant"],
default_value="constant")
learning_rate_init = UniformFloatHyperparameter(
"learning_rate_init", 1e-4, 3e-1, default_value=0.001, log=True)
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, log=True,
default_value=1e-4)
momentum = UniformFloatHyperparameter("momentum", 0.6, 1, q=0.05, default_value=0.9)
nesterovs_momentum = CategoricalHyperparameter("nesterovs_momentum", [True, False], default_value=True)
beta1 = UniformFloatHyperparameter("beta1", 0.6, 1, default_value=0.9)
power_t = UniformFloatHyperparameter("power_t", 1e-5, 1, log=True,
default_value=0.5)
cs.add_hyperparameters(
[hidden_size, activation, solver, alpha, learning_rate, learning_rate_init, tol, momentum,
nesterovs_momentum, beta1,
power_t])
learning_rate_condition = EqualsCondition(learning_rate, solver, "sgd")
momentum_condition = EqualsCondition(momentum, solver, "sgd")
nesterovs_momentum_condition = EqualsCondition(nesterovs_momentum, solver, "sgd")
beta1_condition = EqualsCondition(beta1, solver, "adam")
power_t_condition = EqualsCondition(power_t, learning_rate,
"invscaling")
cs.add_conditions([learning_rate_condition, momentum_condition,
nesterovs_momentum_condition, beta1_condition, power_t_condition])
return cs
elif optimizer == 'tpe':
space = {'hidden_size': hp.randint("mlp_hidden_size", 400) + 100,
'activation': hp.choice('mlp_activation', ["identity", "logistic", "tanh", "relu"]),
'solver': hp.choice('mlp_solver',
[("sgd", {'learning_rate': hp.choice('mlp_learning_rate',
[("adaptive", {}),
("constant", {}),
("invscaling", {
'power_t': hp.uniform('mlp_power_t',
1e-5, 1)})]),
'momentum': hp.uniform('mlp_momentum', 0.6, 1),
'nesterovs_momentum': hp.choice('mlp_nesterovs_momentum',
[True, False])}),
("adam", {'beta1': hp.uniform('mlp_beta1', 0.6, 1)})]),
'alpha': hp.loguniform('mlp_alpha', np.log(1e-7), np.log(1e-1)),
'learning_rate_init': hp.loguniform('mlp_learning_rate_init', np.log(1e-6), np.log(1e-1)),
'tol': hp.loguniform('mlp_tol', np.log(1e-5), np.log(1e-1))}
return space
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
Window_size = UniformIntegerHyperparameter(
name="Window_size", lower=5, upper=50, default_value=20)
Difference = CategoricalHyperparameter(
name="Difference", choices=["True", "False"], default_value="True")
tsfresh_feature = CategoricalHyperparameter(
name="tsfresh_feature", choices=["True", "False"], default_value="True")
C = UniformFloatHyperparameter(
name="C", lower=0.03125, upper=32768, log=True, default_value=1.0)
epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001,
upper=1, default_value=0.1,
log=True)
kernel = CategoricalHyperparameter(
name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'],
default_value="rbf")
degree = UniformIntegerHyperparameter(
name="degree", lower=2, upper=5, default_value=3)
gamma = CategoricalHyperparameter("gamma", ["auto", "value"], default_value="auto")
gamma_value = UniformFloatHyperparameter(
name="gamma_value", lower=0.0001, upper=8, default_value=1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(
name="coef0", lower=-1, upper=1, default_value=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(
name="shrinking", choices=["True", "False"], default_value="True")
tol = UniformFloatHyperparameter(
name="tol", lower=1e-5, upper=1e-1, default_value=1e-3, log=True)
max_iter = UnParametrizedHyperparameter("max_iter", 200000)
cs.add_hyperparameters([Window_size, Difference, tsfresh_feature,C, kernel, degree, gamma, gamma_value, coef0, shrinking,
tol, max_iter, epsilon])
degree_depends_on_kernel = InCondition(child=degree, parent=kernel,
values=["poly"])
gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel,
values=["rbf", "poly", "sigmoid"])
coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel,
values=["poly", "sigmoid"])
gamma_value_depends_on_gamma = InCondition(child=gamma_value, parent=gamma,
values=["value"])
cs.add_conditions([degree_depends_on_kernel, gamma_depends_on_kernel,
coef0_depends_on_kernel, gamma_value_depends_on_gamma])
return cs
