def test__sample_UniformIntegerHyperparameter(self):
hp = UniformIntegerHyperparameter("uihp", 0, 10)
values = []
rs = np.random.RandomState(1)
for i in range(100):
values.append(hp._sample(rs))
self.assertEqual(len(np.unique(values)), 11)
def get_hyperparameter_search_space(dataset_properties=None):
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=10,
upper=100,
default_value=10)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=2,
upper=10,
default_value=5)
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default_value=1)
min_weight_fraction_leaf = Constant('min_weight_fraction_leaf', 1.0)
max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes",
value="None")
bootstrap = CategoricalHyperparameter('bootstrap', ['True', 'False'])
cs = ConfigurationSpace()
cs.add_hyperparameters([
n_estimators, max_depth, min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, max_leaf_nodes, bootstrap
])
return cs
def test__sample_UniformIntegerHyperparameter(self):
hp = UniformIntegerHyperparameter("uihp", 0, 10)
values = []
rs = np.random.RandomState(1)
for i in range(100):
values.append(hp._sample(rs))
self.assertEqual(len(np.unique(values)), 11)
hp = UniformIntegerHyperparameter("uihp", 2, 12)
values = []
rs = np.random.RandomState(1)
for i in range(100):
values.append(hp._sample(rs))
self.assertGreaterEqual(hp._transform(values[-1]), 2)
self.assertLessEqual(hp._transform(values[-1]), 12)
self.assertEqual(len(np.unique(values)), 11)
class DBSCAN(object, metaclass=Metaclass):
# static variables
_name = "DBSCAN"
_model = cluster.DBSCAN
_params = [
UniformFloatHyperparameter("eps", 0.01, 5, default_value=0.01),
UniformIntegerHyperparameter("min_samples",
5,
100,
default_value=5)
]
_params_names = set([p.name for p in _params])
_conditions = []
_forbidden_clauses = []
def get_hyperparameter_search_space(dataset_properties=None):
loss = CategoricalHyperparameter("loss",
["hinge", "squared_hinge"],
default="hinge")
fit_intercept = UnParametrizedHyperparameter("fit_intercept", "True")
n_iter = UniformIntegerHyperparameter("n_iter", 5, 1000, default=20,
log=True)
C = UniformFloatHyperparameter("C", 1e-5, 10, 1, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(loss)
cs.add_hyperparameter(fit_intercept)
cs.add_hyperparameter(n_iter)
cs.add_hyperparameter(C)
return cs
def clf_mlp_configs():
def run(dataset, seed, cfg):
cfg = {k: cfg[k] for k in cfg}
cfg["activation"] = activation[cfg["activation"]]
cfg["solver"] = solver[cfg["solver"]]
clf = MLPClassifier(random_state=seed, **cfg)
scores = cross_val_score(clf, dataset.data, dataset.target, cv=5)
return 1 - np.mean(scores)
activation = [
"identity", "logistic", "tanh", "relu", "identity", "logistic", "tanh",
"relu"
]
solver = ["lbfgs", "sgd", "adam", "lbfgs", "sgd", "adam"]
cs = ConfigurationSpace()
cs.add_hyperparameters([
UniformIntegerHyperparameter("activation", 0, 7),
UniformIntegerHyperparameter("solver", 0, 5),
UniformFloatHyperparameter("tol", 1e-7, 1e-1),
UniformFloatHyperparameter("alpha", 1e-7, 1e-1),
UniformIntegerHyperparameter("max_iter", 10, 1000),
])
return (cs, run, "Clf MLP")
def get_hyperparameter_search_space(dataset_properties=None):
n_components = UniformIntegerHyperparameter("n_components",
10,
2000,
default=100)
kernel = CategoricalHyperparameter(
'kernel', ['poly', 'rbf', 'sigmoid', 'cosine'], 'rbf')
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
log=True,
default=1.0)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0)
cs = ConfigurationSpace()
cs.add_hyperparameters([n_components, kernel, degree, gamma, coef0])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_condition = InCondition(gamma, kernel, ["poly", "rbf"])
cs.add_conditions(
[degree_depends_on_poly, coef0_condition, gamma_condition])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
gamma = UniformFloatHyperparameter("gamma",
3.0517578125e-05,
8,
default_value=1.0,
log=True)
n_components = UniformIntegerHyperparameter("n_components",
50,
10000,
default_value=100,
log=True)
cs = ConfigurationSpace()
cs.add_hyperparameters([gamma, n_components])
return cs
class GaussianMixture(object, metaclass=Metaclass):
# static variables
_name = "GaussianMixture"
_model = mixture.GaussianMixture
_params = [
UniformIntegerHyperparameter("n_components", 1, 80, default_value=5),
CategoricalHyperparameter("covariance_type", ['full', 'tied', 'diag', 'spherical'], default_value='full'),
CategoricalHyperparameter("init_params", ['kmeans', 'random'], default_value='kmeans'),
CategoricalHyperparameter("warm_start", [True, False], default_value=False),
# UniformIntegerHyperparameter("random_state", 0, 9, default_value=0)
]
_params_names = set([p.name for p in _params])
_conditions = []
_forbidden_clauses = []
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
loss = cs.add_hyperparameter(Constant("loss", "deviance"))
learning_rate = cs.add_hyperparameter(UniformFloatHyperparameter(
name="learning_rate", lower=0.01, upper=1, default=0.1, log=True))
n_estimators = cs.add_hyperparameter(UniformIntegerHyperparameter(
name="n_estimators", lower=50, upper=500, default=100))
max_depth = cs.add_hyperparameter(UniformIntegerHyperparameter(
name="max_depth", lower=1, upper=10, default=3))
min_samples_split = cs.add_hyperparameter(UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default=2, log=False))
min_samples_leaf = cs.add_hyperparameter(UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default=1, log=False))
min_weight_fraction_leaf = cs.add_hyperparameter(
UnParametrizedHyperparameter("min_weight_fraction_leaf", 0.))
subsample = cs.add_hyperparameter(UniformFloatHyperparameter(
name="subsample", lower=0.01, upper=1.0, default=1.0, log=False))
max_features = cs.add_hyperparameter(UniformFloatHyperparameter(
"max_features", 0.5, 5, default=1))
max_leaf_nodes = cs.add_hyperparameter(UnParametrizedHyperparameter(
name="max_leaf_nodes", value="None"))
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter("criterion", ["gini", "entropy"],
default_value="gini")
# The maximum number of features used in the forest is calculated as m^max_features, where
# m is the total number of features, and max_features is the hyperparameter specified below.
# The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This
# corresponds with Geurts' heuristic.
max_features = UniformFloatHyperparameter("max_features",
0.,
1.,
default_value=0.5)
max_depth = UnParametrizedHyperparameter("max_depth", "None")
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
1,
20,
default_value=1)
min_weight_fraction_leaf = UnParametrizedHyperparameter(
"min_weight_fraction_leaf", 0.)
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.0)
bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"],
default_value="True")
cs.add_hyperparameters([
n_estimators, criterion, max_features, max_depth,
min_samples_split, min_samples_leaf, min_weight_fraction_leaf,
max_leaf_nodes, bootstrap, min_impurity_decrease
])
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter(
"criterion", ["mse", "mae"], default_value="mse")
# The maximum number of features used in the forest is calculated as m^max_features, where
# m is the total number of features, and max_features is the hyperparameter specified below.
# The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This
# corresponds with Geurts' heuristic.
max_features = UniformFloatHyperparameter(
"max_features", 0., 1., default_value=0.5)
min_samples_split = UniformIntegerHyperparameter(
"min_samples_split", 2, 20, default_value=2)
min_samples_leaf = UniformIntegerHyperparameter(
"min_samples_leaf", 1, 20, default_value=1)
bootstrap = CategoricalHyperparameter(
"bootstrap", ["True", "False"], default_value="False")
cs.add_hyperparameters([n_estimators, criterion, max_features, min_samples_split, min_samples_leaf,
bootstrap])
return cs
elif optimizer == 'tpe':
space = {'n_estimators': hp.choice('et_n_estimators', [100]),
'criterion': hp.choice('et_criterion', ["mse", "mae"]),
'max_features': hp.uniform('et_max_features', 0, 1),
'min_samples_split': hp.randint('et_min_samples_split', 19) + 2,
'min_samples_leaf': hp.randint('et_min_samples_leaf,', 20) + 1,
'bootstrap': hp.choice('et_bootstrap', ["True", "False"])}
init_trial = {'n_estimators': 100, 'criterion': "mse", 'max_features': 0.5,
'min_samples_split': 2, 'min_samples_leaf': 1, 'bootstrap': "False"}
return space
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter("n_estimators", 100, 1000, default_value=500)
num_leaves = UniformIntegerHyperparameter("num_leaves", 31, 1023, default_value=31)
learning_rate = UniformFloatHyperparameter("learning_rate", 0.025, 0.3, default_value=0.1, log=True)
min_child_weight = UniformIntegerHyperparameter("min_child_weight", 1, 10, default_value=1)
subsample = UniformFloatHyperparameter("subsample", 0.5, 1, default_value=1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.5, 1, default_value=1)
reg_alpha = UniformFloatHyperparameter('reg_alpha', 1e-10, 10, log=True, default_value=1e-10)
reg_lambda = UniformFloatHyperparameter("reg_lambda", 1e-10, 10, log=True, default_value=1e-10)
cs.add_hyperparameters([n_estimators, num_leaves, learning_rate, min_child_weight, subsample,
colsample_bytree, reg_alpha, reg_lambda])
return cs
elif optimizer == 'tpe':
from hyperopt import hp
space = {'n_estimators': hp.randint('lgb_n_estimators', 901) + 100,
'num_leaves': hp.randint('lgb_num_leaves', 993) + 31,
'learning_rate': hp.loguniform('lgb_learning_rate', np.log(0.025), np.log(0.3)),
'min_child_weight': hp.randint('lgb_min_child_weight', 10) + 1,
'subsample': hp.uniform('lgb_subsample', 0.5, 1),
'colsample_bytree': hp.uniform('lgb_colsample_bytree', 0.5, 1),
'reg_alpha': hp.loguniform('lgb_reg_alpha', np.log(1e-10), np.log(10)),
'reg_lambda': hp.loguniform('lgb_reg_lambda', np.log(1e-10), np.log(10))
}
init_trial = {'n_estimators': 500,
'num_leaves': 31,
'learning_rate': 0.1,
'min_child_weight': 1,
'subsample': 1,
'colsample_bytree': 1,
'reg_alpha': 1e-10,
'reg_lambda': 1e-10
}
return space
def get_hyperparameter_search_space(dataset_properties=None,
optimizer='smac'):
cs = ConfigurationSpace()
n_estimators = UniformFloatHyperparameter("n_estimators",
100,
1000,
default_value=500,
q=50)
num_leaves = UniformIntegerHyperparameter("num_leaves",
31,
2047,
default_value=128)
max_depth = Constant('max_depth', 15)
learning_rate = UniformFloatHyperparameter("learning_rate",
1e-3,
0.3,
default_value=0.1,
log=True)
min_child_samples = UniformIntegerHyperparameter("min_child_samples",
5,
30,
default_value=20)
subsample = UniformFloatHyperparameter("subsample",
0.7,
1,
default_value=1,
q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree",
0.7,
1,
default_value=1,
q=0.1)
cs.add_hyperparameters([
n_estimators, num_leaves, max_depth, learning_rate,
min_child_samples, subsample, colsample_bytree
])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
if dataset_properties is not None and \
(dataset_properties.get("sparse") is True or
dataset_properties.get("signed") is False):
allow_chi2 = False
else:
allow_chi2 = True
possible_kernels = ['poly', 'rbf', 'sigmoid', 'cosine']
if allow_chi2:
possible_kernels.append("chi2")
kernel = CategoricalHyperparameter('kernel', possible_kernels, 'rbf')
degree = UniformIntegerHyperparameter('degree', 2, 5, 3)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default=0)
n_components = UniformIntegerHyperparameter(
"n_components", 50, 10000, default=100, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(n_components)
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
gamma_kernels = ["poly", "rbf", "sigmoid"]
if allow_chi2:
gamma_kernels.append("chi2")
gamma_condition = InCondition(gamma, kernel, gamma_kernels)
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
cs.add_condition(gamma_condition)
return cs
def get_runhistory(self, num_success, num_capped, num_timeout):
cs = ConfigurationSpace()
cs.add_hyperparameter(
CategoricalHyperparameter(name="cat_a_b",
choices=["a", "b"],
default_value="a"))
cs.add_hyperparameter(
UniformFloatHyperparameter(name="float_0_1",
lower=0,
upper=1,
default_value=0.5))
cs.add_hyperparameter(
UniformIntegerHyperparameter(name='integer_0_100',
lower=-10,
upper=10,
default_value=0))
rh = runhistory.RunHistory()
rs = numpy.random.RandomState(1)
successes = 0
capped = 0
timeouts = 0
while successes < num_success or capped < num_capped or timeouts < num_timeout:
config, seed, runtime, status, instance_id = \
generate_config(cs=cs, rs=rs)
if status == StatusType.SUCCESS and successes < num_success:
successes += 1
add = True
elif status == StatusType.TIMEOUT:
if runtime < 40 and capped < num_capped:
capped += 1
add = True
elif runtime == 40 and timeouts < num_timeout:
timeouts += 1
add = True
else:
add = False
else:
add = False
if add:
rh.add(config=config,
cost=runtime,
time=runtime,
status=status,
instance_id=instance_id,
seed=seed,
additional_info=None)
return cs, rh
def get_hyperparameter_search_space(dataset_properties=None):
# Copied from libsvm_c
C = UniformFloatHyperparameter(
name="C", lower=0.03125, upper=32768, log=True, default=1.0)
kernel = CategoricalHyperparameter(
name="kernel", choices=['linear', 'poly', 'rbf', 'sigmoid'],
default="rbf")
degree = UniformIntegerHyperparameter(
name="degree", lower=1, upper=5, default=3)
# Changed the gamma value to 0.0 (is 0.1 for classification)
gamma = UniformFloatHyperparameter(
name="gamma", lower=3.0517578125e-05, upper=8, log=True, default=0.1)
# TODO this is totally ad-hoc
coef0 = UniformFloatHyperparameter(
name="coef0", lower=-1, upper=1, default=0)
# probability is no hyperparameter, but an argument to the SVM algo
shrinking = CategoricalHyperparameter(
name="shrinking", choices=["True", "False"], default="True")
tol = UniformFloatHyperparameter(
name="tol", lower=1e-5, upper=1e-1, default=1e-3, log=True)
max_iter = UnParametrizedHyperparameter("max_iter", -1)
# Random Guess
epsilon = UniformFloatHyperparameter(name="epsilon", lower=0.001,
upper=1, default=0.1, log=True)
cs = ConfigurationSpace()
cs.add_hyperparameter(C)
cs.add_hyperparameter(kernel)
cs.add_hyperparameter(degree)
cs.add_hyperparameter(gamma)
cs.add_hyperparameter(coef0)
cs.add_hyperparameter(shrinking)
cs.add_hyperparameter(tol)
cs.add_hyperparameter(max_iter)
cs.add_hyperparameter(epsilon)
degree_depends_on_kernel = InCondition(child=degree, parent=kernel,
values=('poly', 'rbf', 'sigmoid'))
gamma_depends_on_kernel = InCondition(child=gamma, parent=kernel,
values=('poly', 'rbf'))
coef0_depends_on_kernel = InCondition(child=coef0, parent=kernel,
values=('poly', 'sigmoid'))
cs.add_condition(degree_depends_on_kernel)
cs.add_condition(gamma_depends_on_kernel)
cs.add_condition(coef0_depends_on_kernel)
return cs
def get_hyperparameter_search_space(dataset_properties=None, optimizer='smac'):
if optimizer == 'smac':
C = UniformFloatHyperparameter("C", 0.03125, 32768, log=True,
default_value=1.0)
# No linear kernel here, because we have liblinear
kernel = CategoricalHyperparameter(name="kernel",
choices=["rbf", "poly", "sigmoid"],
default_value="rbf")
degree = UniformIntegerHyperparameter("degree", 2, 5, default_value=3)
gamma = UniformFloatHyperparameter("gamma", 3.0517578125e-05, 8,
log=True, default_value=0.1)
coef0 = UniformFloatHyperparameter("coef0", -1, 1, default_value=0)
shrinking = CategoricalHyperparameter("shrinking", ["True", "False"],
default_value="True")
tol = UniformFloatHyperparameter("tol", 1e-5, 1e-1, default_value=1e-3,
log=True)
# cache size is not a hyperparameter, but an argument to the program!
max_iter = UnParametrizedHyperparameter("max_iter", 2000)
cs = ConfigurationSpace()
cs.add_hyperparameters([C, kernel, degree, gamma, coef0, shrinking,
tol, max_iter])
degree_depends_on_poly = EqualsCondition(degree, kernel, "poly")
coef0_condition = InCondition(coef0, kernel, ["poly", "sigmoid"])
cs.add_condition(degree_depends_on_poly)
cs.add_condition(coef0_condition)
return cs
elif optimizer == 'tpe':
coef0 = hp.uniform("libsvm_coef0", -1, 1)
space = {'C': hp.loguniform('libsvm_C', np.log(0.03125), np.log(32768)),
'gamma': hp.loguniform('libsvm_gamma', np.log(3.0517578125e-5), np.log(8)),
'shrinking': hp.choice('libsvm_shrinking', ["True", "False"]),
'tol': hp.loguniform('libsvm_tol', np.log(1e-5), np.log(1e-1)),
'max_iter': hp.choice('libsvm_max_iter', [2000]),
'kernel': hp.choice('libsvm_kernel',
[("poly", {'degree': hp.randint('libsvm_degree', 4) + 2, 'coef0': coef0}),
("rbf", {}),
("sigmoid", {'coef0': coef0})])}
init_trial = {'C': 1,
'gamma': 0.1,
'shrinking': "True",
'tol': 1e-3,
'max_iter': 2000,
'kernel': ("rbf", {})}
return space
def run_roar(python_path,
w_dir,
n_iter=5,
input_file='../rawAllx1000.json',
seeds=[1],
task_ids=None,
max_tries=10):
from smac.configspace import ConfigurationSpace
from ConfigSpace.hyperparameters import UniformIntegerHyperparameter
from smac.scenario.scenario import Scenario
from smac.facade.roar_facade import ROAR
def test_func(cutoff):
cutoff = cutoff.get('x1')
print(cutoff)
result = find_cut_off.main(python_path=python_path,
w_dir=w_dir,
iter=n_iter,
input_file=input_file,
cutoffs=[cutoff],
seeds=seeds,
task_ids=task_ids)
cleaned = [x[1] for x in result if 0.0 < x[1] < 1.0]
mean = np.mean(cleaned) if cleaned else 0.0
mean = mean if mean != 1.0 else 0.0
return 1.0 - mean
cs = ConfigurationSpace()
cutoff_parameter = UniformIntegerHyperparameter('x1',
1,
99,
default_value=50)
cs.add_hyperparameter(cutoff_parameter)
scenario = Scenario({
"run_obj": "quality", # we optimize quality (alternatively runtime)
"runcount-limit": max_tries, # maximum function evaluations
"cs": cs, # configuration space
"deterministic": "true",
"abort_on_first_run_crash": "false",
})
roar = ROAR(scenario=scenario, tae_runner=test_func, rng=1234)
x = roar.optimize()
cost = test_func(x)
return x, cost, roar
def get_cs():
cs = ConfigurationSpace()
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=50,
upper=500,
default_value=50,
log=False)
learning_rate = UniformFloatHyperparameter(name="learning_rate",
lower=0.01,
upper=2,
default_value=0.1,
log=True)
algorithm = CategoricalHyperparameter(name="algorithm",
choices=["SAMME.R", "SAMME"],
default_value="SAMME.R")
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=2,
upper=8,
default_value=3,
log=False)
cs.add_hyperparameters([n_estimators, learning_rate, algorithm, max_depth])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
n_estimators = UniformIntegerHyperparameter(name="n_estimators",
lower=10,
upper=100,
default=10)
max_depth = UniformIntegerHyperparameter(name="max_depth",
lower=2,
upper=10,
default=5)
min_samples_split = UniformIntegerHyperparameter(
name="min_samples_split", lower=2, upper=20, default=2)
min_samples_leaf = UniformIntegerHyperparameter(
name="min_samples_leaf", lower=1, upper=20, default=1)
min_weight_fraction_leaf = Constant('min_weight_fraction_leaf', 1.0)
max_leaf_nodes = UnParametrizedHyperparameter(name="max_leaf_nodes",
value="None")
cs = ConfigurationSpace()
cs.add_hyperparameter(n_estimators)
cs.add_hyperparameter(max_depth)
cs.add_hyperparameter(min_samples_split)
cs.add_hyperparameter(min_samples_leaf)
cs.add_hyperparameter(min_weight_fraction_leaf)
cs.add_hyperparameter(max_leaf_nodes)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
"""Get the configuration space used for hyperparameter searching."""
cs = ConfigurationSpace()
hidden_layer_depth = UniformIntegerHyperparameter(
name="hidden_layer_depth", lower=1, upper=5, default_value=1
)
num_nodes_per_layer = UniformIntegerHyperparameter(
name="num_nodes_per_layer", lower=3, upper=20, default_value=5
)
activation = CategoricalHyperparameter(
name="activation",
choices=["identity", "logistic", "tanh", "relu"],
default_value="relu",
)
alpha = UniformFloatHyperparameter(
name="alpha",
lower=0.0001,
upper=1.0,
default_value=0.0001,
log=True,
)
solver = CategoricalHyperparameter(
name="solver",
choices=["lbfgs", "sgd", "adam"],
default_value="lbfgs",
)
cs.add_hyperparameters(
[
hidden_layer_depth,
num_nodes_per_layer,
activation,
alpha,
solver,
]
)
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = cs.add_hyperparameter(Constant("n_estimators", 100))
criterion = cs.add_hyperparameter(Constant("criterion", "mse"))
max_features = cs.add_hyperparameter(
UniformFloatHyperparameter("max_features", 0.5, 5, default=1))
max_depth = cs.add_hyperparameter(
UnParametrizedHyperparameter(name="max_depth", value="None"))
min_samples_split = cs.add_hyperparameter(
UniformIntegerHyperparameter("min_samples_split", 2, 20,
default=2))
min_samples_leaf = cs.add_hyperparameter(
UniformIntegerHyperparameter("min_samples_leaf", 1, 20, default=1))
min_weight_fraction_leaf = cs.add_hyperparameter(
Constant('min_weight_fraction_leaf', 0.))
bootstrap = cs.add_hyperparameter(
CategoricalHyperparameter("bootstrap", ["True", "False"],
default="False"))
return cs
def create_configspace():
cs = ConfigurationSpace()
# n_estimators = UniformFloatHyperparameter("n_estimators", 100, 600, default_value=200, q=10)
eta = UniformFloatHyperparameter("eta", 0.01, 0.9, default_value=0.3, q=0.01)
min_child_weight = UniformFloatHyperparameter("min_child_weight", 0, 10, default_value=1, q=0.1)
max_depth = UniformIntegerHyperparameter("max_depth", 1, 12, default_value=6)
subsample = UniformFloatHyperparameter("subsample", 0.1, 1, default_value=1, q=0.1)
gamma = UniformFloatHyperparameter("gamma", 0, 10, default_value=0, q=0.1)
colsample_bytree = UniformFloatHyperparameter("colsample_bytree", 0.1, 1, default_value=1., q=0.1)
alpha = UniformFloatHyperparameter("alpha", 0, 10, default_value=0., q=0.1)
_lambda = UniformFloatHyperparameter("lambda", 1, 10, default_value=1, q=0.1)
cs.add_hyperparameters([eta, min_child_weight, max_depth, subsample, gamma,
colsample_bytree, alpha, _lambda])
return cs
def test_write_forbidden(self):
cs =ConfigurationSpace()
hp1 = CategoricalHyperparameter("parent", [0, 1])
hp2 = UniformIntegerHyperparameter("child", 0, 2)
hp3 = UniformIntegerHyperparameter("child2", 0, 2)
hp4 = UniformIntegerHyperparameter("child3", 0, 2)
hp5 = CategoricalHyperparameter("child4", [4,5,6,7])
cs.add_hyperparameters([hp1, hp2, hp3, hp4, hp5])
forb2 = ForbiddenEqualsClause(hp1, 1)
forb3 = ForbiddenInClause(hp2, range(2, 3))
forb4 = ForbiddenInClause(hp3, range(2, 3))
forb5 = ForbiddenInClause(hp4, range(2, 3))
forb6 = ForbiddenInClause(hp5, [6,7])
and1 = ForbiddenAndConjunction(forb2, forb3)
and2 = ForbiddenAndConjunction(forb2, forb4)
and3 = ForbiddenAndConjunction(forb2, forb5)
cs.add_forbidden_clauses([forb2, forb3, forb4, forb5, forb6, and1, and2, and3])
value = irace.write(cs) # generates file called forbidden.txt
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigurationSpace()
n_estimators = Constant("n_estimators", 100)
criterion = CategoricalHyperparameter("criterion",
['mse', 'friedman_mse', 'mae'])
max_features = UniformFloatHyperparameter("max_features",
0.1,
1.0,
default_value=1)
max_depth = UnParametrizedHyperparameter(name="max_depth",
value="None")
max_leaf_nodes = UnParametrizedHyperparameter("max_leaf_nodes", "None")
min_samples_split = UniformIntegerHyperparameter("min_samples_split",
2,
20,
default_value=2)
min_samples_leaf = UniformIntegerHyperparameter("min_samples_leaf",
1,
20,
default_value=1)
min_impurity_decrease = UnParametrizedHyperparameter(
'min_impurity_decrease', 0.0)
bootstrap = CategoricalHyperparameter("bootstrap", ["True", "False"],
default_value="False")
cs.add_hyperparameters([
n_estimators, criterion, max_features, max_depth, max_leaf_nodes,
min_samples_split, min_samples_leaf, min_impurity_decrease,
bootstrap
])
return cs
def get_hyperparameter_search_space(**kwargs):
percentile = UniformIntegerHyperparameter(name="percentile",
lower=1,
upper=99,
default_value=10)
score_func = CategoricalHyperparameter(
name="score_func",
choices=["chi2", "f_classif", "mutual_info"],
default_value="f_classif")
cs = ConfigurationSpace()
cs.add_hyperparameters([percentile, score_func])
return cs
def config_space(self):
"""KNN hyperparameter space."""
n_neighbors = UniformIntegerHyperparameter(
'n_neighbors', lower=1, upper=100, default_value=5
)
leaf_size = UniformIntegerHyperparameter(
'leaf_size', lower=1, upper=100, default_value=20
)
metric = CategoricalHyperparameter(
'metric',
['euclidean', 'manhattan', 'chebyshev', 'minkowski'],
default_value='euclidean'
)
p_param = UniformIntegerHyperparameter('p', 1, 5, default_value=2)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters((n_neighbors, leaf_size, metric, p_param))
# Conditionals on hyperparameters specific to kernels.
config.add_condition(
InCondition(child=p_param, parent=metric, values=['minkowski'])
)
return config
def config_space(self):
"""Decision tree hyperparameter space."""
n_estimators = UniformIntegerHyperparameter(
'n_estimators', lower=10, upper=800, default_value=100
)
criterion = CategoricalHyperparameter(
'criterion', ['gini', 'entropy'], default_value='gini'
)
# NOTE: Default value = 'none' is translated to None in base class.
# ConfigSpace does not allow for None values as default.
max_depth = CategoricalHyperparameter(
'max_depth', [5, 10, 20, 'none'], default_value='none'
)
# NOTE: Default value = 'none' is translated to None in base class.
# ConfigSpace does not allow for None values as default.
max_features = CategoricalHyperparameter(
'max_features', ['auto', 'sqrt', 'log2', 'none'],
default_value='none'
)
min_samples_leaf = UniformIntegerHyperparameter(
'min_samples_leaf', lower=2, upper=5, default_value=3
)
# Add hyperparameters to config space.
config = ConfigurationSpace()
config.seed(self.random_state)
config.add_hyperparameters(
(
min_samples_leaf,
n_estimators,
criterion,
max_depth,
max_features
)
)
return config
def get_hyperparameter_search_space(dataset_properties=None):
# More than degree 3 is too expensive!
degree = UniformIntegerHyperparameter("degree", 2, 3, 2)
interaction_only = CategoricalHyperparameter("interaction_only",
["False", "True"],
"False")
include_bias = CategoricalHyperparameter("include_bias",
["True", "False"], "True")
cs = ConfigurationSpace()
cs.add_hyperparameter(degree)
cs.add_hyperparameter(interaction_only)
cs.add_hyperparameter(include_bias)
return cs
def get_hyperparameter_search_space(
dataset_properties: Optional[Dict[str, str]] = None
) -> ConfigurationSpace:
cs = ConfigurationSpace()
scale_offset = UniformFloatHyperparameter('scale_offset', lower=0, upper=0.4, default_value=0.2)
translate_percent_offset = UniformFloatHyperparameter('translate_percent_offset', lower=0, upper=0.4,
default_value=0.2)
shear = UniformIntegerHyperparameter('shear', lower=0, upper=45, default_value=30)
rotate = UniformIntegerHyperparameter('rotate', lower=0, upper=360, default_value=45)
use_augmenter = CategoricalHyperparameter('use_augmenter', choices=[True, False], default_value=True)
cs.add_hyperparameters([scale_offset, translate_percent_offset])
cs.add_hyperparameters([shear, rotate, use_augmenter])
# only add hyperparameters to configuration space if we are using the augmenter
cs.add_condition(CS.EqualsCondition(scale_offset, use_augmenter, True))
cs.add_condition(CS.EqualsCondition(translate_percent_offset, use_augmenter, True))
cs.add_condition(CS.EqualsCondition(shear, use_augmenter, True))
cs.add_condition(CS.EqualsCondition(rotate, use_augmenter, True))
return cs
def test_in_condition(self):
hp1 = CategoricalHyperparameter("parent", [0, 1, 2, 3, 4])
hp2 = UniformIntegerHyperparameter("child", 0, 10)
hp3 = UniformIntegerHyperparameter("child2", 0, 10)
hp4 = CategoricalHyperparameter("grandchild", ["hot", "cold", "warm"])
self.assertRaisesRegex(
ValueError,
"Forbidden clause must be instantiated with a "
"legal hyperparameter value for "
"'parent, Type: Categorical, Choices: {0, 1, 2, 3, 4}, "
"Default: 0', but got '5'",
ForbiddenInClause, hp1, [5],
)
forb1 = ForbiddenInClause(hp2, [5, 6, 7, 8, 9])
forb1_ = ForbiddenInClause(hp2, [9, 8, 7, 6, 5])
forb2 = ForbiddenInClause(hp2, [5, 6, 7, 8])
forb3 = ForbiddenInClause(hp3, [5, 6, 7, 8, 9])
forb4 = ForbiddenInClause(hp4, ["hot", "cold"])
forb4_ = ForbiddenInClause(hp4, ["hot", "cold"])
forb5 = ForbiddenInClause(hp1, [3, 4])
forb5_ = ForbiddenInClause(hp1, [3, 4])
self.assertEqual(forb5, forb5_)
self.assertEqual(forb4, forb4_)
# print("\nTest1:")
self.assertEqual(forb1, forb1_)
# print("\nTest2:")
self.assertNotEqual(forb1, forb2)
# print("\nTest3:")
self.assertNotEqual(forb1, forb3)
# print("\nTest4:")
self.assertEqual("Forbidden: child in {5, 6, 7, 8, 9}", str(forb1))
# print("\nTest5:")
self.assertRaisesRegex(ValueError,
"Is_forbidden must be called with the "
"instanstatiated hyperparameter in the "
"forbidden clause; you are missing "
"'child'", forb1.is_forbidden,
{'parent': 1}, True)
# print("\nTest6:")
self.assertFalse(forb1.is_forbidden({'parent': 1}, strict=False))
# print("\nTest7:")
for i in range(0, 5):
self.assertFalse(forb1.is_forbidden({'child': i}, True))
# print("\nTest8:")
for i in range(5, 10):
self.assertTrue(forb1.is_forbidden({'child': i}, True))
# Test forbidden on vector values
hyperparameter_idx = {
hp1.name: 0,
hp2.name: 1
}
forb1.set_vector_idx(hyperparameter_idx)
# print("\nTest9:")
self.assertFalse(forb1.is_forbidden_vector(np.array([np.NaN, np.NaN]), strict=False))
# print("\nTest10:")
self.assertFalse(forb1.is_forbidden_vector(np.array([np.NaN, 0]), strict=False))
correct_vector_value = hp2._inverse_transform(6)
# print("\nTest11:")
# print(correct_vector_value, np.array([np.NaN, correct_vector_value]))
self.assertTrue(forb1.is_forbidden_vector(np.array([np.NaN, correct_vector_value]), strict=False))