def _get_fidelity_choices(
ntrees_choice: str,
subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]:
assert ntrees_choice in ['fixed', 'variable']
assert subsample_choice in ['fixed', 'variable']
fidelity1 = dict(
# TODO: this value was 100 in the original code. Please check if 100 or 1000.
fixed=CS.Constant('n_estimators', value=1000),
variable=CS.UniformIntegerHyperparameter('n_estimators',
lower=100,
upper=1000,
default_value=1000,
log=False))
fidelity2 = dict(fixed=CS.Constant('subsample', value=1),
variable=CS.UniformFloatHyperparameter(
'subsample',
lower=0.1,
upper=1,
default_value=1,
log=False))
ntrees = fidelity1[ntrees_choice]
subsample = fidelity2[subsample_choice]
return ntrees, subsample
def _get_fidelity_choices(
n_estimators_choice: str,
subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]:
assert n_estimators_choice in ['fixed', 'variable']
assert subsample_choice in ['fixed', 'variable']
fidelity1 = dict(fixed=CS.Constant('n_estimators', value=2000),
variable=CS.UniformIntegerHyperparameter(
'n_estimators',
lower=50,
upper=2000,
default_value=2000,
log=False))
fidelity2 = dict(fixed=CS.Constant('subsample', value=1),
variable=CS.UniformFloatHyperparameter(
'subsample',
lower=0.1,
upper=1,
default_value=1,
log=False))
n_estimators = fidelity1[n_estimators_choice]
subsample = fidelity2[subsample_choice]
return n_estimators, subsample
def _get_fidelity_choices(iter_choice: str, subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]:
"""Fidelity space available --- specifies the fidelity dimensions
For SVM, only a single fidelity exists, i.e., subsample fraction.
if fidelity_choice == 0
uses the entire data (subsample=1), reflecting the black-box setup
else
parameterizes the fraction of data to subsample
"""
assert iter_choice in ['fixed', 'variable']
assert subsample_choice in ['fixed', 'variable']
fidelity1 = dict(
fixed=CS.Constant('iter', value=1000),
variable=CS.UniformIntegerHyperparameter(
'iter', lower=10, upper=1000, default_value=1000, log=False
)
)
fidelity2 = dict(
fixed=CS.Constant('subsample', value=1.0),
variable=CS.UniformFloatHyperparameter(
'subsample', lower=0.1, upper=1.0, default_value=1.0, log=False
)
)
iter = fidelity1[iter_choice]
subsample = fidelity2[subsample_choice]
return iter, subsample
def get_hyperparameter_search_space_micro(seed):
"""
Small version of svm config space, featuring important hyperparameters
as used by:
http://metalearning.ml/2018/papers/metalearn2018_paper70.pdf
Parameters
----------
seed: int
Random seed that will be used to sample random configurations
Returns
-------
cs: ConfigSpace.ConfigurationSpace
The configuration space object
"""
cs = ConfigSpace.ConfigurationSpace('sklearn.svm.SVC', seed)
kernel = ConfigSpace.Constant(name='svc__kernel', value='rbf')
C = ConfigSpace.UniformFloatHyperparameter(name='svc__C', lower=0.03125, upper=32768, log=True, default_value=1.0)
gamma = ConfigSpace.UniformFloatHyperparameter(
name='svc__gamma', lower=3.0517578125e-05, upper=8, log=True, default_value=0.1)
cs.add_hyperparameters([
kernel,
C,
gamma
])
return cs
def get_random_forest_default_search_space():
cs = ConfigSpace.ConfigurationSpace()
imputation = ConfigSpace.CategoricalHyperparameter(
'imputation__strategy', ['mean', 'median', 'most_frequent'])
n_estimators = ConfigSpace.Constant("classifier__n_estimators", 100)
criterion = ConfigSpace.CategoricalHyperparameter("classifier__criterion",
["gini", "entropy"],
default_value="gini")
# The maximum number of features used in the forest is calculated as m^max_features, where
# m is the total number of features, and max_features is the hyperparameter specified below.
# The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This
# corresponds with Geurts' heuristic.
max_features = ConfigSpace.UniformFloatHyperparameter(
"classifier__max_features", 0., 1., default_value=0.5)
# max_depth = ConfigSpace.UnParametrizedHyperparameter("classifier__max_depth", None)
min_samples_split = ConfigSpace.UniformIntegerHyperparameter(
"classifier__min_samples_split", 2, 20, default_value=2)
min_samples_leaf = ConfigSpace.UniformIntegerHyperparameter(
"classifier__min_samples_leaf", 1, 20, default_value=1)
min_weight_fraction_leaf = ConfigSpace.UnParametrizedHyperparameter(
"classifier__min_weight_fraction_leaf", 0.)
# max_leaf_nodes = ConfigSpace.UnParametrizedHyperparameter("classifier__max_leaf_nodes", None)
bootstrap = ConfigSpace.CategoricalHyperparameter("classifier__bootstrap",
["True", "False"],
default_value="True")
cs.add_hyperparameters([
imputation, n_estimators, criterion, max_features, min_samples_split,
min_samples_leaf, min_weight_fraction_leaf, bootstrap
])
return cs
def get_hyperparameter_search_space(seed) -> ConfigSpaceWrapper:
"""
The random forest configuration space based on the search space from
auto-sklearn:
https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/components/classification/random_forest.py
Parameters
----------
seed: int
Random seed that will be used to sample random configurations
Returns
-------
cs: ConfigSpace.ConfigurationSpace
The configuration space object
"""
cs = ConfigSpace.ConfigurationSpace(
'sklearn.ensemble.RandomForestClassifier', seed)
# TODO: parameterize the number of estimators?
n_estimators = ConfigSpace.Constant(name='n_estimators', value=100)
criterion = ConfigSpace.CategoricalHyperparameter(
name='criterion', choices=['gini', 'entropy'], default_value='gini')
max_features = ConfigSpace.UniformFloatHyperparameter(name='max_features',
lower=0.,
upper=1.,
default_value=0.5)
# max_depth = ConfigSpace.UnParametrizedHyperparameter(
# name='randomforestclassifier__max_depth', value=None)
min_samples_split = ConfigSpace.UniformIntegerHyperparameter(
name='min_samples_split', lower=2, upper=20, default_value=2)
min_samples_leaf = ConfigSpace.UniformIntegerHyperparameter(
name='min_samples_leaf', lower=1, upper=20, default_value=1)
min_weight_fraction_leaf = ConfigSpace.UnParametrizedHyperparameter(
name='min_weight_fraction_leaf', value=0.)
# max_leaf_nodes = ConfigSpace.UnParametrizedHyperparameter(
# name='randomforestclassifier__max_leaf_nodes', value=None)
bootstrap = ConfigSpace.CategoricalHyperparameter(name='bootstrap',
choices=[True, False],
default_value=True)
hyperparameters = [
n_estimators,
criterion,
max_features,
# max_depth,
min_samples_split,
min_samples_leaf,
min_weight_fraction_leaf,
# max_leaf_nodes,
bootstrap
]
return ConfigSpaceWrapper(cs, hyperparameters, None)
def _get_fidelity_choices(
iter_choice: str,
subsample_choice: str) -> Tuple[Hyperparameter, Hyperparameter]:
fidelity1 = dict(fixed=CS.Constant('iter', value=243),
variable=CS.UniformIntegerHyperparameter(
'iter',
lower=3,
upper=243,
default_value=243,
log=False))
fidelity2 = dict(fixed=CS.Constant('subsample', value=1),
variable=CS.UniformFloatHyperparameter(
'subsample',
lower=0.1,
upper=1,
default_value=1,
log=False))
iter = fidelity1[iter_choice]
subsample = fidelity2[subsample_choice]
return iter, subsample
def get_fidelity_space(seed=None, fidelity_choice=1):
"""Fidelity space available --- specifies the fidelity dimensions
If fidelity_choice is 0
Fidelity space is the maximal fidelity, akin to a black-box function
If fidelity_choice is 1
Fidelity space is a single fidelity, in this case the number of trees (n_estimators)
If fidelity_choice is 2
Fidelity space is a single fidelity, in this case the fraction of dataset (subsample)
If fidelity_choice is >2
Fidelity space is multi-multi fidelity, all possible fidelities
"""
z_cs = CS.ConfigurationSpace(seed=seed)
subsample_lower_bound = np.max((0.1, (0.1 or self.lower_bound_train_size)))
if fidelity_choice == 0:
# only subsample as fidelity
ntrees = CS.Constant('n_estimators', value=100)
subsample = CS.Constant('subsample', value=1)
elif fidelity_choice == 1:
# only n_estimators as fidelity
ntrees = CS.UniformIntegerHyperparameter(
'n_estimators', lower=2, upper=100, default_value=10, log=False
)
subsample = CS.Constant('subsample', value=1)
elif fidelity_choice == 2:
# only subsample as fidelity
ntrees = CS.Constant('n_estimators', value=100)
subsample = CS.UniformFloatHyperparameter(
'subsample', lower=subsample_lower_bound, upper=1, default_value=0.33, log=False
)
else:
# both n_estimators and subsample as fidelities
ntrees = CS.UniformIntegerHyperparameter(
'n_estimators', lower=2, upper=100, default_value=10, log=False
)
subsample = CS.UniformFloatHyperparameter(
'subsample', lower=subsample_lower_bound, upper=1, default_value=0.33, log=False
)
z_cs.add_hyperparameters([ntrees, subsample])
return z_cs
def _get_fidelity_choices(subsample_choice: str) -> Hyperparameter:
assert subsample_choice in ['fixed', 'variable']
fidelity = dict(fixed=CS.Constant('subsample', value=1),
variable=CS.UniformFloatHyperparameter(
'subsample',
lower=0.1,
upper=1.0,
default_value=1.0,
log=False))
subsample = fidelity[subsample_choice]
return subsample
def get_hyperparameter_search_space(seed: int) -> ConfigSpaceWrapper:
"""
The decision tree configuration space based on the search space from
auto-sklearn:
https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/components/classification/decision_tree.py
Parameters
----------
seed: int
Random seed that will be used to sample random configurations
Returns
-------
cs: ConfigSpace.ConfigurationSpace
The configuration space object
"""
cs = ConfigSpace.ConfigurationSpace("sklearn.tree.DecisionTreeClassifier",
seed)
criterion = ConfigSpace.CategoricalHyperparameter(
name='criterion', choices=['gini', 'entropy'], default_value='gini')
min_samples_split = ConfigSpace.UniformIntegerHyperparameter(
name='min_samples_split', lower=2, upper=20, default_value=2)
min_samples_leaf = ConfigSpace.UniformIntegerHyperparameter(
name='min_samples_leaf', lower=1, upper=20, default_value=1)
min_weight_fraction_leaf = ConfigSpace.Constant(
name='min_weight_fraction_leaf', value=0.0)
max_features = ConfigSpace.UnParametrizedHyperparameter(
name='max_features', value=1.0)
min_impurity_decrease = ConfigSpace.UnParametrizedHyperparameter(
'min_impurity_decrease', 0.0)
# TODO: max_leaf_nodes one can only be tuned once config space allows for this.
hyperparameters = [
criterion, max_features, min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, min_impurity_decrease
]
return ConfigSpaceWrapper(cs, hyperparameters, None)
def get_hyperparameter_search_space():
cs = ConfigSpace.ConfigurationSpace()
cs.add_hyperparameter(ConfigSpace.Constant("sparsity", 0.9))
return cs
def get_fixed_conditional_fc_config(nr_features,
feature_type,
max_nr_layers=19,
nr_units=64):
# Config
optimizers = ['Adam', 'AdamW', 'SGD', 'SGDW']
decay_scheduler = ['cosine_annealing', 'cosine_decay', 'exponential_decay']
include_hyperparameter = ['Yes', 'No']
cs = ConfigSpace.ConfigurationSpace()
# Architecture parameters
cs.add_hyperparameter(ConfigSpace.Constant("num_layers", max_nr_layers))
cs.add_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter("batch_size",
lower=8,
upper=256,
default_value=16,
log=True))
# class weights and feature preprocessing
cs.add_hyperparameter(
ConfigSpace.CategoricalHyperparameter('class_weights',
include_hyperparameter))
cs.add_hyperparameter(ConfigSpace.Constant('feature_type', feature_type))
feature_preprocessing = ConfigSpace.CategoricalHyperparameter(
'feature_preprocessing', include_hyperparameter)
number_pca_components = ConfigSpace.UniformIntegerHyperparameter(
'pca_components',
lower=2,
upper=nr_features,
default_value=nr_features - 1)
cs.add_hyperparameter(feature_preprocessing)
cs.add_hyperparameter(number_pca_components)
cs.add_condition(
ConfigSpace.EqualsCondition(number_pca_components,
feature_preprocessing, 'Yes'))
# Regularization parameters
decay_type = ConfigSpace.CategoricalHyperparameter('decay_type',
decay_scheduler)
cs.add_hyperparameter(decay_type)
lr_fraction = ConfigSpace.UniformFloatHyperparameter("final_lr_fraction",
lower=1e-4,
upper=1.,
default_value=1e-2,
log=True)
cs.add_hyperparameter(lr_fraction)
cs.add_condition(
ConfigSpace.EqualsCondition(lr_fraction, decay_type,
'exponential_decay'))
cs.add_hyperparameter(
ConfigSpace.UniformFloatHyperparameter("learning_rate",
lower=10e-4,
upper=10e-1,
default_value=10e-2,
log=True))
optimizer = ConfigSpace.CategoricalHyperparameter('optimizer', optimizers)
momentum = ConfigSpace.UniformFloatHyperparameter("momentum",
lower=0.0,
upper=0.9,
default_value=0.9)
cs.add_hyperparameter(optimizer)
cs.add_hyperparameter(momentum)
cs.add_condition(
ConfigSpace.OrConjunction(
ConfigSpace.EqualsCondition(momentum, optimizer, 'SGDW'),
ConfigSpace.EqualsCondition(momentum, optimizer, 'SGD')))
weight_decay = ConfigSpace.UniformFloatHyperparameter("weight_decay",
lower=10e-5,
upper=10e-3,
default_value=10e-4)
activate_weight_decay = ConfigSpace.CategoricalHyperparameter(
'activate_weight_decay', include_hyperparameter)
cs.add_hyperparameter(weight_decay)
cs.add_hyperparameter(activate_weight_decay)
cs.add_condition(
ConfigSpace.EqualsCondition(weight_decay, activate_weight_decay,
'Yes'))
cs.add_hyperparameter(
ConfigSpace.CategoricalHyperparameter('activate_batch_norm',
include_hyperparameter))
activate_dropout = ConfigSpace.CategoricalHyperparameter(
'activate_dropout', include_hyperparameter)
cs.add_hyperparameter(activate_dropout)
# it is the upper bound of the nr of layers,
# since the configuration will actually be sampled.
for i in range(1, max_nr_layers + 1):
cs.add_hyperparameter(
ConfigSpace.Constant("num_units_%d" % i, nr_units))
dropout_value = ConfigSpace.UniformFloatHyperparameter(
"dropout_%d" % i, lower=0, upper=0.7, default_value=0.5)
cs.add_hyperparameter(dropout_value)
cs.add_condition(
ConfigSpace.EqualsCondition(dropout_value, activate_dropout,
'Yes'))
return cs
def get_super_fcresnet_config(layers_block=2,
num_res_blocks=18,
super_blocks=3):
# Config
optimizers = ['SGDW', 'AdamW', 'SGD', 'Adam']
decay_scheduler = ['cosine_annealing', 'cosine_decay', 'exponential_decay']
include_hyperparameter = ['Yes', 'No']
cs = ConfigSpace.ConfigurationSpace()
# Architecture parameters
nr_layers_block = ConfigSpace.Constant("num_layers", layers_block)
nr_res_blocks = ConfigSpace.Constant("num_res_blocks", num_res_blocks)
res_block_type = ConfigSpace.Constant('block_type', 'PreRes')
nr_super_blocks = ConfigSpace.Constant('num_super_blocks', super_blocks)
cs.add_hyperparameter(res_block_type)
cs.add_hyperparameter(nr_layers_block)
cs.add_hyperparameter(nr_res_blocks)
cs.add_hyperparameter(nr_super_blocks)
cs.add_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter("batch_size",
lower=8,
upper=256,
default_value=16,
log=True))
# Regularition parameters
decay_type = ConfigSpace.CategoricalHyperparameter('decay_type',
decay_scheduler)
cs.add_hyperparameter(decay_type)
mixout = ConfigSpace.CategoricalHyperparameter('mixout',
include_hyperparameter)
mixout_alpha = ConfigSpace.UniformFloatHyperparameter('mixout_alpha',
lower=0,
upper=1,
default_value=0.2)
cs.add_hyperparameter(mixout)
cs.add_hyperparameter(mixout_alpha)
cs.add_condition(ConfigSpace.EqualsCondition(mixout_alpha, mixout, 'Yes'))
shake_shake = ConfigSpace.CategoricalHyperparameter(
'shake-shake', include_hyperparameter)
cs.add_hyperparameter(shake_shake)
cs.add_hyperparameter(
ConfigSpace.UniformFloatHyperparameter("learning_rate",
lower=10e-4,
upper=10e-1,
default_value=10e-2,
log=True))
optimizer = ConfigSpace.CategoricalHyperparameter('optimizer', optimizers)
momentum = ConfigSpace.UniformFloatHyperparameter("momentum",
lower=0.0,
upper=0.9,
default_value=0.9)
cs.add_hyperparameter(optimizer)
cs.add_hyperparameter(momentum)
cs.add_condition(
ConfigSpace.OrConjunction(
ConfigSpace.EqualsCondition(momentum, optimizer, 'SGDW'),
ConfigSpace.EqualsCondition(momentum, optimizer, 'SGD')))
weight_decay = ConfigSpace.UniformFloatHyperparameter("weight_decay",
lower=10e-5,
upper=10e-3,
default_value=10e-4)
activate_weight_decay = ConfigSpace.CategoricalHyperparameter(
'activate_weight_decay', include_hyperparameter)
cs.add_hyperparameter(weight_decay)
cs.add_hyperparameter(activate_weight_decay)
cs.add_condition(
ConfigSpace.EqualsCondition(weight_decay, activate_weight_decay,
'Yes'))
# it is the upper bound of the nr of layers,
# since the configuration will actually be sampled.
for i in range(1, super_blocks + 1):
for j in range(1, layers_block + 1):
n_units = ConfigSpace.UniformIntegerHyperparameter(
"num_units_%d_%d" % (i, j),
lower=16,
upper=256,
default_value=64,
log=True)
cs.add_hyperparameter(n_units)
activate_dropout = cs.add_hyperparameter(
ConfigSpace.CategoricalHyperparameter('activate_dropout',
include_hyperparameter))
# get dropout value for super_block
# the same value will be used for
# residual blocks in the super_blocks
for i in range(1, super_blocks + 1):
# for now only dropout between layers
# 1 layer dropout for res block
dropout = ConfigSpace.UniformFloatHyperparameter("dropout_%d_1" % i,
lower=0,
upper=0.7,
default_value=0.5)
cs.add_hyperparameter(dropout)
cs.add_condition(
ConfigSpace.EqualsCondition(dropout, activate_dropout, 'Yes'), )
return cs
def get_fixed_conditional_fcresnet_config(nr_features,
feature_type,
layers_block=2,
num_res_blocks=9,
super_blocks=1,
nr_units=64):
# Config
optimizers = ['Adam', 'AdamW', 'SGD', 'SGDW']
shake_shake_config = ['YYY', 'YNY', 'YYN', 'YNN']
decay_scheduler = ['cosine_annealing', 'cosine_decay', 'exponential_decay']
include_hyperparameter = ['Yes', 'No']
cs = ConfigSpace.ConfigurationSpace()
# Architecture parameters
nr_layers_block = ConfigSpace.Constant("num_layers", layers_block)
nr_res_blocks = ConfigSpace.Constant("num_res_blocks", num_res_blocks)
res_block_type = ConfigSpace.Constant('block_type', 'BasicRes')
nr_super_blocks = ConfigSpace.Constant('num_super_blocks', super_blocks)
cs.add_hyperparameter(res_block_type)
cs.add_hyperparameter(nr_layers_block)
cs.add_hyperparameter(nr_res_blocks)
cs.add_hyperparameter(nr_super_blocks)
cs.add_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter("batch_size",
lower=8,
upper=256,
default_value=16,
log=True))
# class weights and feature preprocessing
cs.add_hyperparameter(
ConfigSpace.CategoricalHyperparameter('class_weights',
include_hyperparameter))
cs.add_hyperparameter(ConfigSpace.Constant('feature_type', feature_type))
feature_preprocessing = ConfigSpace.CategoricalHyperparameter(
'feature_preprocessing', include_hyperparameter)
number_pca_components = ConfigSpace.UniformIntegerHyperparameter(
'pca_components',
lower=2,
upper=nr_features - 1,
default_value=nr_features - 1)
cs.add_hyperparameter(feature_preprocessing)
cs.add_hyperparameter(number_pca_components)
cs.add_condition(
ConfigSpace.EqualsCondition(number_pca_components,
feature_preprocessing, 'Yes'))
# Regularition parameters
decay_type = ConfigSpace.CategoricalHyperparameter('decay_type',
decay_scheduler)
cs.add_hyperparameter(decay_type)
lr_fraction = ConfigSpace.UniformFloatHyperparameter("final_lr_fraction",
lower=1e-4,
upper=1.,
default_value=1e-2,
log=True)
cs.add_hyperparameter(lr_fraction)
cs.add_condition(
ConfigSpace.EqualsCondition(lr_fraction, decay_type,
'exponential_decay'))
# add mixup regularization status
mixup = ConfigSpace.CategoricalHyperparameter('mixout',
include_hyperparameter)
# add mixup alpha value
mixup_alpha = ConfigSpace.UniformFloatHyperparameter('mixout_alpha',
lower=0,
upper=1,
default_value=0.2)
cs.add_hyperparameter(mixup)
cs.add_hyperparameter(mixup_alpha)
# add mixup_alpha only if mixup is active
cs.add_condition(ConfigSpace.EqualsCondition(mixup_alpha, mixup, 'Yes'))
# shake-shake status
shake_shake = ConfigSpace.CategoricalHyperparameter(
'shake-shake', include_hyperparameter)
# shake-shake config
shake_config = ConfigSpace.CategoricalHyperparameter(
'shake_config', shake_shake_config)
cs.add_hyperparameter(shake_shake)
cs.add_hyperparameter(shake_config)
# add shake-shake config only if shake-shake is active
cs.add_condition(
ConfigSpace.EqualsCondition(shake_config, shake_shake, 'Yes'))
# batch norm status
cs.add_hyperparameter(
ConfigSpace.CategoricalHyperparameter('activate_batch_norm',
include_hyperparameter))
cs.add_hyperparameter(
ConfigSpace.UniformFloatHyperparameter("learning_rate",
lower=10e-4,
upper=10e-1,
default_value=10e-2,
log=True))
optimizer = ConfigSpace.CategoricalHyperparameter('optimizer', optimizers)
momentum = ConfigSpace.UniformFloatHyperparameter("momentum",
lower=0.0,
upper=0.9,
default_value=0.9)
cs.add_hyperparameter(optimizer)
cs.add_hyperparameter(momentum)
cs.add_condition(
ConfigSpace.OrConjunction(
ConfigSpace.EqualsCondition(momentum, optimizer, 'SGDW'),
ConfigSpace.EqualsCondition(momentum, optimizer, 'SGD')))
# weight decay status
weight_decay = ConfigSpace.CategoricalHyperparameter(
'activate_weight_decay', include_hyperparameter)
# weight decay value
weight_decay_value = ConfigSpace.UniformFloatHyperparameter(
"weight_decay", lower=10e-5, upper=10e-3, default_value=10e-4)
cs.add_hyperparameter(weight_decay)
cs.add_hyperparameter(weight_decay_value)
# add weight decay value only if active
cs.add_condition(
ConfigSpace.EqualsCondition(weight_decay_value, weight_decay, 'Yes'))
# nr units for input layer
cs.add_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter('input_layer_units',
lower=8,
upper=256,
default_value=16,
log=True))
# it is the upper bound of the nr of layers,
# since the configuration will actually be sampled.
for i in range(1, super_blocks + 1):
for j in range(1, layers_block + 1):
n_units = ConfigSpace.UniformIntegerHyperparameter(
"num_units_%d_%d" % (i, j),
lower=8,
upper=256,
default_value=16,
log=True)
cs.add_hyperparameter(n_units)
# dropout status
dropout = ConfigSpace.CategoricalHyperparameter('activate_dropout',
include_hyperparameter)
cs.add_hyperparameter(dropout)
# get dropout value for super_block
# the same value will be used for
# residual blocks in the super_blocks
for i in range(1, super_blocks + 1):
# for now only dropout between layers
# 1 layer dropout for res block
# dropout value
dropout_value = ConfigSpace.UniformFloatHyperparameter(
"dropout_%d_1" % i, lower=0, upper=0.7, default_value=0.5)
cs.add_hyperparameter(dropout_value)
# add dropout value only if active
cs.add_condition(
ConfigSpace.EqualsCondition(dropout_value, dropout, 'Yes'))
return cs
def get_hyperparameter_search_space(seed: int) -> ConfigSpaceWrapper:
"""
The extra trees configuration space based on the search space from
auto-sklearn:
https://github.com/automl/auto-sklearn/blob/master/autosklearn/pipeline/components/classification/extra_trees.py
Parameters
----------
seed: int
Random seed that will be used to sample random configurations
Returns
-------
cs: ConfigSpace.ConfigurationSpace
The configuration space object
"""
cs = ConfigSpace.ConfigurationSpace(
'sklearn.ensemble.ExtraTreesClassifier', seed)
# TODO: parameterize the number of estimators?
n_estimators = ConfigSpace.Constant(
name='extratreesclassifier__n_estimators', value=100)
criterion = ConfigSpace.CategoricalHyperparameter(
name='criterion', choices=['gini', 'entropy'], default_value='gini')
# The maximum number of features used in the forest is calculated as m^max_features, where
# m is the total number of features, and max_features is the hyperparameter specified below.
# The default is 0.5, which yields sqrt(m) features as max_features in the estimator. This
# corresponds with Geurts' heuristic.
max_features = ConfigSpace.UniformFloatHyperparameter(name='max_features',
lower=0.,
upper=1.,
default_value=0.5)
# max_depth = ConfigSpace.UnParametrizedHyperparameter(name='extratreesclassifier__max_depth', value=None)
min_samples_split = ConfigSpace.UniformIntegerHyperparameter(
name='min_samples_split', lower=2, upper=20, default_value=2)
min_samples_leaf = ConfigSpace.UniformIntegerHyperparameter(
name='min_samples_leaf', lower=1, upper=20, default_value=1)
min_weight_fraction_leaf = ConfigSpace.UnParametrizedHyperparameter(
name='min_weight_fraction_leaf', value=0.)
# max_leaf_nodes = ConfigSpace.UnParametrizedHyperparameter(name='max_leaf_nodes', value=None)
min_impurity_decrease = ConfigSpace.UnParametrizedHyperparameter(
name='min_impurity_decrease', value=0.0)
bootstrap = ConfigSpace.CategoricalHyperparameter('bootstrap',
[True, False],
default_value=False)
hyperparameters = [
n_estimators,
criterion,
max_features,
# max_depth,
min_samples_split,
min_samples_leaf,
min_weight_fraction_leaf,
# max_leaf_nodes,
min_impurity_decrease,
bootstrap
]
return ConfigSpaceWrapper(cs, hyperparameters, None)
def set_hyper_constant(cs, hp_name, value):
hp = cs.get_hyperparameter(hp_name)
name = hp.name
new_hp = CS.Constant(name=name, value=value)
cs._hyperparameters[name] = new_hp
