def get_configspace():
""" Returns the configuration space for the network to be configured in the example. """
config_space = CS.ConfigurationSpace()
config_space.add_hyperparameters([
CSH.CategoricalHyperparameter('activation', ['tanh', 'relu']),
CS.UniformFloatHyperparameter('learning_rate_init',
lower=1e-6,
upper=1e-2,
log=True)
])
solver = CSH.CategoricalHyperparameter('solver', ['sgd', 'adam'])
config_space.add_hyperparameter(solver)
beta_1 = CS.UniformFloatHyperparameter('beta_1', lower=0, upper=1)
config_space.add_hyperparameter(beta_1)
condition = CS.EqualsCondition(beta_1, solver, 'adam')
config_space.add_condition(condition)
beta_2 = CS.UniformFloatHyperparameter('beta_2', lower=0, upper=1)
config_space.add_hyperparameter(beta_2)
condition = CS.EqualsCondition(beta_2, solver, 'adam')
config_space.add_condition(condition)
return config_space
def get_hyperparameter_search_space(self, **pipeline_config):
pipeline_config = self.pipeline.get_pipeline_config(**pipeline_config)
cs = ConfigSpace.ConfigurationSpace()
hp_batch_loss_computation = cs.add_hyperparameter(
CSH.CategoricalHyperparameter(
"batch_loss_computation_technique",
sorted(self.batch_loss_computation_techniques.keys())))
for name, technique in self.batch_loss_computation_techniques.items():
parent = {
'parent': hp_batch_loss_computation,
'value': name
} if hp_batch_loss_computation is not None else None
cs.add_configuration_space(
prefix=name,
configuration_space=technique.get_hyperparameter_search_space(
**pipeline_config),
delimiter=ConfigWrapper.delimiter,
parent_hyperparameter=parent)
possible_loss_comps = sorted(
list(
set(pipeline_config["batch_loss_computation_techniques"]).
intersection(self.batch_loss_computation_techniques.keys())))
if 'batch_loss_computation_techniques' not in pipeline_config.keys():
cs.add_hyperparameter(
CSH.CategoricalHyperparameter(
"batch_loss_computation_technique", possible_loss_comps))
self._check_search_space_updates()
return cs
def cs_ab():
"""
Defining the configuration space in case of
AdaBoost Classifier
"""
cs1 = ConfigSpace.ConfigurationSpace()
cs2 = ConfigSpace.ConfigurationSpace()
hp1 = csh.CategoricalHyperparameter('algorithm', choices=['0', '1'])
hp2 = csh.CategoricalHyperparameter('imputation', choices=['0', '1', '2'])
hp3 = csh.UniformIntegerHyperparameter('max_depth',
lower=1,
upper=10,
log=False)
hp4 = csh.UniformFloatHyperparameter('learning_rate',
lower=0.01,
upper=2,
log=True)
hp5 = csh.UniformIntegerHyperparameter('n_estimators',
lower=50,
upper=500,
log=False)
# imputation case
cs1.add_hyperparameters([hp1, hp2, hp3, hp4, hp5])
# no imputation case
cs2.add_hyperparameters([hp1, hp3, hp4, hp5])
return cs1, cs2
def cs_svm(per_kernel=False):
"""
Defining the configuration space in case of
SVM
"""
cs1 = ConfigSpace.ConfigurationSpace()
cs2 = ConfigSpace.ConfigurationSpace()
hp1 = csh.CategoricalHyperparameter('shrinking', choices=['0', '1'])
hp2 = csh.CategoricalHyperparameter('imputation', choices=['0', '1', '2'])
hp3 = csh.CategoricalHyperparameter('kernel', choices=['0', '1'])
hp4 = csh.UniformFloatHyperparameter('C', lower=2**(-5), upper=2**15, log=True)
hp5 = csh.UniformFloatHyperparameter('coef0', lower=-1, upper=1, log=False)
hp6 = csh.UniformFloatHyperparameter('gamma', lower=2**(-15), upper=2**3, log=True)
hp7 = csh.UniformFloatHyperparameter('tol', lower=10**(-5), upper=10**(-1), log=True)
if per_kernel:
cs1.add_hyperparameters([hp1, hp2, hp4, hp5, hp6, hp7])
cs2.add_hyperparameters([hp1, hp4, hp5, hp6, hp7])
else:
cs1.add_hyperparameters([hp1, hp2, hp3, hp4, hp5, hp6, hp7])
cs2.add_hyperparameters([hp1, hp3, hp4, hp5, hp6, hp7])
return cs1, cs2
def get_configspace():
cs = CS.ConfigurationSpace()
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(name='sl_weight',
lower=1,
upper=100000,
log=True,
default_value=100))
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(name='sl_exp',
lower=0.5,
upper=2,
log=True,
default_value=1))
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(name='pl_exp',
lower=0.5,
upper=2,
log=True,
default_value=1))
cs.add_hyperparameter(
CSH.CategoricalHyperparameter(name='forward_flow',
choices=[False, True],
default_value=False))
cs.add_hyperparameter(
CSH.CategoricalHyperparameter(name='weighted_sl_loss',
choices=[False, True],
default_value=False))
return cs
def get_hyperparameter_search_space(self, **pipeline_config):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
cs = CS.ConfigurationSpace()
augment = cs.add_hyperparameter(
CSH.CategoricalHyperparameter('augment', [True, False]))
autoaugment = cs.add_hyperparameter(
CSH.CategoricalHyperparameter('autoaugment', [True, False]))
fastautoaugment = cs.add_hyperparameter(
CSH.CategoricalHyperparameter('fastautoaugment', [True, False]))
cutout = cs.add_hyperparameter(
CSH.CategoricalHyperparameter('cutout', [True, False]))
cutout_length = cs.add_hyperparameter(
CSH.UniformIntegerHyperparameter('length',
lower=0,
upper=20,
log=False))
cutout_holes = cs.add_hyperparameter(
CSH.UniformIntegerHyperparameter('cutout_holes',
lower=1,
upper=3,
log=False))
cs.add_condition(CS.EqualsCondition(cutout_length, cutout, True))
cs.add_condition(CS.EqualsCondition(cutout_holes, cutout, True))
cs.add_condition(CS.EqualsCondition(autoaugment, augment, True))
cs.add_condition(CS.EqualsCondition(fastautoaugment, augment, True))
return cs
def cs_gb():
"""
Defining the configuration space in case of
GradientBoosting Classifier
"""
cs1 = ConfigSpace.ConfigurationSpace()
cs2 = ConfigSpace.ConfigurationSpace()
hp1 = csh.CategoricalHyperparameter('criterion', choices=['0', '1'])
hp2 = csh.CategoricalHyperparameter('imputation', choices=['0', '1', '2'])
hp3 = csh.UniformIntegerHyperparameter('max_depth', lower=1, upper=10, log=False)
hp4 = csh.UniformFloatHyperparameter('learning_rate', lower=0.01, upper=1, log=True)
hp5 = csh.UniformIntegerHyperparameter('n_estimators', lower=50, upper=500, log=False)
hp6 = csh.UniformFloatHyperparameter('max_features', lower=0.1, upper=0.9, log=False)
hp7 = csh.UniformIntegerHyperparameter('min_samples_leaf', lower=1,
upper=20, log=False)
hp8 = csh.UniformIntegerHyperparameter('min_samples_split', lower=2,
upper=20, log=False)
# imputation case
cs1.add_hyperparameters([hp1, hp2, hp3, hp4, hp5, hp6, hp7, hp8])
# no imputation case
cs2.add_hyperparameters([hp1, hp3, hp4, hp5, hp6, hp7, hp8])
return cs1, cs2
def get_configspace(self):
cs = CS.ConfigurationSpace()
cs.add_hyperparameter(
CSH.UniformIntegerHyperparameter(name='gtn_score_transform_type',
lower=0,
upper=7,
log=False,
default_value=7))
cs.add_hyperparameter(
CSH.UniformFloatHyperparameter(name='gtn_step_size',
lower=1e-2,
upper=100,
log=True,
default_value=1))
cs.add_hyperparameter(
CSH.CategoricalHyperparameter(name='gtn_mirrored_sampling',
choices=[False, True],
default_value=False))
cs.add_hyperparameter(
CSH.CategoricalHyperparameter(name='gtn_nes_step_size',
choices=[False, True],
default_value=False))
return cs
def get_configspace(self):
cs = CS.ConfigurationSpace()
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='gtn_score_transform_type', lower=0, upper=7, log=False, default_value=7))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='gtn_step_size', lower=0.1, upper=1, log=True, default_value=0.5))
cs.add_hyperparameter(CSH.CategoricalHyperparameter(name='gtn_mirrored_sampling', choices=[False, True], default_value=True))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='gtn_noise_std', lower=0.01, upper=1, log=True, default_value=0.1))
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='ddqn_init_episodes', lower=1, upper=20, log=True, default_value=10))
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='ddqn_batch_size', lower=64, upper=256, log=False, default_value=128))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='ddqn_gamma', lower=0.001, upper=0.1, log=True, default_value=0.01))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='ddqn_lr', lower=1e-4, upper=5e-3, log=True, default_value=1e-3))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='ddqn_tau', lower=0.005, upper=0.05, log=True, default_value=0.01))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='ddqn_eps_init', lower=0.8, upper=1, log=True, default_value=0.9))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='ddqn_eps_min', lower=0.005, upper=0.05, log=True, default_value=0.05))
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(name='ddqn_eps_decay', lower=0.01, upper=0.2, log=True, default_value=0.1))
cs.add_hyperparameter(CSH.CategoricalHyperparameter(name='ddqn_activation_fn', choices=['tanh', 'relu', 'leakyrelu', 'prelu'], default_value='relu'))
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='ddqn_hidden_size', lower=48, upper=192, log=True, default_value=128))
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='ddqn_hidden_layer', lower=1, upper=2, log=False, default_value=2))
cs.add_hyperparameter(CSH.CategoricalHyperparameter(name='acrobot_activation_fn', choices=['tanh', 'relu', 'leakyrelu', 'prelu'], default_value='leakyrelu'))
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='acrobot_hidden_size', lower=48, upper=192, log=True, default_value=128))
cs.add_hyperparameter(CSH.UniformIntegerHyperparameter(name='acrobot_hidden_layer', lower=1, upper=2, log=False, default_value=1))
return cs
def get_configspace():
cs = CS.ConfigurationSpace()
topology = CSH.CategoricalHyperparameter(
'topology', [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12])
lr = CSH.UniformFloatHyperparameter('lr',
lower=1e-4,
upper=1,
default_value='1e-1',
log=True)
momentum = CSH.UniformFloatHyperparameter('momentum',
lower=0.5,
upper=1,
default_value=0.9,
log=False)
weight_decay = CSH.UniformFloatHyperparameter('weight_decay',
lower=0.00001,
upper=0.00005,
default_value=0.00001,
log=False)
batch_size = CSH.CategoricalHyperparameter('batch_size',
[1, 2, 4, 8, 16],
default_value=8)
cs.add_hyperparameters(
[topology, lr, momentum, weight_decay, batch_size])
return cs
def cs_dt():
"""
Defining the configuration space in case of
Decision Tree Classifier
"""
cs1 = ConfigSpace.ConfigurationSpace()
cs2 = ConfigSpace.ConfigurationSpace()
hp1 = csh.CategoricalHyperparameter('criterion',
choices=['0', '1'])
hp2 = csh.CategoricalHyperparameter('imputation',
choices=['0', '1', '2'])
hp3 = csh.UniformFloatHyperparameter('max_features', lower=0.1,
upper=0.9, log=False)
hp4 = csh.UniformIntegerHyperparameter('min_samples_leaf', lower=1,
upper=20, log=False)
hp5 = csh.UniformIntegerHyperparameter('min_samples_split', lower=2,
upper=20, log=False)
# imputation case
cs1.add_hyperparameters([hp1, hp2, hp3, hp4, hp5])
# no imputation case
cs2.add_hyperparameters([hp1, hp3, hp4, hp5])
return cs1, cs2
def get_hyperparameter_search_space(self, **pipeline_config):
pipeline_config = self.pipeline.get_pipeline_config(**pipeline_config)
cs = ConfigSpace.ConfigurationSpace()
possible_over_sampling_methods = set(pipeline_config["over_sampling_methods"]).intersection(self.over_sampling_methods.keys())
possible_under_sampling_methods = set(pipeline_config["under_sampling_methods"]).intersection(self.under_sampling_methods.keys())
possible_target_size_strategies = set(pipeline_config["target_size_strategies"]).intersection(self.target_size_strategies.keys())
selector_over_sampling = cs.add_hyperparameter(CSH.CategoricalHyperparameter("over_sampling_method", possible_over_sampling_methods))
selector_under_sampling = cs.add_hyperparameter(CSH.CategoricalHyperparameter("under_sampling_method", possible_under_sampling_methods))
cs.add_hyperparameter(CSH.CategoricalHyperparameter("target_size_strategy", possible_target_size_strategies))
for method_name, method_type in self.over_sampling_methods.items():
if method_name not in possible_over_sampling_methods:
continue
method_cs = method_type.get_hyperparameter_search_space()
cs.add_configuration_space( prefix=method_name, configuration_space=method_cs, delimiter=ConfigWrapper.delimiter,
parent_hyperparameter={'parent': selector_over_sampling, 'value': method_name})
for method_name, method_type in self.under_sampling_methods.items():
if method_name not in possible_under_sampling_methods:
continue
method_cs = method_type.get_hyperparameter_search_space()
cs.add_configuration_space( prefix=method_name, configuration_space=method_cs, delimiter=ConfigWrapper.delimiter,
parent_hyperparameter={'parent': selector_under_sampling, 'value': method_name})
return self._apply_user_updates(cs)
def get_configspace():
config_space = CS.ConfigurationSpace()
filter_count = [4] #[4, 8, 16]
filter_size = [7] #[3, 5, 7, 9]
l1_fc = CSH.CategoricalHyperparameter('l1_fc', filter_count)
l1_fs = CSH.CategoricalHyperparameter('l1_fs', filter_size)
config_space.add_hyperparameters([l1_fc, l1_fs])
#l2_fc = CSH.CategoricalHyperparameter('l2_fc', filter_count)
#l2_fs = CSH.CategoricalHyperparameter('l2_fs', filter_size)
#config_space.add_hyperparameters([l2_fc, l2_fs])
l3 = CSH.UniformIntegerHyperparameter('l3', lower=20, upper=500)
l4 = CSH.UniformIntegerHyperparameter('l4', lower=20, upper=500)
l5 = CSH.UniformIntegerHyperparameter('l5', lower=10, upper=300)
config_space.add_hyperparameters([l3, l4, l5])
latent = CSH.UniformIntegerHyperparameter('latent',
lower=20,
upper=100)
#latent = CSH.CategoricalHyperparameter('latent', [33])
config_space.add_hyperparameters([latent])
return config_space
def get_hyperparameter_search_space(
self,
dataset_properties: Optional[Dict[str, Any]] = None,
default: Optional[str] = None,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None) -> ConfigurationSpace:
cs = ConfigurationSpace()
if dataset_properties is None:
dataset_properties = dict()
dataset_properties = {**self.dataset_properties, **dataset_properties}
available_preprocessors = self.get_available_components(
dataset_properties=dataset_properties,
include=include,
exclude=exclude)
if len(available_preprocessors) == 0:
raise ValueError("no encoders found, please add a encoder")
if default is None:
defaults = ['OneHotEncoder', 'OrdinalEncoder', 'NoEncoder']
for default_ in defaults:
if default_ in available_preprocessors:
if include is not None and default_ not in include:
continue
if exclude is not None and default_ in exclude:
continue
default = default_
break
# add only no encoder to choice hyperparameters in case the dataset is only numerical
if len(dataset_properties['categorical_columns']) == 0:
default = 'NoEncoder'
preprocessor = CSH.CategoricalHyperparameter('__choice__',
['NoEncoder'],
default_value=default)
else:
preprocessor = CSH.CategoricalHyperparameter(
'__choice__',
list(available_preprocessors.keys()),
default_value=default)
cs.add_hyperparameter(preprocessor)
# add only child hyperparameters of early_preprocessor choices
for name in preprocessor.choices:
preprocessor_configuration_space = available_preprocessors[name].\
get_hyperparameter_search_space(dataset_properties)
parent_hyperparameter = {'parent': preprocessor, 'value': name}
cs.add_configuration_space(
name,
preprocessor_configuration_space,
parent_hyperparameter=parent_hyperparameter)
self.configuration_space = cs
self.dataset_properties = dataset_properties
return cs
def get_configspace():
""":return: ConfigurationsSpace-Object
Here is the main place to create particular hyperparameters to tune.
Particular hyperparameter should be defined as:
hyperparameter = type_of_parameter(name, lower_range, upper_range, default_value, logging)
add.hyperparameter([hyperparameter])
"""
cs = CS.ConfigurationSpace()
# num_pca = CSH.UniformIntegerHyperparameter('num_pca', lower=850, upper=930, default_value=900, log=True)
# cs.add_hyperparameters([num_pca])
lr = CSH.UniformFloatHyperparameter('lr',
lower=1e-6,
upper=1e-1,
default_value='1e-2',
log=True)
optimizer = CSH.CategoricalHyperparameter('optimizer', ['Adam', 'SGD'])
sgd_momentum = CSH.UniformFloatHyperparameter('sgd_momentum',
lower=0.0,
upper=0.99,
default_value=0.9,
log=False)
dropout_rate_1 = CSH.UniformFloatHyperparameter('dropout_rate_1',
lower=0.0,
upper=0.5,
default_value=0.1,
log=False)
dropout_rate_2 = CSH.UniformFloatHyperparameter('dropout_rate_2',
lower=0.0,
upper=0.5,
default_value=0.1,
log=False)
num_fc_units_1 = CSH.UniformIntegerHyperparameter('num_fc_units_1',
lower=512,
upper=2048,
default_value=1024,
log=True)
num_fc_units_2 = CSH.UniformIntegerHyperparameter('num_fc_units_2',
lower=256,
upper=512,
default_value=256,
log=True)
activation = CSH.CategoricalHyperparameter('activation',
['tanh', 'relu'])
cs.add_hyperparameters([
lr, optimizer, sgd_momentum, dropout_rate_1, dropout_rate_2,
num_fc_units_1, num_fc_units_2, activation
])
# The hyperparameter sgd_momentum will be used,if the configuration
# contains 'SGD' as optimizer.
cond = CS.EqualsCondition(sgd_momentum, optimizer, 'SGD')
cs.add_condition(cond)
return cs
def get_hyperparameter_search_space(self,
dataset_properties: Optional[Dict[str, BaseDatasetPropertiesType]] = None,
default: Optional[str] = None,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None) -> ConfigurationSpace:
cs = ConfigurationSpace()
if dataset_properties is None:
dataset_properties = dict()
dataset_properties = {**self.dataset_properties, **dataset_properties}
available_preprocessors = self.get_available_components(dataset_properties=dataset_properties,
include=include,
exclude=exclude)
if len(available_preprocessors) == 0:
raise ValueError("no image normalizers found, please add an image normalizer")
if default is None:
defaults = ['ImageNormalizer', 'NoNormalizer']
for default_ in defaults:
if default_ in available_preprocessors:
if include is not None and default_ not in include:
continue
if exclude is not None and default_ in exclude:
continue
default = default_
break
updates = self._get_search_space_updates()
if '__choice__' in updates.keys():
choice_hyperparameter = updates['__choice__']
if not set(choice_hyperparameter.value_range).issubset(available_preprocessors):
raise ValueError("Expected given update for {} to have "
"choices in {} got {}".format(self.__class__.__name__,
available_preprocessors,
choice_hyperparameter.value_range))
preprocessor = CSH.CategoricalHyperparameter('__choice__',
choice_hyperparameter.value_range,
default_value=choice_hyperparameter.default_value)
else:
preprocessor = CSH.CategoricalHyperparameter('__choice__',
list(available_preprocessors.keys()),
default_value=default)
cs.add_hyperparameter(preprocessor)
# add only child hyperparameters of preprocessor choices
for name in preprocessor.choices:
preprocessor_configuration_space = available_preprocessors[name].\
get_hyperparameter_search_space(dataset_properties)
parent_hyperparameter = {'parent': preprocessor, 'value': name}
cs.add_configuration_space(name, preprocessor_configuration_space,
parent_hyperparameter=parent_hyperparameter)
self.configuration_space = cs
self.dataset_properties = dataset_properties
return cs
def get_configspace():
""" It builds the configuration space with the needed hyperparameters.
It is easily possible to implement different types of hyperparameters.
Beside float-hyperparameters on a log scale, it is also able to handle
categorical input parameter.
:return: ConfigurationsSpace-Object
"""
cs = CS.ConfigurationSpace()
cs.add_hyperparameters([
CSH.UniformFloatHyperparameter(
'optimizer:lr',
lower=0.001,
upper=0.01,
log=True,
),
CSH.UniformIntegerHyperparameter(
'model:temp_reg_params.history',
lower=4,
upper=12,
default_value=12,
),
CSH.UniformIntegerHyperparameter(
'model:temp_reg_params.n_layers',
lower=2,
upper=8,
default_value=3,
),
CSH.OrdinalHyperparameter(
'model:temp_reg_params.n_channels',
sequence=[2, 4, 8, 16, 32],
default_value=8,
),
CSH.OrdinalHyperparameter(
'model:temp_reg_params.kernel_size',
sequence=[1, 3, 5],
default_value=1,
),
CSH.CategoricalHyperparameter(
'model:temp_reg_params.activation',
choices="ReLU ELU LeakyReLU SELU".split(),
),
CSH.CategoricalHyperparameter(
'model:biases_type.location',
choices="LxT L+T".split(),
),
CSH.CategoricalHyperparameter(
'model:biases_type.weekday',
choices=["", "W", "WxT"],
),
CSH.CategoricalHyperparameter(
'model:biases_type.month',
choices=["", "M", "MxT"],
),
])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
degree = CSH.UniformIntegerHyperparameter("degree", lower=2, upper=3)
interaction_only = CSH.CategoricalHyperparameter(
"interaction_only", [False, True])
include_bias = CSH.CategoricalHyperparameter("include_bias",
[True, False])
cs = ConfigSpace.ConfigurationSpace()
cs.add_hyperparameters([degree, interaction_only, include_bias])
return cs
def get_configspace():
"""
Here we define the configuration space for the hyperparameters for the model.
Returns:
ConfigSpace-object
"""
cs = CS.ConfigurationSpace()
cs.add_hyperparameter(CSH.UniformFloatHyperparameter(
'lr', lower=1e-6, upper=1e-2, default_value='1e-2', log=True))
cs.add_hyperparameter(CSH.CategoricalHyperparameter(
'act_f', ['ReLU', 'Tanh'], default_value='ReLU'))
# For demonstration purposes, we add different optimizers as categorical hyperparameters.
# To show how to use conditional hyperparameters with ConfigSpace, we'll add the optimizers 'Adam' and 'SGD'.
# SGD has a different parameter 'momentum'.
optimizer = CSH.CategoricalHyperparameter('optimizer',
['Adam', 'SGD'])
cs.add_hyperparameter(optimizer)
sgd_momentum = CSH.UniformFloatHyperparameter('sgd_momentum',
lower=0.0, upper=0.99, default_value=0.9,
log=False)
cs.add_hyperparameter(sgd_momentum)
# The hyperparameter sgd_momentum will be used,
# if the configuration contains 'SGD' as optimizer.
cond = CS.EqualsCondition(sgd_momentum, optimizer, 'SGD')
cs.add_condition(cond)
# The hyperparameters (hidden units for layer 2 and 3) are conditional parameters conditioned by
# the number of hidden layers.
# These dependencies are realised with inequality conditions.
num_hidden_layers = CSH.UniformIntegerHyperparameter('num_hidden_layers', lower=1, upper=3, default_value=1)
cs.add_hyperparameter(num_hidden_layers)
hidden_dim_1 = CSH.UniformIntegerHyperparameter('hidden_dim_1', lower=100, upper=1000, log=False)
cs.add_hyperparameter(hidden_dim_1)
hidden_dim_2 = CSH.UniformIntegerHyperparameter('hidden_dim_2', lower=100, upper=1000, log=False)
cs.add_hyperparameter(hidden_dim_2)
hidden_dim_3 = CSH.UniformIntegerHyperparameter('hidden_dim_3', lower=100, upper=1000, log=False)
cs.add_hyperparameter(hidden_dim_3)
# Use inequality conditions
cond = CS.GreaterThanCondition(hidden_dim_2, num_hidden_layers, 1)
cs.add_condition(cond)
cond = CS.GreaterThanCondition(hidden_dim_3, num_hidden_layers, 2)
cs.add_condition(cond)
return cs
def get_configspace():
"""
It builds the configuration space with the needed hyperparameters.
It is easily possible to implement different types of
hyperparameters. Beside float-hyperparameters on a log scale, it
is also able to handle categorical input parameter.
:return: ConfigurationsSpace-Object
"""
cs = CS.ConfigurationSpace()
if True:
lr = CSH.UniformFloatHyperparameter('lr',
lower=0.0001, upper=0.01, default_value=0.001,
log=True)
hidden_dim = CSH.CategoricalHyperparameter('hidden_dim',
[16, 32, 64, 128, 256])
activation = CSH.CategoricalHyperparameter('activation',
['tanh', 'relu'])
epsilon = CSH.UniformFloatHyperparameter('epsilon',
lower=0.01, upper=1., default_value=1., log=True)
decay_rate = CSH.UniformFloatHyperparameter('decay_rate',
lower=0.001, upper=.1, default_value=.01, log=True)
gamma = CSH.CategoricalHyperparameter('gamma',[0.99])
action_dim = CSH.UniformIntegerHyperparameter('action_dim',
lower=3, upper=30)
seed = CSH.UniformIntegerHyperparameter('seed',
lower=0, upper=4000, default_value=42)
dropout_rate = CSH.UniformFloatHyperparameter('dropout_rate',
lower=0.0, upper=0.8, default_value=0.0)
else:
lr = CSH.UniformFloatHyperparameter('lr',
lower=0.001, upper=0.0012, default_value=0.001,
log=True)
hidden_dim = CSH.CategoricalHyperparameter('hidden_dim',
[16, 32, 64, 128, 256])
activation = CSH.CategoricalHyperparameter('activation',
['tanh', 'relu'])
epsilon = CSH.UniformFloatHyperparameter('epsilon',
lower=0.01, upper=1., default_value=1., log=True)
decay_rate = CSH.UniformFloatHyperparameter('decay_rate',
lower=0.001, upper=.1, default_value=.01)
gamma = CSH.UniformFloatHyperparameter('gamma',
lower=0.5, upper=0.999, default_value=0.99, log=True)
action_dim = CSH.UniformIntegerHyperparameter('action_dim',
lower=3, upper=100)
seed = CSH.UniformIntegerHyperparameter('seed',
lower=0, upper=1000, default_value=42)
dropout_rate = CSH.UniformFloatHyperparameter('dropout_rate',
lower=0.0, upper=0.3, default_value=0.0)
cs.add_hyperparameters([lr, hidden_dim, activation, epsilon,
gamma, action_dim, seed, dropout_rate,
decay_rate,])
return cs
def get_hyperparameter_search_space(dataset_properties=None):
cs = ConfigSpace.ConfigurationSpace()
n_components = CSH.UniformIntegerHyperparameter("n_components", lower=10, upper=2000)
algorithm = CSH.CategoricalHyperparameter('algorithm', ['parallel', 'deflation'])
whiten = CSH.CategoricalHyperparameter('whiten', [True, False])
fun = CSH.CategoricalHyperparameter('fun', ['logcosh', 'exp', 'cube'])
cs.add_hyperparameters([n_components, algorithm, whiten, fun])
cs.add_condition(CSC.EqualsCondition(n_components, whiten, True))
return cs
def get_configspace():
"""
Define all the hyperparameters that need to be optimised and store them in config
"""
cs = CS.ConfigurationSpace()
dense_units = CSH.UniformIntegerHyperparameter('dense_units',
lower=64,
upper=256,
default_value=128)
initial_lr = CSH.UniformFloatHyperparameter('initial_lr',
lower=1e-3,
upper=1e-1,
default_value='1e-2',
log=True)
optimizer = CSH.CategoricalHyperparameter('optimizer',
get('opti_dict').keys())
sgd_momentum = CSH.UniformFloatHyperparameter('sgd_momentum',
lower=0.0,
upper=0.99,
default_value=0.9,
log=False)
nesterov = CSH.CategoricalHyperparameter('nesterov', ['True', 'False'])
cs.add_hyperparameters(
[initial_lr, optimizer, sgd_momentum, nesterov, dense_units])
lr_scheduler = CSH.CategoricalHyperparameter('lr_scheduler',
['Exponential', 'Cosine'])
weight_decay = CSH.UniformFloatHyperparameter('weight_decay',
lower=1e-5,
upper=1e-3,
default_value=3e-4,
log=True)
drop_path_prob = CSH.UniformFloatHyperparameter('drop_path_prob',
lower=0,
upper=0.4,
default_value=0.3,
log=False)
grad_clip_value = CSH.UniformIntegerHyperparameter('grad_clip_value',
lower=4,
upper=8,
default_value=5)
cs.add_hyperparameters(
[lr_scheduler, drop_path_prob, weight_decay, grad_clip_value])
cond = CS.EqualsCondition(sgd_momentum, optimizer, 'sgd')
cs.add_condition(cond)
cond2 = CS.EqualsCondition(nesterov, optimizer, 'sgd')
cs.add_condition(cond2)
return cs
def _create_config_space(dict_hyperparams):
"""Create the hyperparameters hyperspace."""
config_space = ConfigurationSpace()
if not isinstance(dict_hyperparams, dict):
raise TypeError('Hyperparams must be a dictionary.')
for name, hyperparam in dict_hyperparams.items():
hp_type = hyperparam['type']
if hp_type == 'int':
hp_range = hyperparam.get('range') or hyperparam.get('values')
hp_min = min(hp_range)
hp_max = max(hp_range)
hp_default = hyperparam.get('default') or hp_min
config_space.add_hyperparameter(
hp.UniformIntegerHyperparameter(name,
hp_min,
hp_max,
default_value=hp_default))
elif hp_type == 'float':
hp_range = hyperparam.get('range') or hyperparam.get('values')
hp_min = min(hp_range)
hp_max = max(hp_range)
hp_default = hyperparam.get('default') or hp_min
config_space.add_hyperparameter(
hp.UniformFloatHyperparameter(name,
hp_min,
hp_max,
default_value=hp_default))
elif hp_type == 'bool':
hp_default = bool(hyperparam.get('default'))
config_space.add_hyperparameter(
hp.CategoricalHyperparameter(name, ['true', 'false'],
default_value=hp_default))
elif hp_type == 'str':
hp_range = hyperparam.get('range') or hyperparam.get('values')
hp_range = [_NONE if hp is None else hp for hp in hp_range]
hp_default = hyperparam.get('default') or hp_range[0]
hp_default = _NONE if hp_default is None else hp_default
config_space.add_hyperparameter(
hp.CategoricalHyperparameter(name,
hp_range,
default_value=hp_default))
return config_space
def get_duc2004_space():
cs = CS.ConfigurationSpace(seed=1)
beam = CSH.CategoricalHyperparameter('beam', choices=[2, 3, 4, 5, 6])
ngram = CSH.CategoricalHyperparameter('no_repeat_ngram_size',
choices=[2, 3, 4])
lenpen = CSH.UniformFloatHyperparameter('lenpen',
lower=0.1,
upper=2.0,
q=0.1)
min_len = CSH.UniformIntegerHyperparameter('min_len', lower=5, upper=19)
max_len_b = CSH.CategoricalHyperparameter('max_len_b', choices=[20])
max_len_a = CSH.CategoricalHyperparameter('max_len_a', choices=[0])
cs.add_hyperparameters(
[beam, ngram, lenpen, min_len, max_len_b, max_len_a])
return cs
def create_hyperparameter(var_type, name, lower=None, upper=None, log=False, q=None, choices=None):
"""
Parameters
----------
var_type: type
int or float or str
dist: string
"u" (uniform) or "c" (categorical)
q: float or int
any positive real number
lower: float or int
upper bound of the variable
upper: float or int
lower bound of the variable
choices: list of str or float or int
the choices of categorical parameter
Returns
-------
ConfigSpace.hyperparameters object
the information of hyperparameter
"""
if var_type == int:
return CSH.UniformIntegerHyperparameter(name=name, lower=lower, upper=upper, log=log, q=q)
elif var_type == float:
return CSH.UniformFloatHyperparameter(name=name, lower=lower, upper=upper, log=log, q=q)
elif var_type == str or var_type == bool:
return CSH.CategoricalHyperparameter(name=name, choices=choices)
else:
raise ValueError("The hp_type must be chosen from [int, float, str, bool]")
def _convert_hyper_parameters_to_cs(self):
# type: () -> CS.ConfigurationSpace
cs = CS.ConfigurationSpace(seed=self._seed)
for p in self._hyper_parameters:
if isinstance(p, UniformParameterRange):
hp = CSH.UniformFloatHyperparameter(p.name,
lower=p.min_value,
upper=p.max_value,
log=False,
q=p.step_size)
elif isinstance(p, UniformIntegerParameterRange):
hp = CSH.UniformIntegerHyperparameter(p.name,
lower=p.min_value,
upper=p.max_value,
log=False,
q=p.step_size)
elif isinstance(p, DiscreteParameterRange):
hp = CSH.CategoricalHyperparameter(p.name, choices=p.values)
else:
raise ValueError(
"HyperParameter type {} not supported yet with OptimizerBOHB"
.format(type(p)))
cs.add_hyperparameter(hp)
return cs
def get_hyperparameter_search_space(self,
dataset_info=None,
**pipeline_config):
pipeline_config = self.pipeline.get_pipeline_config(**pipeline_config)
cs = ConfigSpace.ConfigurationSpace()
possible_techniques = set(
pipeline_config['batch_loss_computation_techniques']).intersection(
self.batch_loss_computation_techniques.keys())
hp_batch_loss_computation = CSH.CategoricalHyperparameter(
"batch_loss_computation_technique", sorted(possible_techniques))
cs.add_hyperparameter(hp_batch_loss_computation)
for name, technique in self.batch_loss_computation_techniques.items():
if name not in possible_techniques:
continue
technique = self.batch_loss_computation_techniques[name]
technique_cs = technique.get_hyperparameter_search_space(
**self._get_search_space_updates(
prefix=("batch_loss_computation_technique", name)))
cs.add_configuration_space(prefix=name,
configuration_space=technique_cs,
delimiter=ConfigWrapper.delimiter,
parent_hyperparameter={
'parent': hp_batch_loss_computation,
'value': name
})
self._check_search_space_updates((possible_techniques, "*"))
return cs
def get_hyperparameter_search_space(
self,
dataset_properties: Optional[Dict[str, str]] = None,
default: Optional[str] = None,
include: Optional[List[str]] = None,
exclude: Optional[List[str]] = None,
) -> ConfigurationSpace:
"""Returns the configuration space of the current chosen components
Args:
dataset_properties (Optional[Dict[str, str]]): Describes the dataset to work on
default (Optional[str]): Default component to use
include: Optional[Dict[str, Any]]: what components to include. It is an exhaustive
list, and will exclusively use this components.
exclude: Optional[Dict[str, Any]]: which components to skip
Returns:
ConfigurationSpace: the configuration space of the hyper-parameters of the
chosen component
"""
cs = ConfigurationSpace()
if dataset_properties is None:
dataset_properties = {}
# Compile a list of legal preprocessors for this problem
available_initializers = self.get_available_components(
dataset_properties=dataset_properties,
include=include,
exclude=exclude)
if len(available_initializers) == 0:
raise ValueError("No initializers found")
if default is None:
defaults = [
'MLPNet',
]
for default_ in defaults:
if default_ in available_initializers:
default = default_
break
initializer = CSH.CategoricalHyperparameter(
'__choice__',
list(available_initializers.keys()),
default_value=default)
cs.add_hyperparameter(initializer)
for name in available_initializers:
initializer_configuration_space = available_initializers[name]. \
get_hyperparameter_search_space(dataset_properties)
parent_hyperparameter = {'parent': initializer, 'value': name}
cs.add_configuration_space(
name,
initializer_configuration_space,
parent_hyperparameter=parent_hyperparameter)
self.configuration_space_ = cs
self.dataset_properties_ = dataset_properties
return cs
def get_space():
"""
Defines the search space to sample from for each hyperparameter for the hyperparameter
optimization. Define all parameters to tune in the given model here.
Returns:
--------
ConfigSpace object containing the search space
"""
space = CS.ConfigurationSpace()
timesteps_per_batch = CSH.CategoricalHyperparameter(
'timesteps_per_batch', [512, 1024, 2048, 4096, 8192])
vf_stepsize = CSH.UniformFloatHyperparameter('vf_stepsize',
lower=2**-5,
upper=2**-2,
log=True)
max_kl = CSH.UniformFloatHyperparameter('max_kl',
lower=2**-2.5,
upper=2**-0.5,
log=True)
gamma = CSH.UniformFloatHyperparameter('gamma',
lower=(1 - (1 / ((10**(-1)) * 4))),
upper=(1 - (1 / ((10**(1.5)) * 4))))
lam = CSH.UniformFloatHyperparameter('lam',
lower=(1 - (1 / ((10**(-1)) * 4))),
upper=(1 - (1 / ((10**(1.5)) * 4))))
space.add_hyperparameters(
[timesteps_per_batch, vf_stepsize, max_kl, gamma, lam])
# Store the defined configuration space to a json file
#with open('configspace.json', 'w') as fh:
# fh.write(json.write(space))
return space
def convert_hyperparameter_ranges(hp_ranges: HyperparameterRanges_Impl) -> \
(HyperparameterRanges_CS, List[str]):
names = []
hps = []
for hp_range in hp_ranges.hp_ranges:
name = hp_range.name
names.append(name)
if isinstance(hp_range, HyperparameterRangeContinuous):
hp = CSH.UniformFloatHyperparameter(name=name,
lower=hp_range.lower_bound,
upper=hp_range.upper_bound,
log=isinstance(
hp_range.scaling,
LogScaling))
elif isinstance(hp_range, HyperparameterRangeInteger):
hp = CSH.UniformIntegerHyperparameter(name=name,
lower=hp_range.lower_bound,
upper=hp_range.upper_bound,
log=isinstance(
hp_range.scaling,
LogScaling))
elif isinstance(hp_range, HyperparameterRangeCategorical):
hp = CSH.CategoricalHyperparameter(name=name,
choices=hp_range.choices)
else:
raise AssertionError(
"Parameter '{}' has unknown type".format(name))
hps.append(hp)
cs = CS.ConfigurationSpace()
cs.add_hyperparameters(hps)
return HyperparameterRanges_CS(cs), names
