def get_config_space(T_max=(10, 500), eta_min=(1e-8, 1e-8)):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'T_max',
T_max)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'eta_min',
eta_min)
return cs
def get_config_space(num_layers=(1, 15),
max_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
mlp_shape=('funnel', 'long_funnel', 'diamond',
'hexagon', 'brick', 'triangle', 'stairs'),
max_dropout=(0, 1.0),
use_dropout=(True, False)):
cs = CS.ConfigurationSpace()
mlp_shape_hp = get_hyperparameter(CSH.CategoricalHyperparameter,
'mlp_shape', mlp_shape)
cs.add_hyperparameter(mlp_shape_hp)
num_layers_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
'num_layers', num_layers)
cs.add_hyperparameter(num_layers_hp)
max_units_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
"max_units", max_units)
cs.add_hyperparameter(max_units_hp)
use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_dropout", use_dropout)
max_dropout_hp = add_hyperparameter(cs, CSH.UniformFloatHyperparameter,
"max_dropout", max_dropout)
cs.add_condition(
CS.EqualsCondition(max_dropout_hp, use_dropout_hp, True))
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'activation',
activation)
return cs
def get_config_space(growth_rate_range=(12, 40), nr_blocks=(3, 4), layer_range=([1, 12], [6, 24], [12, 64], [12, 64]), num_init_features=(32, 128), **kwargs):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter
cs = CS.ConfigurationSpace()
growth_rate_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'growth_rate', growth_rate_range)
cs.add_hyperparameter(growth_rate_hp)
# add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'bn_size', [2, 4])
# add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'num_init_features', num_init_features, log=True)
# add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'bottleneck', [True, False])
blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'blocks', nr_blocks)
cs.add_hyperparameter(blocks_hp)
use_dropout = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'use_dropout', [True, False])
dropout = add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'dropout', [0.0, 1.0])
cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))
if type(nr_blocks[0]) == int:
min_blocks = nr_blocks[0]
max_blocks = nr_blocks[1]
else:
min_blocks = nr_blocks[0][0]
max_blocks = nr_blocks[0][1]
for i in range(1, max_blocks+1):
layer_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'layer_in_block_%d' % i, layer_range[i-1])
cs.add_hyperparameter(layer_hp)
if i > min_blocks:
cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i-1))
return cs
def get_config_space(T_max=(1, 20), T_mult=(1.0, 2.0)):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'T_max',
T_max)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'T_mult',
T_mult)
return cs
def get_config_space(factor=(0.05, 0.5), patience=(3, 10)):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'factor',
factor)
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'patience',
patience)
return cs
def get_config_space(learning_rate=((0.0001, 0.1), True),
weight_decay=(0.0001, 0.1)):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'learning_rate',
learning_rate)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'weight_decay',
weight_decay)
return cs
def get_config_space(
step_size=(1, 10),
gamma=(0.001, 0.9)
):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'step_size', step_size)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'gamma', gamma)
return cs
def get_hyperparameter_search_space(
dataset_info=None,
standardize=(True, False),
method=("yeo-johnson", "box-cox"),
):
cs = ConfigSpace.ConfigurationSpace()
add_hyperparameter(cs, CSH.CategoricalHyperparameter, "standardize", standardize)
if dataset_info is None or (
(dataset_info.x_min_value is None or dataset_info.x_min_value > 0) and not any(dataset_info.categorical_features)):
add_hyperparameter(cs, CSH.CategoricalHyperparameter, "method", method)
return cs
def get_hyperparameter_search_space(self, dataset_info=None, **pipeline_config):
import ConfigSpace
import ConfigSpace.hyperparameters as CSH
pipeline_config = self.pipeline.get_pipeline_config(**pipeline_config)
cs = ConfigSpace.ConfigurationSpace()
batch_size_range = self._get_search_space_updates().get('batch_size', ((32, 500), True))
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'batch_size', batch_size_range)
self._check_search_space_updates('batch_size')
return cs
def get_config_space(num_layers=((1, 15), False),
num_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
dropout=(0.0, 0.8),
use_dropout=(True, False),
**kwargs):
cs = CS.ConfigurationSpace()
num_layers_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
'num_layers', num_layers)
cs.add_hyperparameter(num_layers_hp)
use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_dropout", use_dropout)
for i in range(1, num_layers[0][1] + 1):
n_units_hp = get_hyperparameter(
CSH.UniformIntegerHyperparameter, "num_units_%d" % i,
kwargs.pop("num_units_%d" % i, num_units))
cs.add_hyperparameter(n_units_hp)
if i > num_layers[0][0]:
cs.add_condition(
CS.GreaterThanCondition(n_units_hp, num_layers_hp, i - 1))
if True in use_dropout:
dropout_hp = get_hyperparameter(
CSH.UniformFloatHyperparameter, "dropout_%d" % i,
kwargs.pop("dropout_%d" % i, dropout))
cs.add_hyperparameter(dropout_hp)
dropout_condition_1 = CS.EqualsCondition(
dropout_hp, use_dropout_hp, True)
if i > num_layers[0][0]:
dropout_condition_2 = CS.GreaterThanCondition(
dropout_hp, num_layers_hp, i - 1)
cs.add_condition(
CS.AndConjunction(dropout_condition_1,
dropout_condition_2))
else:
cs.add_condition(dropout_condition_1)
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'activation',
activation)
assert len(
kwargs
) == 0, "Invalid hyperparameter updates for mlpnet: %s" % str(kwargs)
return (cs)
def get_config_space(
learning_rate=((1e-4, 0.1), True),
momentum=((0.1, 0.99), True),
weight_decay=(1e-5, 0.1),
alpha=(0.1,0.99)
):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'learning_rate', learning_rate)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'momentum', momentum)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'weight_decay', weight_decay)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'alpha', alpha)
return cs
def get_config_space(
T_max=(300,1000),
patience=(2,5),
T_mult=(1.0,2.0),
threshold=(0.001, 0.5)
):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'T_max', T_max)
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'patience', patience)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'T_mult', T_mult)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'threshold', threshold)
return cs
def get_hyperparameter_search_space(dataset_info=None,
kernel=('poly', 'rbf', 'sigmoid',
'cosine'),
n_components=(10, 2000),
gamma=((3.0517578125e-05, 8), True),
degree=(2, 5),
coef0=(-1, 1)):
cs = ConfigSpace.ConfigurationSpace()
kernel_hp = add_hyperparameter(cs, CSH.CategoricalHyperparameter,
'kernel', kernel)
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter,
"n_components", n_components)
if "poly" in kernel:
degree_hp = add_hyperparameter(cs,
CSH.UniformIntegerHyperparameter,
'degree', degree)
cs.add_condition(CSC.EqualsCondition(degree_hp, kernel_hp, "poly"))
if set(["poly", "sigmoid"]) & set(kernel):
coef0_hp = add_hyperparameter(cs, CSH.UniformFloatHyperparameter,
"coef0", coef0)
cs.add_condition(
CSC.InCondition(coef0_hp, kernel_hp,
list(set(["poly", "sigmoid"]) & set(kernel))))
if set(["poly", "rbf", "sigmoid"]) & set(kernel):
gamma_hp = add_hyperparameter(cs, CSH.UniformFloatHyperparameter,
"gamma", gamma)
cs.add_condition(
CSC.InCondition(
gamma_hp, kernel_hp,
list(set(["poly", "rbf", "sigmoid"]) & set(kernel))))
return cs
def get_config_space(
max_factor=(1.0, 2),
min_factor=(0.001, 1.0),
cycle_length=(3, 10)
):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'max_factor', max_factor)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'min_factor', min_factor)
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'cycle_length', cycle_length)
return cs
def get_hyperparameter_search_space(
dataset_info=None,
degree=(2, 3),
interaction_only=(True, False),
include_bias=(True, False)
):
cs = ConfigSpace.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, "degree", degree)
add_hyperparameter(cs, CSH.CategoricalHyperparameter, "interaction_only", [False, True])
add_hyperparameter(cs, CSH.CategoricalHyperparameter, "include_bias", [True, False])
return cs
def get_hyperparameter_search_space(initialize_bias=("Yes", "No", "Zero")):
cs = ConfigSpace.ConfigurationSpace()
add_hyperparameter(cs, ConfigSpace.CategoricalHyperparameter,
"initialize_bias", initialize_bias)
return cs
def get_hyperparameter_search_space(alpha=(0, 1)):
cs = ConfigSpace.ConfigurationSpace()
add_hyperparameter(
cs, ConfigSpace.hyperparameters.UniformFloatHyperparameter,
"alpha", alpha)
return cs
def get_config_space(num_groups=((1, 9), False),
blocks_per_group=((1, 4), False),
num_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
max_shake_drop_probability=(0, 1),
dropout=(0, 1.0),
use_shake_drop=(True, False),
use_shake_shake=(True, False),
use_dropout=(True, False),
**kwargs):
cs = ConfigSpace.ConfigurationSpace()
num_groups_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "num_groups", num_groups)
cs.add_hyperparameter(num_groups_hp)
blocks_per_group_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "blocks_per_group",
blocks_per_group)
cs.add_hyperparameter(blocks_per_group_hp)
add_hyperparameter(cs, ConfigSpace.CategoricalHyperparameter,
"activation", activation)
use_dropout_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, "use_dropout", use_dropout)
cs.add_hyperparameter(use_dropout_hp)
add_hyperparameter(cs, ConfigSpace.CategoricalHyperparameter,
"use_shake_shake", use_shake_shake)
use_shake_drop_hp = add_hyperparameter(
cs, ConfigSpace.CategoricalHyperparameter, "use_shake_drop",
use_shake_drop)
if True in use_shake_drop:
shake_drop_prob_hp = add_hyperparameter(
cs, ConfigSpace.UniformFloatHyperparameter,
"max_shake_drop_probability", max_shake_drop_probability)
cs.add_condition(
ConfigSpace.EqualsCondition(shake_drop_prob_hp,
use_shake_drop_hp, True))
# it is the upper bound of the nr of groups, since the configuration will actually be sampled.
for i in range(0, num_groups[0][1] + 1):
n_units_hp = add_hyperparameter(
cs, ConfigSpace.UniformIntegerHyperparameter,
"num_units_%d" % i, kwargs.pop("num_units_%d" % i, num_units))
if i > 1:
cs.add_condition(
ConfigSpace.GreaterThanCondition(n_units_hp, num_groups_hp,
i - 1))
if True in use_dropout:
dropout_hp = add_hyperparameter(
cs, ConfigSpace.UniformFloatHyperparameter,
"dropout_%d" % i, kwargs.pop("dropout_%d" % i, dropout))
dropout_condition_1 = ConfigSpace.EqualsCondition(
dropout_hp, use_dropout_hp, True)
if i > 1:
dropout_condition_2 = ConfigSpace.GreaterThanCondition(
dropout_hp, num_groups_hp, i - 1)
cs.add_condition(
ConfigSpace.AndConjunction(dropout_condition_1,
dropout_condition_2))
else:
cs.add_condition(dropout_condition_1)
assert len(
kwargs
) == 0, "Invalid hyperparameter updates for resnet: %s" % str(kwargs)
return cs
def get_hyperparameter_search_space(dataset_info=None,
target_dim=(10, 256)):
cs = ConfigSpace.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, "target_dim",
target_dim)
return cs
def get_config_space(num_groups=(1, 9),
blocks_per_group=(1, 4),
max_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
max_shake_drop_probability=(0, 1),
max_dropout=(0, 0.8),
resnet_shape=('funnel', 'long_funnel', 'diamond',
'hexagon', 'brick', 'triangle',
'stairs'),
dropout_shape=('funnel', 'long_funnel', 'diamond',
'hexagon', 'brick', 'triangle',
'stairs'),
use_dropout=(True, False),
use_shake_shake=(True, False),
use_shake_drop=(True, False)):
cs = CS.ConfigurationSpace()
num_groups_hp = get_hyperparameter(CS.UniformIntegerHyperparameter,
"num_groups", num_groups)
cs.add_hyperparameter(num_groups_hp)
blocks_per_group_hp = get_hyperparameter(
CS.UniformIntegerHyperparameter, "blocks_per_group",
blocks_per_group)
cs.add_hyperparameter(blocks_per_group_hp)
add_hyperparameter(cs, CS.CategoricalHyperparameter, "activation",
activation)
use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_dropout", use_dropout)
add_hyperparameter(cs, CS.CategoricalHyperparameter, "use_shake_shake",
use_shake_shake)
shake_drop_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_shake_drop", use_shake_drop)
if True in use_shake_drop:
shake_drop_prob_hp = add_hyperparameter(
cs, CS.UniformFloatHyperparameter,
"max_shake_drop_probability", max_shake_drop_probability)
cs.add_condition(
CS.EqualsCondition(shake_drop_prob_hp, shake_drop_hp, True))
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'resnet_shape',
resnet_shape)
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, "max_units",
max_units)
if True in use_dropout:
dropout_shape_hp = add_hyperparameter(
cs, CSH.CategoricalHyperparameter, 'dropout_shape',
dropout_shape)
max_dropout_hp = add_hyperparameter(cs,
CSH.UniformFloatHyperparameter,
"max_dropout", max_dropout)
cs.add_condition(
CS.EqualsCondition(dropout_shape_hp, use_dropout_hp, True))
cs.add_condition(
CS.EqualsCondition(max_dropout_hp, use_dropout_hp, True))
return cs
def get_config_space( growth_rate_range=(5, 128), nr_blocks=(1, 5), kernel_range=(2, 7),
layer_range=(5, 50), activations=all_activations.keys(),
conv_init=('random', 'kaiming_normal', 'constant_0', 'constant_1', 'constant_05'),
batchnorm_weight_init=('random', 'constant_0', 'constant_1', 'constant_05'),
batchnorm_bias_init=('random', 'constant_0', 'constant_1', 'constant_05'),
linear_bias_init=('random', 'constant_0', 'constant_1', 'constant_05'), **kwargs):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter
cs = CS.ConfigurationSpace()
growth_rate_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'growth_rate', growth_rate_range)
first_conv_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'first_conv_kernel', kernel_range)
first_pool_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'first_pool_kernel', kernel_range)
conv_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'conv_init', conv_init)
batchnorm_weight_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'batchnorm_weight_init', batchnorm_weight_init)
batchnorm_bias_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'batchnorm_bias_init', batchnorm_bias_init)
linear_bias_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'linear_bias_init', linear_bias_init)
first_activation_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'first_activation', sorted(set(activations).intersection(all_activations)))
blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'blocks', nr_blocks)
cs.add_hyperparameter(growth_rate_hp)
cs.add_hyperparameter(first_conv_kernel_hp)
cs.add_hyperparameter(first_pool_kernel_hp)
cs.add_hyperparameter(conv_init_hp)
cs.add_hyperparameter(batchnorm_weight_init_hp)
cs.add_hyperparameter(batchnorm_bias_init_hp)
cs.add_hyperparameter(linear_bias_init_hp)
cs.add_hyperparameter(first_activation_hp)
cs.add_hyperparameter(blocks_hp)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'channel_reduction', [0.1, 0.9])
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'last_image_size', [0, 1])
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'bottleneck', [True, False])
use_dropout = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'use_dropout', [True, False])
if type(nr_blocks[0]) == int:
min_blocks = nr_blocks[0]
max_blocks = nr_blocks[1]
else:
min_blocks = nr_blocks[0][0]
max_blocks = nr_blocks[0][1]
for i in range(1, max_blocks+1):
layer_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'layer_in_block_%d' % i, layer_range)
pool_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'pool_kernel_%d' % i, kernel_range)
activation_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'activation_%d' % i, sorted(set(activations).intersection(all_activations)))
cs.add_hyperparameter(layer_hp)
cs.add_hyperparameter(pool_kernel_hp)
cs.add_hyperparameter(activation_hp)
dropout = add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'dropout_%d' % i, [0.0, 1.0])
conv_kernel = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'conv_kernel_%d' % i, [3, 5, 7])
if i > min_blocks:
cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i-1))
cs.add_condition(CS.GreaterThanCondition(conv_kernel, blocks_hp, i-1))
cs.add_condition(CS.GreaterThanCondition(pool_kernel_hp, blocks_hp, i-1))
cs.add_condition(CS.GreaterThanCondition(activation_hp, blocks_hp, i-1))
cs.add_condition(CS.AndConjunction(CS.EqualsCondition(dropout, use_dropout, True), CS.GreaterThanCondition(dropout, blocks_hp, i-1)))
else:
cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))
return cs
def get_config_space(
gamma=(0.8, 0.9999)
):
cs = CS.ConfigurationSpace()
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'gamma', gamma)
return cs
