def get_config_space(growth_rate_range=(12, 40), nr_blocks=(3, 4), layer_range=([1, 12], [6, 24], [12, 64], [12, 64]), num_init_features=(32, 128), **kwargs):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter
cs = CS.ConfigurationSpace()
growth_rate_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'growth_rate', growth_rate_range)
cs.add_hyperparameter(growth_rate_hp)
# add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'bn_size', [2, 4])
# add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'num_init_features', num_init_features, log=True)
# add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'bottleneck', [True, False])
blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'blocks', nr_blocks)
cs.add_hyperparameter(blocks_hp)
use_dropout = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'use_dropout', [True, False])
dropout = add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'dropout', [0.0, 1.0])
cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))
if type(nr_blocks[0]) == int:
min_blocks = nr_blocks[0]
max_blocks = nr_blocks[1]
else:
min_blocks = nr_blocks[0][0]
max_blocks = nr_blocks[0][1]
for i in range(1, max_blocks+1):
layer_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'layer_in_block_%d' % i, layer_range[i-1])
cs.add_hyperparameter(layer_hp)
if i > min_blocks:
cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i-1))
return cs
def get_config_space(num_layers=(1, 15),
max_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
mlp_shape=('funnel', 'long_funnel', 'diamond',
'hexagon', 'brick', 'triangle', 'stairs'),
max_dropout=(0, 1.0),
use_dropout=(True, False)):
cs = CS.ConfigurationSpace()
mlp_shape_hp = get_hyperparameter(CSH.CategoricalHyperparameter,
'mlp_shape', mlp_shape)
cs.add_hyperparameter(mlp_shape_hp)
num_layers_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
'num_layers', num_layers)
cs.add_hyperparameter(num_layers_hp)
max_units_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
"max_units", max_units)
cs.add_hyperparameter(max_units_hp)
use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_dropout", use_dropout)
max_dropout_hp = add_hyperparameter(cs, CSH.UniformFloatHyperparameter,
"max_dropout", max_dropout)
cs.add_condition(
CS.EqualsCondition(max_dropout_hp, use_dropout_hp, True))
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'activation',
activation)
return cs
def get_config_space(num_groups=(1, 9),
blocks_per_group=(1, 4),
max_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
max_shake_drop_probability=(0, 1),
max_dropout=(0, 0.8),
resnet_shape=('funnel', 'long_funnel', 'diamond',
'hexagon', 'brick', 'triangle',
'stairs'),
dropout_shape=('funnel', 'long_funnel', 'diamond',
'hexagon', 'brick', 'triangle',
'stairs'),
use_dropout=(True, False),
use_shake_shake=(True, False),
use_shake_drop=(True, False)):
cs = CS.ConfigurationSpace()
num_groups_hp = get_hyperparameter(CS.UniformIntegerHyperparameter,
"num_groups", num_groups)
cs.add_hyperparameter(num_groups_hp)
blocks_per_group_hp = get_hyperparameter(
CS.UniformIntegerHyperparameter, "blocks_per_group",
blocks_per_group)
cs.add_hyperparameter(blocks_per_group_hp)
add_hyperparameter(cs, CS.CategoricalHyperparameter, "activation",
activation)
use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_dropout", use_dropout)
add_hyperparameter(cs, CS.CategoricalHyperparameter, "use_shake_shake",
use_shake_shake)
shake_drop_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_shake_drop", use_shake_drop)
if True in use_shake_drop:
shake_drop_prob_hp = add_hyperparameter(
cs, CS.UniformFloatHyperparameter,
"max_shake_drop_probability", max_shake_drop_probability)
cs.add_condition(
CS.EqualsCondition(shake_drop_prob_hp, shake_drop_hp, True))
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'resnet_shape',
resnet_shape)
add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, "max_units",
max_units)
if True in use_dropout:
dropout_shape_hp = add_hyperparameter(
cs, CSH.CategoricalHyperparameter, 'dropout_shape',
dropout_shape)
max_dropout_hp = add_hyperparameter(cs,
CSH.UniformFloatHyperparameter,
"max_dropout", max_dropout)
cs.add_condition(
CS.EqualsCondition(dropout_shape_hp, use_dropout_hp, True))
cs.add_condition(
CS.EqualsCondition(max_dropout_hp, use_dropout_hp, True))
return cs
def get_hyperparameter_search_space(
dataset_info=None,
n_components=((50, 10000), True),
gamma=((3.0517578125e-05, 8), True),
):
n_components_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
"n_components", n_components)
gamma_hp = get_hyperparameter(CSH.UniformFloatHyperparameter, "gamma",
gamma)
cs = ConfigSpace.ConfigurationSpace()
cs.add_hyperparameters([gamma_hp, n_components_hp])
return cs
def get_config_space(num_layers=((1, 15), False),
num_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
dropout=(0.0, 0.8),
use_dropout=(True, False),
**kwargs):
cs = CS.ConfigurationSpace()
num_layers_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
'num_layers', num_layers)
cs.add_hyperparameter(num_layers_hp)
use_dropout_hp = add_hyperparameter(cs, CS.CategoricalHyperparameter,
"use_dropout", use_dropout)
for i in range(1, num_layers[0][1] + 1):
n_units_hp = get_hyperparameter(
CSH.UniformIntegerHyperparameter, "num_units_%d" % i,
kwargs.pop("num_units_%d" % i, num_units))
cs.add_hyperparameter(n_units_hp)
if i > num_layers[0][0]:
cs.add_condition(
CS.GreaterThanCondition(n_units_hp, num_layers_hp, i - 1))
if True in use_dropout:
dropout_hp = get_hyperparameter(
CSH.UniformFloatHyperparameter, "dropout_%d" % i,
kwargs.pop("dropout_%d" % i, dropout))
cs.add_hyperparameter(dropout_hp)
dropout_condition_1 = CS.EqualsCondition(
dropout_hp, use_dropout_hp, True)
if i > num_layers[0][0]:
dropout_condition_2 = CS.GreaterThanCondition(
dropout_hp, num_layers_hp, i - 1)
cs.add_condition(
CS.AndConjunction(dropout_condition_1,
dropout_condition_2))
else:
cs.add_condition(dropout_condition_1)
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'activation',
activation)
assert len(
kwargs
) == 0, "Invalid hyperparameter updates for mlpnet: %s" % str(kwargs)
return (cs)
def get_hyperparameter_search_space(
dataset_info=None,
n_components=(10, 2000),
algorithm=('parallel', 'deflation'),
whiten=(True, False),
fun=('logcosh', 'exp', 'cube'),
):
cs = ConfigSpace.ConfigurationSpace()
n_components_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
"n_components", n_components)
algorithm_hp = get_hyperparameter(CSH.CategoricalHyperparameter,
'algorithm', algorithm)
whiten_hp = get_hyperparameter(CSH.CategoricalHyperparameter, 'whiten',
whiten)
fun_hp = get_hyperparameter(CSH.CategoricalHyperparameter, 'fun', fun)
if True in whiten:
cs.add_hyperparameters(
[n_components_hp, algorithm_hp, whiten_hp, fun_hp])
cs.add_condition(
CSC.EqualsCondition(n_components_hp, whiten_hp, True))
return cs
def get_config_space(categorical_features=None,
min_unique_values_for_embedding=((3, 300), True),
dimension_reduction=(0, 1),
**kwargs):
# dimension of entity embedding layer is a hyperparameter
if categorical_features is None or not any(categorical_features):
return CS.ConfigurationSpace()
cs = CS.ConfigurationSpace()
min_hp = get_hyperparameter(CSH.UniformIntegerHyperparameter,
"min_unique_values_for_embedding",
min_unique_values_for_embedding)
cs.add_hyperparameter(min_hp)
for i in range(len([x for x in categorical_features if x])):
ee_dimensions_hp = get_hyperparameter(
CSH.UniformFloatHyperparameter,
"dimension_reduction_" + str(i),
kwargs.pop("dimension_reduction_" + str(i),
dimension_reduction))
cs.add_hyperparameter(ee_dimensions_hp)
assert len(
kwargs
) == 0, "Invalid hyperparameter updates for learned embedding: %s" % str(
kwargs)
return cs
def get_config_space(num_groups=((1, 9), False),
blocks_per_group=((1, 4), False),
num_units=((10, 1024), True),
activation=('sigmoid', 'tanh', 'relu'),
max_shake_drop_probability=(0, 1),
dropout=(0, 1.0),
use_shake_drop=(True, False),
use_shake_shake=(True, False),
use_dropout=(True, False),
**kwargs):
cs = ConfigSpace.ConfigurationSpace()
num_groups_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "num_groups", num_groups)
cs.add_hyperparameter(num_groups_hp)
blocks_per_group_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "blocks_per_group",
blocks_per_group)
cs.add_hyperparameter(blocks_per_group_hp)
add_hyperparameter(cs, ConfigSpace.CategoricalHyperparameter,
"activation", activation)
use_dropout_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, "use_dropout", use_dropout)
cs.add_hyperparameter(use_dropout_hp)
add_hyperparameter(cs, ConfigSpace.CategoricalHyperparameter,
"use_shake_shake", use_shake_shake)
use_shake_drop_hp = add_hyperparameter(
cs, ConfigSpace.CategoricalHyperparameter, "use_shake_drop",
use_shake_drop)
if True in use_shake_drop:
shake_drop_prob_hp = add_hyperparameter(
cs, ConfigSpace.UniformFloatHyperparameter,
"max_shake_drop_probability", max_shake_drop_probability)
cs.add_condition(
ConfigSpace.EqualsCondition(shake_drop_prob_hp,
use_shake_drop_hp, True))
# it is the upper bound of the nr of groups, since the configuration will actually be sampled.
for i in range(0, num_groups[0][1] + 1):
n_units_hp = add_hyperparameter(
cs, ConfigSpace.UniformIntegerHyperparameter,
"num_units_%d" % i, kwargs.pop("num_units_%d" % i, num_units))
if i > 1:
cs.add_condition(
ConfigSpace.GreaterThanCondition(n_units_hp, num_groups_hp,
i - 1))
if True in use_dropout:
dropout_hp = add_hyperparameter(
cs, ConfigSpace.UniformFloatHyperparameter,
"dropout_%d" % i, kwargs.pop("dropout_%d" % i, dropout))
dropout_condition_1 = ConfigSpace.EqualsCondition(
dropout_hp, use_dropout_hp, True)
if i > 1:
dropout_condition_2 = ConfigSpace.GreaterThanCondition(
dropout_hp, num_groups_hp, i - 1)
cs.add_condition(
ConfigSpace.AndConjunction(dropout_condition_1,
dropout_condition_2))
else:
cs.add_condition(dropout_condition_1)
assert len(
kwargs
) == 0, "Invalid hyperparameter updates for resnet: %s" % str(kwargs)
return cs
def get_hyperparameter_search_space(k_neighbors=(3, 7)):
k_neighbors = get_hyperparameter(CSH.UniformIntegerHyperparameter,
"k_neighbors", k_neighbors)
cs = ConfigSpace.ConfigurationSpace()
cs.add_hyperparameter(k_neighbors)
return cs
def get_config_space( growth_rate_range=(5, 128), nr_blocks=(1, 5), kernel_range=(2, 7),
layer_range=(5, 50), activations=all_activations.keys(),
conv_init=('random', 'kaiming_normal', 'constant_0', 'constant_1', 'constant_05'),
batchnorm_weight_init=('random', 'constant_0', 'constant_1', 'constant_05'),
batchnorm_bias_init=('random', 'constant_0', 'constant_1', 'constant_05'),
linear_bias_init=('random', 'constant_0', 'constant_1', 'constant_05'), **kwargs):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
from autoPyTorch.utils.config_space_hyperparameter import add_hyperparameter
cs = CS.ConfigurationSpace()
growth_rate_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'growth_rate', growth_rate_range)
first_conv_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'first_conv_kernel', kernel_range)
first_pool_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'first_pool_kernel', kernel_range)
conv_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'conv_init', conv_init)
batchnorm_weight_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'batchnorm_weight_init', batchnorm_weight_init)
batchnorm_bias_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'batchnorm_bias_init', batchnorm_bias_init)
linear_bias_init_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'linear_bias_init', linear_bias_init)
first_activation_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'first_activation', sorted(set(activations).intersection(all_activations)))
blocks_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'blocks', nr_blocks)
cs.add_hyperparameter(growth_rate_hp)
cs.add_hyperparameter(first_conv_kernel_hp)
cs.add_hyperparameter(first_pool_kernel_hp)
cs.add_hyperparameter(conv_init_hp)
cs.add_hyperparameter(batchnorm_weight_init_hp)
cs.add_hyperparameter(batchnorm_bias_init_hp)
cs.add_hyperparameter(linear_bias_init_hp)
cs.add_hyperparameter(first_activation_hp)
cs.add_hyperparameter(blocks_hp)
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'channel_reduction', [0.1, 0.9])
add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'last_image_size', [0, 1])
add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'bottleneck', [True, False])
use_dropout = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'use_dropout', [True, False])
if type(nr_blocks[0]) == int:
min_blocks = nr_blocks[0]
max_blocks = nr_blocks[1]
else:
min_blocks = nr_blocks[0][0]
max_blocks = nr_blocks[0][1]
for i in range(1, max_blocks+1):
layer_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'layer_in_block_%d' % i, layer_range)
pool_kernel_hp = get_hyperparameter(ConfigSpace.UniformIntegerHyperparameter, 'pool_kernel_%d' % i, kernel_range)
activation_hp = get_hyperparameter(ConfigSpace.CategoricalHyperparameter, 'activation_%d' % i, sorted(set(activations).intersection(all_activations)))
cs.add_hyperparameter(layer_hp)
cs.add_hyperparameter(pool_kernel_hp)
cs.add_hyperparameter(activation_hp)
dropout = add_hyperparameter(cs, CSH.UniformFloatHyperparameter, 'dropout_%d' % i, [0.0, 1.0])
conv_kernel = add_hyperparameter(cs, CSH.CategoricalHyperparameter, 'conv_kernel_%d' % i, [3, 5, 7])
if i > min_blocks:
cs.add_condition(CS.GreaterThanCondition(layer_hp, blocks_hp, i-1))
cs.add_condition(CS.GreaterThanCondition(conv_kernel, blocks_hp, i-1))
cs.add_condition(CS.GreaterThanCondition(pool_kernel_hp, blocks_hp, i-1))
cs.add_condition(CS.GreaterThanCondition(activation_hp, blocks_hp, i-1))
cs.add_condition(CS.AndConjunction(CS.EqualsCondition(dropout, use_dropout, True), CS.GreaterThanCondition(dropout, blocks_hp, i-1)))
else:
cs.add_condition(CS.EqualsCondition(dropout, use_dropout, True))
return cs
def get_config_space(
nr_main_blocks=[3, 7],
initial_filters=([8, 32], True),
nr_sub_blocks=([1, 4], False),
op_types=["inverted_residual", "dwise_sep_conv"],
kernel_sizes=[3, 5],
strides=[1, 2],
output_filters=[
[12, 16, 20],
[18, 24,
30], [24, 32,
40],
[48, 64, 80], [72, 96,
120],
[120, 160,
200], [240, 320, 400]
], # the idea is to search for e.g. 0.75, 1, 1.25* output_filters(mainblock number)
skip_connection=[True, False],
se_ratios=[0, 0.25],
**kwargs):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
cs = CS.ConfigurationSpace()
main_blocks_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "nr_main_blocks",
nr_main_blocks)
initial_filters_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "initial_filters",
initial_filters)
cs.add_hyperparameter(main_blocks_hp)
cs.add_hyperparameter(initial_filters_hp)
if type(nr_main_blocks[0]) == int:
min_blocks = nr_main_blocks[0]
max_blocks = nr_main_blocks[1]
else:
min_blocks = nr_main_blocks[0][0]
max_blocks = nr_main_blocks[0][1]
for i in range(1, max_blocks + 1):
sub_blocks_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter,
'nr_sub_blocks_%d' % i, nr_sub_blocks)
op_type_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, 'op_type_%d' % i,
op_types)
kernel_size_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, 'kernel_size_%d' % i,
kernel_sizes)
stride_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, 'stride_%d' % i,
strides)
out_filters_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, 'out_filters_%d' % i,
output_filters[i -
1]) # take output_filters list i-1 as options
se_ratio_hp = get_hyperparameter(
ConfigSpace.CategoricalHyperparameter, 'se_ratio_%d' % i,
se_ratios)
cs.add_hyperparameter(sub_blocks_hp)
cs.add_hyperparameter(op_type_hp)
cs.add_hyperparameter(kernel_size_hp)
cs.add_hyperparameter(stride_hp)
cs.add_hyperparameter(out_filters_hp)
cs.add_hyperparameter(se_ratio_hp)
skip_con = cs.add_hyperparameter(
CSH.CategoricalHyperparameter('skip_con_%d' % i,
[True, False]))
if i > min_blocks:
cs.add_condition(
CS.GreaterThanCondition(sub_blocks_hp, main_blocks_hp,
i - 1))
cs.add_condition(
CS.GreaterThanCondition(op_type_hp, main_blocks_hp, i - 1))
cs.add_condition(
CS.GreaterThanCondition(kernel_size_hp, main_blocks_hp,
i - 1))
cs.add_condition(
CS.GreaterThanCondition(stride_hp, main_blocks_hp, i - 1))
cs.add_condition(
CS.GreaterThanCondition(out_filters_hp, main_blocks_hp,
i - 1))
cs.add_condition(
CS.GreaterThanCondition(skip_con, main_blocks_hp, i - 1))
cs.add_condition(
CS.GreaterThanCondition(se_ratio_hp, main_blocks_hp,
i - 1))
return cs
def get_config_space(nr_main_blocks=[1, 8],
nr_residual_blocks=([1, 16], True),
initial_filters=([8, 32], True),
widen_factor=([0.5, 4], True),
res_branches=([1, 5], False),
filters_size=[3, 3],
**kwargs):
import ConfigSpace as CS
import ConfigSpace.hyperparameters as CSH
cs = CS.ConfigurationSpace()
nr_main_blocks_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "nr_main_blocks",
nr_main_blocks)
cs.add_hyperparameter(nr_main_blocks_hp)
initial_filters_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter, "initial_filters",
initial_filters)
cs.add_hyperparameter(initial_filters_hp)
# add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'nr_convs', nr_convs, log=True)
death_rate_hp = get_hyperparameter(
ConfigSpace.UniformFloatHyperparameter, "death_rate",
([0, 1], False))
cs.add_hyperparameter(death_rate_hp)
if type(nr_main_blocks[0]) is int:
main_blocks_min = nr_main_blocks[0]
main_blocks_max = nr_main_blocks[1]
else:
main_blocks_min = nr_main_blocks[0][0]
main_blocks_max = nr_main_blocks[0][1]
for i in range(1, main_blocks_max + 1):
blocks_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter,
'nr_residual_blocks_%d' % i, nr_residual_blocks)
blocks = cs.add_hyperparameter(blocks_hp)
widen_hp = get_hyperparameter(
ConfigSpace.UniformFloatHyperparameter, 'widen_factor_%d' % i,
widen_factor)
widen = cs.add_hyperparameter(widen_hp)
branches_hp = get_hyperparameter(
ConfigSpace.UniformIntegerHyperparameter,
'res_branches_%d' % i, res_branches)
branches = cs.add_hyperparameter(branches_hp)
# filters = add_hyperparameter(cs, CSH.UniformIntegerHyperparameter, 'filters_size_%d' % i, filters_size, log=False)
if i > main_blocks_min:
cs.add_condition(
CS.GreaterThanCondition(blocks_hp, nr_main_blocks_hp,
i - 1))
cs.add_condition(
CS.GreaterThanCondition(widen_hp, nr_main_blocks_hp,
i - 1))
cs.add_condition(
CS.GreaterThanCondition(branches_hp, nr_main_blocks_hp,
i - 1))
# cs.add_condition(CS.GreaterThanCondition(filters, main_blocks, i-1))
return cs