def get_searchspace_binary():
spaces = {
# See docs: https://docs.fast.ai/tabular.models.html
'layers':
Categorical(None, [200, 100], [200], [500], [1000], [500, 200],
[50, 25], [1000, 500], [200, 100, 50], [500, 200, 100],
[1000, 500, 200]),
'emb_drop':
Real(0.0, 0.5, default=0.1),
'ps':
Real(0.0, 0.5, default=0.1),
'bs':
Categorical(256, 64, 128, 512, 1024, 2048, 4096),
'lr':
Real(5e-5, 1e-1, default=1e-2, log=True),
'epochs':
Int(lower=5, upper=30, default=30),
'early.stopping.min_delta':
0.0001,
'early.stopping.patience':
20,
'smoothing':
Real(0.0, 0.3, default=0.0, log=True),
}
return spaces
def get_searchspace_quantile(framework):
if framework != 'pytorch':
raise ValueError("Only pytorch tabular neural network is currently supported for quantile regression.")
params = {
'activation': Categorical('relu', 'elu', 'tanh'),
'weight_decay': Real(1e-12, 1.0, default=1e-6, log=True),
'gamma': Real(0.1, 10.0, default=5.0),
'alpha': Categorical(0.001, 0.01, 0.1, 1.0),
}
return params
def get_searchspace_regression(framework):
params = {
'weight_decay': Real(1e-12, 1.0, default=1e-6, log=True),
}
mxnet_params = {
'activation': Categorical('relu', 'softrelu', 'tanh'),
}
pytorch_params = {
'activation': Categorical('relu', 'elu', 'tanh'),
}
return merge_framework_params(framework=framework, shared_params=params, mxnet_params=mxnet_params, pytorch_params=pytorch_params)
def get_searchspace_quantile():
params = {
'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True),
'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True),
'dropout_prob': Real(0.0, 0.5, default=0.1),
'gamma': Real(0.1, 10.0, default=5.0),
'num_layers': Categorical(2, 3, 4),
'hidden_size': Categorical(128, 256, 512),
'embedding_size_factor': Real(0.5, 1.5, default=1.0),
'alpha': Categorical(0.001, 0.01, 0.1, 1.0),
}
return params
def get_default_searchspace():
params = {
'lr': Real(5e-5, 5e-3, default=1e-3, log=True),
'weight_decay': Real(1e-6, 5e-2, default=1e-6, log=True),
'p_dropout': Categorical(0.1, 0, 0.5),
'n_heads': Categorical(8, 4),
'hidden_dim': Categorical(128, 32, 64, 256),
'n_layers': Categorical(2, 1, 3, 4, 5),
'feature_dim': Int(8, 128, default=64),
'num_output_layers': Categorical(1, 2),
}
return params.copy()
def get_searchspace_regression():
params = {
'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True),
'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True),
'dropout_prob': Real(0.0, 0.5, default=0.1),
# 'layers': Categorical(None, [200, 100], [256], [2056], [1024, 512, 128], [1024, 1024, 1024]),
'layers': Categorical(None, [200, 100], [256], [100, 50], [200, 100, 50], [50, 25], [300, 150]),
'embedding_size_factor': Real(0.5, 1.5, default=1.0),
'network_type': Categorical('widedeep', 'feedforward'),
'use_batchnorm': Categorical(True, False),
'activation': Categorical('relu', 'softrelu', 'tanh'),
# 'batch_size': Categorical(512, 1024, 2056, 128), # this is used in preprocessing so cannot search atm
}
return params
def get_searchspace_multiclass(num_classes):
# Search space we use by default (only specify non-fixed hyperparameters here): # TODO: move to separate file
params = {
'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True),
'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True),
'dropout_prob': Real(0.0, 0.5, default=0.1),
# 'layers': Categorical(None, [200, 100], [256], [2056], [1024, 512, 128], [1024, 1024, 1024]),
'layers': Categorical(None, [200, 100], [256], [100, 50], [200, 100, 50], [50, 25], [300, 150]),
'embedding_size_factor': Real(0.5, 1.5, default=1.0),
'network_type': Categorical('widedeep', 'feedforward'),
'use_batchnorm': Categorical(True, False),
'activation': Categorical('relu', 'softrelu'),
# 'batch_size': Categorical(512, 1024, 2056, 128), # this is used in preprocessing so cannot search atm
}
return params
def get_default_searchspace():
params = {
'lr':
Real(5e-5, 5e-3, default=1e-3, log=True),
'weight_decay':
Real(1e-6, 5e-2, default=1e-6, log=True),
'p_dropout':
Categorical(0.1, 0, 0.2, 0.3, 0.4, 0.5),
'n_heads':
Categorical(8, 2, 4),
'hidden_dim':
Categorical(128, 32, 64, 256),
'n_layers':
Categorical(1, 2, 3, 4, 5),
'feature_dim':
Int(8, 128, default=64),
'tab_readout':
Categorical('none', 'readout_emb', 'mean', 'concat_pool',
'concat_pool_all', 'concat_pool_add', 'all_feat_embs',
'mean_feat_embs'),
'num_output_layers':
Categorical(2, 1, 3),
}
return params.copy()
def sanitize_batch_size(batch_size, min_value=1, max_value=np.inf):
if isinstance(batch_size, Categorical):
valid_bs = []
bs_values = batch_size.data
for bs_value in bs_values:
if isinstance(bs_value, int) and min_value < bs_value < max_value:
valid_bs.append(bs_value)
if valid_bs != bs_values:
warnings.warn(
f'Pruning batch size from {batch_size} to {valid_bs} due to memory limit.'
)
if len(valid_bs) == 1:
new_bs = valid_bs[0]
else:
new_bs = Categorical(*valid_bs)
elif isinstance(batch_size, Int):
lower = batch_size.lower
upper = batch_size.upper
if not isinstance(lower, int) or not isinstance(upper, int):
raise TypeError(
f'Invalid lower {lower} or upper {upper} bound for Int space')
lower = max(lower, min_value)
upper = min(upper, max_value)
new_bs = Int(lower=lower, upper=upper)
if lower != batch_size.lower or upper != batch_size.higher:
warnings.warn(
f'Adjusting batch size range from {batch_size} to {new_bs} due to memory limit.'
)
elif isinstance(batch_size, int):
new_bs = max(min(batch_size, max_value), min_value)
if new_bs != batch_size:
warnings.warn(
f'Adjusting batch size from {batch_size} to {new_bs} due to memory limit.'
)
else:
raise TypeError(
f'Expecting batch size to be (Categorical/Int/int), given {type(batch_size)}.'
)
return new_bs
def _call(*args, **kwargs):
gargs, gkwargs = set_presets(preset_name, *args, **kwargs)
return f(*gargs, **gkwargs)
return _call
return _unpack_inner
# Dictionary of preset fit() parameter configurations for ImagePredictor.
preset_image_predictor = dict(
# Best predictive accuracy with little consideration to inference time or model size. Achieve even better results by specifying a large time_limit value.
# Recommended for applications that benefit from the best possible model accuracy.
best_quality={
'hyperparameters': {
'model': Categorical('coat_lite_small', 'twins_pcpvt_base', 'swin_base_patch4_window7_224', 'resnet101d') \
if timm != None else Categorical('resnet50_v1b', 'resnet101_v1d', 'resnest200'),
'lr': Real(1e-5, 1e-2, log=True),
'batch_size': Categorical(8, 16, 32, 64),
'epochs': 200,
'early_stop_patience': 50
},
'hyperparameter_tune_kwargs': {
'num_trials': 1024,
'searcher': 'random',
},
'time_limit': 12*3600,
},
# Good predictive accuracy with fast inference.
# Recommended for applications that require reasonable inference speed and/or model size.
def get_default_searchspace(problem_type, framework, num_classes=None):
params = {
'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True),
'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True),
'dropout_prob': Categorical(0.1, 0.0, 0.5, 0.2, 0.3, 0.4),
'embedding_size_factor': Categorical(1.0, 0.5, 1.5, 0.7, 0.6, 0.8, 0.9, 1.1, 1.2, 1.3, 1.4),
'proc.embed_min_categories': Categorical(4, 3, 10, 100, 1000),
'proc.impute_strategy': Categorical('median', 'mean', 'most_frequent'),
'proc.max_category_levels': Categorical(100, 10, 20, 200, 300, 400, 500, 1000, 10000),
'proc.skew_threshold': Categorical(0.99, 0.2, 0.3, 0.5, 0.8, 0.9, 0.999, 1.0, 10.0, 100.0),
}
mxnet_params = {
'use_batchnorm': Categorical(True, False),
'layers': Categorical(None, [200, 100], [256], [100, 50], [200, 100, 50], [1024], [32], [300, 150]),
'network_type': Categorical('widedeep', 'feedforward'),
'activation': Categorical('relu', 'softrelu'),
'batch_size': Categorical(512, 1024, 2056, 128),
}
pytorch_params = {
'use_batchnorm': Categorical(False, True),
'num_layers': Categorical(2, 3, 4),
'hidden_size': Categorical(128, 256, 512),
'activation': Categorical('relu', 'elu'),
}
params = merge_framework_params(framework=framework, shared_params=params, mxnet_params=mxnet_params, pytorch_params=pytorch_params)
if problem_type == QUANTILE:
problem_params = get_searchspace_quantile(framework)
elif problem_type == BINARY:
problem_params = get_searchspace_binary(framework)
elif problem_type == MULTICLASS:
problem_params = get_searchspace_multiclass(framework, num_classes=num_classes)
elif problem_type == REGRESSION:
problem_params = get_searchspace_regression(framework)
params.update(problem_params)
return params.copy()
def _call(*args, **kwargs):
gargs, gkwargs = set_presets(preset_name, *args, **kwargs)
return f(*gargs, **gkwargs)
return _call
return _unpack_inner
# Dictionary of preset fit() parameter configurations for ImagePredictor.
preset_image_predictor = dict(
# Best predictive accuracy with little consideration to inference time or model size. Achieve even better results by specifying a large time_limit value.
# Recommended for applications that benefit from the best possible model accuracy.
best_quality={
'hyperparameters': {
'model': Categorical('resnet50_v1b', 'resnet101_v1d',
'resnest200'),
'lr': Real(1e-5, 1e-2, log=True),
'batch_size': Categorical(8, 16, 32, 64, 128),
'epochs': 200,
'early_stop_patience': 50
},
'hyperparameter_tune_kwargs': {
'num_trials': 1024,
'search_strategy': 'bayesopt',
},
'time_limit': 12 * 3600,
},
# Good predictive accuracy with fast inference.
# Recommended for applications that require reasonable inference speed and/or model size.
good_quality_fast_inference={
def get_searchspace_quantile():
spaces = get_searchspace_regression()
# residual threshold parameter in HuberPinballLoss
spaces.update({'alpha': Categorical(0.001, 0.01, 0.1, 1.0)})
return spaces
Dict(kernel=3,
num_repeat=1,
output_filters=16,
expand_ratio=1,
stride=1,
se_ratio=0.25,
input_filters=32),
Dict(kernel=3,
num_repeat=1,
output_filters=16,
expand_ratio=1,
stride=1,
se_ratio=0.25,
input_filters=16,
with_zero=True),
Dict(kernel=Categorical(3, 5, 7),
num_repeat=1,
output_filters=24,
expand_ratio=Categorical(3, 6),
stride=2,
se_ratio=0.25,
input_filters=16),
Dict(kernel=Categorical(3, 5, 7),
num_repeat=3,
output_filters=24,
expand_ratio=Categorical(3, 6),
stride=1,
se_ratio=0.25,
input_filters=24,
with_zero=True),
Dict(kernel=Categorical(3, 5, 7),
def get_searchspace_regression():
params = {
'weight_decay': Real(1e-12, 1.0, default=1e-6, log=True),
'activation': Categorical('relu', 'softrelu', 'tanh'),
}
return params
