def get_searchspace_binary(): spaces = { # See docs: https://docs.fast.ai/tabular.models.html 'layers': Categorical(None, [200, 100], [200], [500], [1000], [500, 200], [50, 25], [1000, 500], [200, 100, 50], [500, 200, 100], [1000, 500, 200]), 'emb_drop': Real(0.0, 0.5, default=0.1), 'ps': Real(0.0, 0.5, default=0.1), 'bs': Categorical(256, 64, 128, 512, 1024, 2048, 4096), 'lr': Real(5e-5, 1e-1, default=1e-2, log=True), 'epochs': Int(lower=5, upper=30, default=30), 'early.stopping.min_delta': 0.0001, 'early.stopping.patience': 20, 'smoothing': Real(0.0, 0.3, default=0.0, log=True), } return spaces
def get_searchspace_quantile(framework): if framework != 'pytorch': raise ValueError("Only pytorch tabular neural network is currently supported for quantile regression.") params = { 'activation': Categorical('relu', 'elu', 'tanh'), 'weight_decay': Real(1e-12, 1.0, default=1e-6, log=True), 'gamma': Real(0.1, 10.0, default=5.0), 'alpha': Categorical(0.001, 0.01, 0.1, 1.0), } return params
def get_searchspace_regression(framework): params = { 'weight_decay': Real(1e-12, 1.0, default=1e-6, log=True), } mxnet_params = { 'activation': Categorical('relu', 'softrelu', 'tanh'), } pytorch_params = { 'activation': Categorical('relu', 'elu', 'tanh'), } return merge_framework_params(framework=framework, shared_params=params, mxnet_params=mxnet_params, pytorch_params=pytorch_params)
def get_searchspace_quantile(): params = { 'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True), 'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True), 'dropout_prob': Real(0.0, 0.5, default=0.1), 'gamma': Real(0.1, 10.0, default=5.0), 'num_layers': Categorical(2, 3, 4), 'hidden_size': Categorical(128, 256, 512), 'embedding_size_factor': Real(0.5, 1.5, default=1.0), 'alpha': Categorical(0.001, 0.01, 0.1, 1.0), } return params
def get_default_searchspace(): params = { 'lr': Real(5e-5, 5e-3, default=1e-3, log=True), 'weight_decay': Real(1e-6, 5e-2, default=1e-6, log=True), 'p_dropout': Categorical(0.1, 0, 0.5), 'n_heads': Categorical(8, 4), 'hidden_dim': Categorical(128, 32, 64, 256), 'n_layers': Categorical(2, 1, 3, 4, 5), 'feature_dim': Int(8, 128, default=64), 'num_output_layers': Categorical(1, 2), } return params.copy()
def get_searchspace_regression(): params = { 'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True), 'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True), 'dropout_prob': Real(0.0, 0.5, default=0.1), # 'layers': Categorical(None, [200, 100], [256], [2056], [1024, 512, 128], [1024, 1024, 1024]), 'layers': Categorical(None, [200, 100], [256], [100, 50], [200, 100, 50], [50, 25], [300, 150]), 'embedding_size_factor': Real(0.5, 1.5, default=1.0), 'network_type': Categorical('widedeep', 'feedforward'), 'use_batchnorm': Categorical(True, False), 'activation': Categorical('relu', 'softrelu', 'tanh'), # 'batch_size': Categorical(512, 1024, 2056, 128), # this is used in preprocessing so cannot search atm } return params
def get_searchspace_multiclass(num_classes): # Search space we use by default (only specify non-fixed hyperparameters here): # TODO: move to separate file params = { 'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True), 'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True), 'dropout_prob': Real(0.0, 0.5, default=0.1), # 'layers': Categorical(None, [200, 100], [256], [2056], [1024, 512, 128], [1024, 1024, 1024]), 'layers': Categorical(None, [200, 100], [256], [100, 50], [200, 100, 50], [50, 25], [300, 150]), 'embedding_size_factor': Real(0.5, 1.5, default=1.0), 'network_type': Categorical('widedeep', 'feedforward'), 'use_batchnorm': Categorical(True, False), 'activation': Categorical('relu', 'softrelu'), # 'batch_size': Categorical(512, 1024, 2056, 128), # this is used in preprocessing so cannot search atm } return params
def get_default_searchspace(): params = { 'lr': Real(5e-5, 5e-3, default=1e-3, log=True), 'weight_decay': Real(1e-6, 5e-2, default=1e-6, log=True), 'p_dropout': Categorical(0.1, 0, 0.2, 0.3, 0.4, 0.5), 'n_heads': Categorical(8, 2, 4), 'hidden_dim': Categorical(128, 32, 64, 256), 'n_layers': Categorical(1, 2, 3, 4, 5), 'feature_dim': Int(8, 128, default=64), 'tab_readout': Categorical('none', 'readout_emb', 'mean', 'concat_pool', 'concat_pool_all', 'concat_pool_add', 'all_feat_embs', 'mean_feat_embs'), 'num_output_layers': Categorical(2, 1, 3), } return params.copy()
def sanitize_batch_size(batch_size, min_value=1, max_value=np.inf): if isinstance(batch_size, Categorical): valid_bs = [] bs_values = batch_size.data for bs_value in bs_values: if isinstance(bs_value, int) and min_value < bs_value < max_value: valid_bs.append(bs_value) if valid_bs != bs_values: warnings.warn( f'Pruning batch size from {batch_size} to {valid_bs} due to memory limit.' ) if len(valid_bs) == 1: new_bs = valid_bs[0] else: new_bs = Categorical(*valid_bs) elif isinstance(batch_size, Int): lower = batch_size.lower upper = batch_size.upper if not isinstance(lower, int) or not isinstance(upper, int): raise TypeError( f'Invalid lower {lower} or upper {upper} bound for Int space') lower = max(lower, min_value) upper = min(upper, max_value) new_bs = Int(lower=lower, upper=upper) if lower != batch_size.lower or upper != batch_size.higher: warnings.warn( f'Adjusting batch size range from {batch_size} to {new_bs} due to memory limit.' ) elif isinstance(batch_size, int): new_bs = max(min(batch_size, max_value), min_value) if new_bs != batch_size: warnings.warn( f'Adjusting batch size from {batch_size} to {new_bs} due to memory limit.' ) else: raise TypeError( f'Expecting batch size to be (Categorical/Int/int), given {type(batch_size)}.' ) return new_bs
def _call(*args, **kwargs): gargs, gkwargs = set_presets(preset_name, *args, **kwargs) return f(*gargs, **gkwargs) return _call return _unpack_inner # Dictionary of preset fit() parameter configurations for ImagePredictor. preset_image_predictor = dict( # Best predictive accuracy with little consideration to inference time or model size. Achieve even better results by specifying a large time_limit value. # Recommended for applications that benefit from the best possible model accuracy. best_quality={ 'hyperparameters': { 'model': Categorical('coat_lite_small', 'twins_pcpvt_base', 'swin_base_patch4_window7_224', 'resnet101d') \ if timm != None else Categorical('resnet50_v1b', 'resnet101_v1d', 'resnest200'), 'lr': Real(1e-5, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64), 'epochs': 200, 'early_stop_patience': 50 }, 'hyperparameter_tune_kwargs': { 'num_trials': 1024, 'searcher': 'random', }, 'time_limit': 12*3600, }, # Good predictive accuracy with fast inference. # Recommended for applications that require reasonable inference speed and/or model size.
def get_default_searchspace(problem_type, framework, num_classes=None): params = { 'learning_rate': Real(1e-4, 3e-2, default=3e-4, log=True), 'weight_decay': Real(1e-12, 0.1, default=1e-6, log=True), 'dropout_prob': Categorical(0.1, 0.0, 0.5, 0.2, 0.3, 0.4), 'embedding_size_factor': Categorical(1.0, 0.5, 1.5, 0.7, 0.6, 0.8, 0.9, 1.1, 1.2, 1.3, 1.4), 'proc.embed_min_categories': Categorical(4, 3, 10, 100, 1000), 'proc.impute_strategy': Categorical('median', 'mean', 'most_frequent'), 'proc.max_category_levels': Categorical(100, 10, 20, 200, 300, 400, 500, 1000, 10000), 'proc.skew_threshold': Categorical(0.99, 0.2, 0.3, 0.5, 0.8, 0.9, 0.999, 1.0, 10.0, 100.0), } mxnet_params = { 'use_batchnorm': Categorical(True, False), 'layers': Categorical(None, [200, 100], [256], [100, 50], [200, 100, 50], [1024], [32], [300, 150]), 'network_type': Categorical('widedeep', 'feedforward'), 'activation': Categorical('relu', 'softrelu'), 'batch_size': Categorical(512, 1024, 2056, 128), } pytorch_params = { 'use_batchnorm': Categorical(False, True), 'num_layers': Categorical(2, 3, 4), 'hidden_size': Categorical(128, 256, 512), 'activation': Categorical('relu', 'elu'), } params = merge_framework_params(framework=framework, shared_params=params, mxnet_params=mxnet_params, pytorch_params=pytorch_params) if problem_type == QUANTILE: problem_params = get_searchspace_quantile(framework) elif problem_type == BINARY: problem_params = get_searchspace_binary(framework) elif problem_type == MULTICLASS: problem_params = get_searchspace_multiclass(framework, num_classes=num_classes) elif problem_type == REGRESSION: problem_params = get_searchspace_regression(framework) params.update(problem_params) return params.copy()
def _call(*args, **kwargs): gargs, gkwargs = set_presets(preset_name, *args, **kwargs) return f(*gargs, **gkwargs) return _call return _unpack_inner # Dictionary of preset fit() parameter configurations for ImagePredictor. preset_image_predictor = dict( # Best predictive accuracy with little consideration to inference time or model size. Achieve even better results by specifying a large time_limit value. # Recommended for applications that benefit from the best possible model accuracy. best_quality={ 'hyperparameters': { 'model': Categorical('resnet50_v1b', 'resnet101_v1d', 'resnest200'), 'lr': Real(1e-5, 1e-2, log=True), 'batch_size': Categorical(8, 16, 32, 64, 128), 'epochs': 200, 'early_stop_patience': 50 }, 'hyperparameter_tune_kwargs': { 'num_trials': 1024, 'search_strategy': 'bayesopt', }, 'time_limit': 12 * 3600, }, # Good predictive accuracy with fast inference. # Recommended for applications that require reasonable inference speed and/or model size. good_quality_fast_inference={
def get_searchspace_quantile(): spaces = get_searchspace_regression() # residual threshold parameter in HuberPinballLoss spaces.update({'alpha': Categorical(0.001, 0.01, 0.1, 1.0)}) return spaces
Dict(kernel=3, num_repeat=1, output_filters=16, expand_ratio=1, stride=1, se_ratio=0.25, input_filters=32), Dict(kernel=3, num_repeat=1, output_filters=16, expand_ratio=1, stride=1, se_ratio=0.25, input_filters=16, with_zero=True), Dict(kernel=Categorical(3, 5, 7), num_repeat=1, output_filters=24, expand_ratio=Categorical(3, 6), stride=2, se_ratio=0.25, input_filters=16), Dict(kernel=Categorical(3, 5, 7), num_repeat=3, output_filters=24, expand_ratio=Categorical(3, 6), stride=1, se_ratio=0.25, input_filters=24, with_zero=True), Dict(kernel=Categorical(3, 5, 7),
def get_searchspace_regression(): params = { 'weight_decay': Real(1e-12, 1.0, default=1e-6, log=True), 'activation': Categorical('relu', 'softrelu', 'tanh'), } return params