Exemple #1
0
 def __init__(self):
     self.search_space = {
         'learning_rate':
         hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
         'L1_flag':
         hp.choice('L1_flag', [True, False]),
         'hidden_size':
         scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(256),
                                  1)),
         'batch_size':
         scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),
                                  1)),
         'lmbda':
         hp.loguniform('lmbda', np.log(0.00001), np.log(0.001)),
         'optimizer':
         hp.choice('optimizer', ["adam", "sgd", 'rms']),
         'margin':
         hp.uniform('margin', 0.5, 8.0),
         'distance_measure':
         hp.choice('distance_measure',
                   ["kl_divergence", "expected_likelihood"]),
         'cmax':
         hp.loguniform('cmax', np.log(0.05), np.log(0.2)),
         'cmin':
         hp.loguniform('cmin', np.log(1), np.log(5)),
         'epochs':
         hp.choice('epochs', [10])  # always choose 10 training epochs.
     }
Exemple #2
0
class skLassoCVReg(_LassoCVReg, BaseWrapperReg, metaclass=MetaBaseWrapperRegWithABC):

    def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True, normalize=False, precompute='auto',
                 max_iter=1000, tol=1e-4, copy_X=True, cv=None, verbose=False, n_jobs=1, positive=False,
                 random_state=None, selection='cyclic'):
        n_alphas = int(n_alphas)
        _LassoCVReg.__init__(
            self, eps, n_alphas, alphas, fit_intercept, normalize, precompute, max_iter, tol, copy_X, cv,
            verbose, n_jobs, positive, random_state, selection)
        BaseWrapperReg.__init__(self)

    HyperOpt_space = {
        'n_alphas': hp.qloguniform('n_alphas', 2, 5, 1),
        'eps': hp.loguniform('eps', -5, 0),
        'max_iter': hp.qloguniform('max_iter', 4, 8, 1),
        'tol': hp.loguniform('tol', -8, 0),
    }

    tuning_grid = {
        'eps': 1e-3,
        'n_alphas': 100,
        'alphas': None,
        'fit_intercept': True,
        'normalize': False,
        'precompute': 'auto',
        'max_iter': 1000,
        'tol': 1e-4,
        'copy_X': True,
        'cv': None,
        'verbose': False,
        'n_jobs': 1,
        'positive': False,
        'random_state': None,
        'selection': 'cyclic',
    }
Exemple #3
0
    def get_default_parameter_space():
        """
        Return:
            dict of DistributionWrappers
        """

        return {
            'n_estimators':
            scope.int(hp.quniform('n_estimators', 5, 10, 5)),
            'learning_rate':
            hp.loguniform('learning_rate', -7, 0),
            'num_leaves':
            scope.int(hp.qloguniform('num_leaves', 1, 7, 1)),
            'feature_fraction':
            hp.uniform('feature_fraction', 0.5, 1),
            'bagging_fraction':
            hp.uniform('bagging_fraction', 0.5, 1),
            'min_data_in_leaf':
            scope.int(hp.qloguniform('min_data_in_leaf', 0, 6, 1)),
            'min_sum_hessian_in_leaf':
            hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
            'lambda_l1':
            hp.loguniform('lambda_l1', -16, 2),
            'lambda_l2':
            hp.loguniform('lambda_l2', -16, 2),
        }
Exemple #4
0
    def _config_tuning_space(tuning_space_raw):
        if tuning_space_raw is None:
            return None

        hyper_obj = {}
        if "learning_rate" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"learning_rate": hp.loguniform('learning_rate', np.log(tuning_space_raw['learning_rate']['min']), np.log(tuning_space_raw['learning_rate']['max']))}}
        if "hidden_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"hidden_size": scope.int(hp.qloguniform('hidden_size', np.log(tuning_space_raw['hidden_size']['min']), np.log(tuning_space_raw['hidden_size']['max']), 1))}}
        if "ent_hidden_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"ent_hidden_size": scope.int(hp.qloguniform("ent_hidden_size", np.log(tuning_space_raw['ent_hidden_size']['min']), np.log(tuning_space_raw['ent_hidden_size']['max']), 1))}}
        if "rel_hidden_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"rel_hidden_size": scope.int(hp.qloguniform("rel_hidden_size", np.log(tuning_space_raw['rel_hidden_size']['min']), np.log(tuning_space_raw['rel_hidden_size']['max']), 1))}}
        if "batch_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"batch_size": scope.int(hp.qloguniform("batch_size", np.log(tuning_space_raw['batch_size']['min']), np.log(tuning_space_raw['batch_size']['max']), 1))}}
        if "margin" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"margin": hp.uniform("margin", tuning_space_raw["margin"]["min"], tuning_space_raw["margin"]["max"])}}
        if "lmbda" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"lmbda": hp.loguniform('lmbda', np.log(tuning_space_raw["lmbda"]["min"]), np.log(tuning_space_raw["lmbda"]["max"]))}}
        if "distance_measure" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"distance_measure": hp.choice('distance_measure', tuning_space_raw["distance_measure"])}}
        if "cmax" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"cmax": hp.loguniform('cmax', np.log(tuning_space_raw["cmax"]["min"]), np.log(tuning_space_raw["cmax"]["max"]))}}
        if "cmin" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"cmin": hp.loguniform('cmin', np.log(tuning_space_raw["cmin"]["min"]), np.log(tuning_space_raw["cmin"]["max"]))}}
        if "optimizer" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"optimizer": hp.choice("optimizer", tuning_space_raw["optimizer"])}}
        if "bilinear" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"bilinear": hp.choice('bilinear', tuning_space_raw["bilinear"])}}
        if "epochs" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"epochs": hp.choice("epochs", tuning_space_raw["epochs"])}}

        return hyper_obj
Exemple #5
0
class skLassoLarsCVReg(_LassoLarsCV, BaseWrapperReg, metaclass=MetaBaseWrapperRegWithABC):
    def __init__(self, fit_intercept=True, verbose=False, max_iter=500, normalize=True, precompute='auto', cv=None,
                 max_n_alphas=1000, n_jobs=1, eps=np.finfo(np.float).eps, copy_X=True, positive=False):
        _LassoLarsCV.__init__(
            self, fit_intercept, verbose, max_iter, normalize, precompute, cv, max_n_alphas, n_jobs, eps, copy_X,
            positive)
        BaseWrapperReg.__init__(self)

    HyperOpt_space = {
        'max_n_alphas': hp.qloguniform('max_n_alphas', 2, 5, 1),
        'eps': hp.loguniform('eps', -5, 0),
        'max_iter': hp.qloguniform('max_iter', 4, 8, 1),
    }

    tuning_grid = {
        'fit_intercept': True,
        'verbose': False,
        'max_iter': 500,
        'normalize': True,
        'precompute': 'auto',
        'cv': None,
        'max_n_alphas': 1000,
        'n_jobs': 1,
        'eps': np.finfo(np.float).eps,
        'copy_X': True,
        'positive': False,
    }
Exemple #6
0
def bernoulli_rbm_hp_space(n_components=None,
                           learning_rate=None,
                           batch_size=None,
                           n_iter=None,
                           verbose=False,
                           random_state=None):

    rval = dict(
        n_components=scope.int(
            hp.qloguniform(
                'n_components', low=np.log(0.51), high=np.log(999.5), q=1.0))
        if n_components is None else n_components,
        learning_rate=hp.lognormal(
            'learning_rate',
            np.log(0.01),
            np.log(10),
        ) if learning_rate is None else learning_rate,
        batch_size=scope.int(
            hp.qloguniform(
                '.batch_size',
                np.log(1),
                np.log(100),
                q=1,
            )) if batch_size is None else batch_size,
        n_iter=scope.int(
            hp.qloguniform(
                'n_iter',
                np.log(1),
                np.log(1000),  # -- max sweeps over the *whole* train set
                q=1,
            )) if n_iter is None else n_iter,
        verbose=verbose,
        random_state=_random_state('rstate', random_state),
    )
    return rval
Exemple #7
0
    def __init__(
        self,
        learning_task,
        n_estimators=100,
        max_hyperopt_evals=50,
        categorical_features=None,
        early_stopping_round=5,
        random_state=0,
        use_gpu=False,
        output_folder_path="./",
    ):
        Experiment.__init__(
            self,
            learning_task,
            "lgb",
            n_estimators,
            max_hyperopt_evals,
            categorical_features,
            early_stopping_round,
            random_state,
            output_folder_path,
            use_gpu,
        )

        # hard-coded params search space here
        self.space = {
            "learning_rate":
            hp.loguniform("learning_rate", -7, 0),
            "num_leaves":
            hp.qloguniform("num_leaves", 0, 7, 1),
            "feature_fraction":
            hp.uniform("feature_fraction", 0.5, 1),
            "bagging_fraction":
            hp.uniform("bagging_fraction", 0.5, 1),
            "min_data_in_leaf":
            hp.qloguniform("min_data_in_leaf", 0, 6, 1),
            "min_sum_hessian_in_leaf":
            hp.loguniform("min_sum_hessian_in_leaf", -16, 5),
            "lambda_l1":
            hp.choice("lambda_l1",
                      [0, hp.loguniform("lambda_l1_positive", -16, 2)]),
            "lambda_l2":
            hp.choice("lambda_l2",
                      [0, hp.loguniform("lambda_l2_positive", -16, 2)]),
        }
        # hard-coded default params here
        self.default_params = {
            "learning_rate": 0.1,
            "num_leaves": 127,
            "feature_fraction": 1.0,
            "bagging_fraction": 1.0,
            "min_data_in_leaf": 20,
            "min_sum_hessian_in_leaf": 1e-3,
            "lambda_l1": 0,
            "lambda_l2": 0,
        }
        self.default_params = self.preprocess_params(self.default_params)
        self.title = "LightGBM"
Exemple #8
0
def hyper_GAN_3_5(P_args):
    P_search = P_args.copy()
    P_search.set_keys(
        name = 'Hyper_GAN_3.5',
        dataset = 'SHL_ext',
        
        epochs = 100,
        runs = 5,
        
        batch_size = 512,
        FX_num = 150,
        
        FX_sel = 'all',
        cross_val = 'combined',
        
        sample_no = 512,
        undersampling = False,
        oversampling = False,
        
        C_aco_func = 'gumbel',
        
        D_optim = 'SGD', 

        G_optim = 'SGD',
        
        ) 
    
    param_space={
        'GD_ratio'        : hp.uniform('GD_ratio', 0, 0.9),
        
        'CLR'             : hp.loguniform('CLR', np.log(0.00001), np.log(0.1)),
        'CB1'             : hp.loguniform('CB1', np.log(0.001), np.log(0.99)),
        
        'C_ac_func'       : hp.choice('C_ac_func',['relu','leaky','leaky20','sig']),
        'C_hidden'        : scope.int(hp.qloguniform('C_hidden', np.log(16), np.log(4096), q=1)),
        'C_hidden_no'     : scope.int(hp.quniform('C_hidden_no', 1, 8, q=1)), 
        'C_optim'         : hp.choice('C_optim',['AdamW','SGD']),
        'C_tau'           : hp.loguniform('C_tau', np.log(0.01), np.log(10.)),
        
        'GLR'             : hp.loguniform('GLR', np.log(0.00001), np.log(0.1)),
        'GB1'             : hp.loguniform('GB1', np.log(0.001), np.log(0.99)),
        'DLR'             : hp.loguniform('DLR', np.log(0.00001), np.log(0.1)),
        'DB1'             : hp.loguniform('DB1', np.log(0.001), np.log(0.99)),

        'G_ac_func'       : hp.choice('G_ac_func',['relu','leaky','leaky20','sig']),
        'G_hidden'        : scope.int(hp.qloguniform('G_hidden', np.log(16), np.log(4096), q=1)),
        'G_hidden_no'     : scope.int(hp.quniform('G_hidden_no', 1, 8, q=1)), 
        
        'D_ac_func'       : hp.choice('D_ac_func',['relu','leaky','leaky20','sig']),
        'D_hidden'        : scope.int(hp.qloguniform('D_hidden', np.log(16), np.log(4096), q=1)),
        'D_hidden_no'     : scope.int(hp.quniform('D_hidden_no', 1, 8, q=1)), 
    }   
    
    hyperopt_GAN(P_search,param_space,eval_step=5,max_evals=None,P_val=P_search.copy().set_keys(
        sample_no = None,
        undersampling = False,
        oversampling = False,))
Exemple #9
0
 def __init__(self):
     self.search_space = {
       'learning_rate': hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
       'hidden_size': scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(256),1)),
       'batch_size': scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),1)),
       'lmbda': hp.loguniform('lmbda', np.log(0.00001), np.log(0.001)),
       'optimizer': hp.choice('optimizer', ["adam", "sgd", 'rms']),
       'epochs': hp.choice('epochs', [10]) # always choose 10 training epochs.
     }
def define_search_space(objective, starting_params):
    if objective == "rmse":
        prediction = starting_params["collaborative_params"][
            "prediction_network_params"]
        rmse_space = {
            'lr':
            hp.qlognormal("lr", np.log(prediction["lr"]),
                          0.5 * prediction["lr"], 0.05 * prediction["lr"]),
            'epochs':
            hp.quniform('epochs', prediction["epochs"] - 20,
                        prediction["epochs"] + 20, 5),
            'kernel_l2':
            hp.choice('kernel_l2', [
                0.0,
                hp.qloguniform('kernel_l2_choice', np.log(1e-9), np.log(1e-5),
                               5e-9)
            ]),
            'batch_size':
            hp.qloguniform('batch_size', np.log(512), np.log(1023), 512),
            # 'batch_size': hp.choice('batch_size', [512]),
            'conv_depth':
            hp.quniform('conv_depth', 1, prediction["conv_depth"] + 1, 1),
            'gaussian_noise':
            hp.qlognormal('gaussian_noise',
                          np.log(prediction["gaussian_noise"]),
                          0.5 * prediction["gaussian_noise"], 0.005),
            'network_depth':
            hp.quniform('network_depth', 1, prediction['network_depth'] + 1,
                        1),
            'n_dims':
            hp.quniform('n_dims', starting_params["n_dims"] - 32,
                        starting_params["n_dims"] + 64, 16),
        }
        return rmse_space
    if objective == "ndcg":

        embedding = starting_params["collaborative_params"]["user_item_params"]
        ndcg_space = {
            'gcn_lr':
            hp.qlognormal("gcn_lr", np.log(embedding["gcn_lr"]),
                          0.5 * embedding["gcn_lr"],
                          0.05 * embedding["gcn_lr"]),
            'gcn_epochs':
            hp.quniform('gcn_epochs', embedding["gcn_epochs"],
                        embedding["gcn_epochs"] + 20, 5),
            'gaussian_noise':
            hp.qlognormal('gaussian_noise',
                          np.log(embedding["gaussian_noise"]),
                          1.0 * embedding["gaussian_noise"], 0.005),
            'margin':
            hp.quniform('margin', 0.8, 1.8, 0.2),
            'n_dims':
            hp.quniform('n_dims', starting_params["n_dims"],
                        starting_params["n_dims"] + 96, 16),
        }

        return ndcg_space
Exemple #11
0
    def __call__(self):
        print_exp(self.exp_name)
        if self.only_q:
            pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
                                              np.log(1e-4)),
                             q=(18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
                                30),
                             epochs=(250, ),
                             batch=(32, ))
        elif self.only_batch:
            pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
                                              np.log(1e-3)),
                             q=(23, ),
                             epochs=(250, ),
                             batch=(16, 32, 50, 100, 128, 200))
        elif self.only_epochs:
            pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
                                              np.log(1e-4)),
                             q=(23, ),
                             epochs=(100, 120, 150, 170, 200, 250, 300, 400,
                                     500),
                             batch=(32, ))
        elif self.only_lr:
            pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
                                              np.log(1e-3)),
                             q=(23, ),
                             epochs=(250, ),
                             batch=(32, ))
        else:
            pv = ParamValues(
                lr=hp.loguniform("lr", np.log(1e-4), np.log(1e-3)),
                q=scope.int(hp.qloguniform("q", np.log(10), np.log(100), 1)),
                #q=(18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,35),
                epochs=scope.int(
                    hp.qloguniform("epochs", np.log(10), np.log(500), 10)),
                #epochs=(100, 125, 150, 175, 200, 250, 300, 400, 500, 700),
                batch=(4, 8, 16, 32, 64, 128, 256))

        hyper_params = {
            "data": {},
            "model": {
                "lr": pv.lr,
                #"encoding_dim": hp.choice("encoding_dim", pv.q),  ##
                "encoding_dim": pv.q,
            },
            "fit": {
                #"epochs": hp.choice("epochs", pv.epochs),  #
                "epochs": pv.epochs,
                "batch_size": hp.choice("batch_size", pv.batch)
            }
        }

        run = RunFN(self.metric, hyper_params, pv, self.data_path, self.sep,
                    self.run_on_mongodb, self.start_mongodb, self.db_name,
                    self.exp_name, self.ip, self.port, self.nr_of_trials,
                    self.nr_of_workers)
        run()
Exemple #12
0
def get_hp_space():
    space = (hp.qloguniform('n_hidden', log(100), log(5000), 10),
             hp.qloguniform('temporal_order', log(1), log(30), 1),
             hp.loguniform('learning_rate', log(0.0001), log(1)),
             hp.quniform('batch_size', 10, 500, 10)
             #  hp.choice('activation_func', ['tanh', 'sigmoid']),
             #  hp.choice('sampling_positive', ['true', 'false'])
             # gibbs_sampling_step_test ???
             )
    return space
Exemple #13
0
 def __init__(self):
     self.search_space = {
       'learning_rate': hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
       'L1_flag': hp.choice('L1_flag', [True, False]),
       'hidden_size': scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(128),1)),
       'batch_size': scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),1)),
       'margin': hp.uniform('margin', 0.0, 2.0),
       'optimizer': hp.choice('optimizer', ["adam", "sgd", 'rms']),
       'epochs': hp.choice('epochs', [10]) # always choose 10 training epochs.
     }
 def tpe_configspace(self):
     from hyperopt import hp
     import numpy as np
     space = {
         'l_rate': hp.loguniform('l_rate', np.log(1e-6), np.log(1e-1)),
         'burn_in': hp.uniform('burn_in', 0, .8),
         'n_units_1': hp.qloguniform('n_units_1', np.log(16), np.log(512), 1),
         'n_units_2': hp.qloguniform('n_units_2', np.log(16), np.log(512), 1),
         'mdecay': hp.uniform('mdecay', 0, 1)
     }
     return(space)
class GradientBoostingModel(TreeBasedModel):
    @staticmethod
    def build_estimator(args, train_data=None):
        return GradientBoostingRegressor(random_state=RANDOM_STATE,
                                         presort=True,
                                         **args)

    loss_alpha = hp.choice('loss_alpha',
                           [('ls', 0.9), ('lad', 0.9),
                            ('huber', hp.uniform('gbr_alpha', 0.85, 0.95)),
                            ('quantile', 0.5)])

    hp_space = {
        'n_estimators':
        scope.int(
            hp.qloguniform('n_estimators', np.log(10.5), np.log(1000.5), 1)),
        'learning_rate':
        hp.lognormal('learning_rate', np.log(0.01), np.log(10.0)),
        'criterion':
        hp.choice('criterion', ['mse', 'friedman_mse', 'mae']),
        'max_depth':
        hp.pchoice('max_depth', [(0.2, 2), (0.5, 3), (0.2, 4), (0.1, 5)]),
        'min_samples_leaf':
        hp.choice(
            'min_samples_leaf_enabled',
            [
                1,  # most common choice.
                scope.int(
                    hp.qloguniform('min_samples_leaf', np.log(1.5),
                                   np.log(50.5), 1))
            ]),
        'subsample':
        hp.pchoice(
            'subsample_enabled',
            [
                (0.2, 1.0),  # default choice.
                (0.8, hp.uniform('subsample', 0.5, 1.0)
                 )  # stochastic grad boosting.
            ]),
        'max_features':
        hp.pchoice(
            'max_features_str',
            [
                (0.1, 'sqrt'),  # most common choice.
                (0.2, 'log2'),  # less common choice.
                (0.1, None),  # all features, less common choice.
                (0.6, hp.uniform('max_features_str_frac', 0., 1.))
            ]),
        'loss':
        loss_alpha[0],
        'alpha':
        loss_alpha[1]
    }
Exemple #16
0
def define_search_space(objective, starting_params):
    prediction = starting_params["collaborative_params"][
        "prediction_network_params"]
    space = {
        'lr':
        hp.qlognormal("lr", np.log(prediction["lr"]), 0.5 * prediction["lr"],
                      0.05 * prediction["lr"]),
        'epochs':
        hp.quniform('epochs', prediction["epochs"] - 10,
                    prediction["epochs"] + 20, 5),
        'kernel_l2':
        hp.choice('kernel_l2', [
            0.0,
            hp.qloguniform('kernel_l2_choice', np.log(1e-9), np.log(1e-5),
                           5e-9)
        ]),
        'batch_size':
        hp.qloguniform('batch_size', np.log(1024), np.log(4096), 1024),
        'conv_depth':
        hp.quniform('conv_depth', 1, prediction["conv_depth"] + 2, 1),
        'gcn_layers':
        hp.quniform('gcn_layers', 1, prediction["gcn_layers"] + 1, 1),
        'ncf_layers':
        hp.quniform('ncf_layers', 1, prediction["ncf_layers"] + 1, 1),
        'ps_proportion':
        hp.choice('ps_proportion', [
            0.0,
            hp.qloguniform('ps_proportion_choice', np.log(0.1),
                           np.log(prediction["ps_proportion"] + 1.0), 0.05)
        ]),
        'ns_proportion':
        hp.quniform('ns_proportion', 0.0, prediction["ns_proportion"] + 2.0,
                    0.1),
        'nsh':
        hp.quniform('nsh', 0.0, prediction["nsh"] + 2.0, 0.1),
        # 'gaussian_noise': hp.qlognormal('gaussian_noise', np.log(prediction["gaussian_noise"]),
        #                                 0.5 * prediction["gaussian_noise"], 0.005),
        'gaussian_noise':
        hp.choice('gaussian_noise', [
            0.0,
            hp.qloguniform('gaussian_noise_choice', np.log(1e-3), np.log(0.5),
                           1e-3)
        ]),
        'margin':
        hp.choice('margin', [
            0.0,
            hp.qloguniform('margin_choice', np.log(1e-4), np.log(0.05), 5e-4)
        ]),
        'n_dims':
        hp.quniform('n_dims', starting_params["n_dims"] - 16,
                    starting_params["n_dims"] + 64, 16),
    }
    return space
Exemple #17
0
class GradientBoostingModel(TreeBasedModel):
    @staticmethod
    def build_estimator(args, train_data=None):
        return GradientBoostingRegressor(
            random_state=RANDOM_STATE, presort=True, **args
        )

    loss_alpha = hp.choice(
        "loss_alpha",
        [
            ("ls", 0.9),
            ("lad", 0.9),
            ("huber", hp.uniform("gbr_alpha", 0.85, 0.95)),
            ("quantile", 0.5),
        ],
    )

    hp_space = {
        "n_estimators": scope.int(
            hp.qloguniform("n_estimators", np.log(10.5), np.log(1000.5), 1)
        ),
        "learning_rate": hp.lognormal("learning_rate", np.log(0.01), np.log(10.0)),
        "criterion": hp.choice("criterion", ["mse", "friedman_mse", "mae"]),
        "max_depth": hp.pchoice("max_depth", [(0.2, 2), (0.5, 3), (0.2, 4), (0.1, 5)]),
        "min_samples_leaf": hp.choice(
            "min_samples_leaf_enabled",
            [
                1,  # most common choice.
                scope.int(
                    hp.qloguniform("min_samples_leaf", np.log(1.5), np.log(50.5), 1)
                ),
            ],
        ),
        "subsample": hp.pchoice(
            "subsample_enabled",
            [
                (0.2, 1.0),  # default choice.
                (0.8, hp.uniform("subsample", 0.5, 1.0)),  # stochastic grad boosting.
            ],
        ),
        "max_features": hp.pchoice(
            "max_features_str",
            [
                (0.1, "sqrt"),  # most common choice.
                (0.2, "log2"),  # less common choice.
                (0.1, None),  # all features, less common choice.
                (0.6, hp.uniform("max_features_str_frac", 0.0, 1.0)),
            ],
        ),
        "loss": loss_alpha[0],
        "alpha": loss_alpha[1],
    }
def optimize(max_evals, max_ppn):
    space = hp.choice('schedule', [
        ('static', hp.qloguniform('chunk_s', 2, 11, 10),
         hp.randint('ppn_s', max_ppn)),
        ('dynamic', hp.qloguniform('chunk_d', 2, 11, 10),
         hp.randint('ppn_d', max_ppn)),
        ('guided', hp.qloguniform('chunk_g', 2, 11, 10),
         hp.randint('ppn_g', max_ppn)),
    ])
    trials = Trials()
    best = fmin(function, space=space, algo=tpe.suggest,
                max_evals=max_evals, trials=trials)
    return best, trials
Exemple #19
0
def get_hp_space():
    space = (hp.qloguniform('n_hidden', log(100), log(5000), 10),
             hp.qloguniform('n_hidden_recurrent', log(100), log(5000), 1),
             hp.qloguniform('temporal_order', log(10), log(100), 10),
             hp.loguniform('learning_rate_RBM', log(0.0001), log(1)),
             hp.loguniform('learning_rate_RNN', log(0.0001), log(1)),
             hp.qloguniform('K', log(2), log(20), 1)
             # Different learninf rate for RBM and RNN
             # hp.choice('activation_func', ['tanh', 'sigmoid']),
             # hp.choice('sampling_positive', ['true', 'false'])
             # gibbs_sampling_step_test ???
             )
    return space
 def __init__(self, train_fname, test_fname, feat_types_fname, cat_count_fname, work_dir, task,\
         max_evals, n_cv_folds, num_trees, num_threads, time_log_fname, train_query_fname=None, test_query_fname=None):
     super().__init__(train_fname, test_fname, feat_types_fname, work_dir, task,\
         max_evals, n_cv_folds, time_log_fname, train_query_fname=train_query_fname, test_query_fname=test_query_fname)
     print("using LightGBM in " + str(lgb.__file__))
     if self.task == "ranking":
         self.objective = "lambdarank"
     else:
         self.objective = self.task
     self.num_threads = num_threads
     self.num_trees = num_trees
     param_dict = {
         'learning_rate':
         hp.loguniform('learning_rate', -7, 0),
         'num_leaves':
         hp.qloguniform('num_leaves', 1, 7, 1),
         'max_depth':
         0,
         'max_bin':
         2**(hp.randint('max_bin_minus_4', 7) + 4),
         'feature_fraction':
         hp.uniform('feature_fraction', 0.5, 1),
         'bagging_fraction':
         hp.uniform('bagging_fraction', 0.5, 1),
         'min_data_in_leaf':
         hp.qloguniform('min_data_in_leaf', 0, 6, 1),
         'min_sum_hessian_in_leaf':
         hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
         'lambda_l1':
         hp.choice('lambda_l1',
                   [0, hp.loguniform('lambda_l1_log', -16, 2)]),
         'lambda_l2':
         hp.choice('lambda_l2',
                   [0, hp.loguniform('lambda_l2_log', -16, 2)]),
         "tree_learner":
         "serial"
     }
     if len(self.categorical_features) > 0:
         max_cat_count = 0
         with open(cat_count_fname, "r") as in_file:
             for line in in_file:
                 _, cat_count = line.strip().split("\t")
                 cat_count = int(cat_count)
                 if cat_count > max_cat_count:
                     max_cat_count = cat_count
         param_dict["cat_smooth"] = hp.qloguniform('cat_smooth', 0, 8, 1)
         param_dict["cat_l2"] = hp.qloguniform('cat_l2', 0, 6, 1)
         param_dict["max_cat_threshold"] = hp.randint(
             'max_cat_threshold', max_cat_count // 2) + 1
     self.param_space = param_dict
     self.metric = None
    def set_multilayer_dropout_space(self):
        ''' defines a hyperspace for a "modern" neural networks: at least two layers with dropout + reLU '''

        # Force at least 2 layers, cuz we're modern
        min_layers = 2
        max_layers = 3

        # sets up the neural network
        nnets = [None] * (max_layers - min_layers + 1)

        for i, num_layers in enumerate(range(min_layers, max_layers + 1)):
            num_hids = [None] * num_layers
            for j in range(num_layers):
                num_hids[j] = hp.qloguniform(
                    'num_hid_%i%i' % (i, j), log(100), log(1000), 1)

            nnets[i] = num_hids

        default_mln_model_params = {
            'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy', 'dropout']}

        search_mln_model_params = {
            'arch': hp.choice('arch', nnets),
            'input_p': hp.uniform('ip', 0, 1),
            'hidden_p': hp.uniform('hp', 0, 1),
            'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
            'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}

        default_mln_optim_params = {
            'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}

        search_mln_optim_params = {
            'learn_rate': hp.uniform('learn_rate', 0, 1),
            'rho': hp.uniform('rho', 0, 1),
            'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
            'batch_size': hp.quniform('batch_size', 128, 1024, 1),
            'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
            'scale_factor': hp.uniform('scale_factor', 0, 1)}

        # merge the default and search spaces
        mln_model_params = self.merge_default_search(
            default_mln_model_params, search_mln_model_params)
        mln_optim_params = self.merge_default_search(
            default_mln_optim_params, search_mln_optim_params)

        # define the hyperparamater space to search
        hyperspace = {'mln_model_params': mln_model_params,
                      'mln_optim_params': mln_optim_params}

        return hyperspace
Exemple #22
0
    def _config_tuning_space(tuning_space_raw):
        if tuning_space_raw is None:
            return None

        hyper_obj = {}
        if "learning_rate" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"learning_rate": hp.loguniform('learning_rate', np.log(tuning_space_raw['learning_rate']['min']), np.log(tuning_space_raw['learning_rate']['max']))}}
        if "hidden_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"hidden_size": scope.int(hp.qloguniform('hidden_size', np.log(tuning_space_raw['hidden_size']['min']), np.log(tuning_space_raw['hidden_size']['max']), 1))}}
        if "ent_hidden_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"ent_hidden_size": scope.int(hp.qloguniform("ent_hidden_size", np.log(tuning_space_raw['ent_hidden_size']['min']), np.log(tuning_space_raw['ent_hidden_size']['max']), 1))}}
        if "rel_hidden_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"rel_hidden_size": scope.int(hp.qloguniform("rel_hidden_size", np.log(tuning_space_raw['rel_hidden_size']['min']), np.log(tuning_space_raw['rel_hidden_size']['max']), 1))}}
        if "batch_size" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"batch_size": scope.int(hp.qloguniform("batch_size", np.log(tuning_space_raw['batch_size']['min']), np.log(tuning_space_raw['batch_size']['max']), 1))}}
        if "margin" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"margin": hp.uniform("margin", tuning_space_raw["margin"]["min"], tuning_space_raw["margin"]["max"])}}
        if "lmbda" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"lmbda": hp.loguniform('lmbda', np.log(tuning_space_raw["lmbda"]["min"]), np.log(tuning_space_raw["lmbda"]["max"]))}}
        if "distance_measure" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"distance_measure": hp.choice('distance_measure', tuning_space_raw["distance_measure"])}}
        if "cmax" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"cmax": hp.loguniform('cmax', np.log(tuning_space_raw["cmax"]["min"]), np.log(tuning_space_raw["cmax"]["max"]))}}
        if "cmin" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"cmin": hp.loguniform('cmin', np.log(tuning_space_raw["cmin"]["min"]), np.log(tuning_space_raw["cmin"]["max"]))}}
        if "optimizer" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"optimizer": hp.choice("optimizer", tuning_space_raw["optimizer"])}}
        if "bilinear" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"bilinear": hp.choice('bilinear', tuning_space_raw["bilinear"])}}
        if "epochs" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"epochs": hp.choice("epochs", tuning_space_raw["epochs"])}}
        if "feature_map_dropout" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"feature_map_dropout": hp.choice('feature_map_dropout', tuning_space_raw["feature_map_dropout"])}}
        if "input_dropout" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"input_dropout": hp.choice('input_dropout', tuning_space_raw["input_dropout"])}}
        if "hidden_dropout" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"hidden_dropout": hp.choice('hidden_dropout', tuning_space_raw["hidden_dropout"])}}
        if "use_bias" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"use_bias": hp.choice('use_bias', tuning_space_raw["use_bias"])}}
        if "label_smoothing" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"label_smoothing": hp.choice('label_smoothing', tuning_space_raw["label_smoothing"])}}
        if "lr_decay" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"lr_decay": hp.choice('lr_decay', tuning_space_raw["lr_decay"])}}
        if "l1_flag" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"l1_flag": hp.choice('l1_flag', tuning_space_raw["l1_flag"])}}
        if "sampling" in tuning_space_raw:
            hyper_obj = {**hyper_obj, **{"sampling": hp.choice('sampling', tuning_space_raw["sampling"])}}

        return hyper_obj
Exemple #23
0
    def __init__(self,
                 learning_task,
                 n_estimators=5000,
                 max_hyperopt_evals=50,
                 counters_sort_col=None,
                 holdout_size=0,
                 train_path=None,
                 test_path=None,
                 cd_path=None,
                 num_class=2,
                 output_folder_path='./'):
        Experiment.__init__(self, learning_task, 'lgb', n_estimators,
                            max_hyperopt_evals, True, counters_sort_col,
                            holdout_size, train_path, test_path, cd_path,
                            num_class, output_folder_path)

        self.space = {
            'learning_rate':
            hp.loguniform('learning_rate', -7, 0),
            'num_leaves':
            hp.qloguniform('num_leaves', 0, 7, 1),
            'feature_fraction':
            hp.uniform('feature_fraction', 0.5, 1),
            'bagging_fraction':
            hp.uniform('bagging_fraction', 0.5, 1),
            'min_data_in_leaf':
            hp.qloguniform('min_data_in_leaf', 0, 6, 1),
            'min_sum_hessian_in_leaf':
            hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
            'lambda_l1':
            hp.choice('lambda_l1',
                      [0, hp.loguniform('lambda_l1_positive', -16, 2)]),
            'lambda_l2':
            hp.choice('lambda_l2',
                      [0, hp.loguniform('lambda_l2_positive', -16, 2)]),
        }

        self.default_params = {
            'learning_rate': 0.1,
            'num_leaves': 127,
            'feature_fraction': 1.0,
            'bagging_fraction': 1.0,
            'min_data_in_leaf': 100,
            'min_sum_hessian_in_leaf': 10,
            'lambda_l1': 0,
            'lambda_l2': 0
        }
        self.default_params = self.preprocess_params(self.default_params)
        self.title = 'LightGBM'
    def tpe_configspace(self):

        import numpy as np
        from hyperopt import hp

        space = {
            'learning_rate': hp.loguniform('learning_rate', np.log(1e-7), np.log(1e-1)),
            'batch_size': hp.qloguniform('batch_size', np.log(8), np.log(256), 1),
            'n_units_1': hp.qloguniform('n_units_1', np.log(8), np.log(128), 1),
            'n_units_2': hp.qloguniform('n_units_2', np.log(8), np.log(128), 1),
            'discount': hp.uniform('discount', 0, 1),
            'likelihood_ratio_clipping': hp.uniform('likelihood_ratio_clipping', 0, 1),
            'entropy_regularization': hp.uniform('entropy_regularization', 0, 1)
        }
        return(space)
 def __init__(self, train_fname, test_fname, feat_types_fname, work_dir, task, max_evals,\
         n_cv_folds, num_trees, num_threads, time_log_fname, train_query_fname=None, test_query_fname=None):
     super().__init__(train_fname, test_fname, feat_types_fname, work_dir, task, max_evals,\
         n_cv_folds, time_log_fname, train_query_fname=train_query_fname, test_query_fname=test_query_fname)
     print("using CatBoost in " + str(cat.__file__))
     if self.task == "ranking":
         self.objective = "YetiRank"
     elif self.task == "binary":
         self.objective = "Logloss"
     elif self.task == "regression":
         self.objective = "RMSE"
     else:
         raise NotImplementedError("Unknown task " + self.task)
     self.num_threads = num_threads
     self.num_trees = num_trees
     param_dict = {
         'learning_rate':
         hp.loguniform('learning_rate', -7, 0),
         'num_leaves':
         hp.qloguniform('num_leaves', 1, 7, 1),
         'border_count':
         2**(hp.randint('max_bin_minus_4', 7) + 4),
         'random_strength':
         hp.randint('random_strength', 20) + 1,
         'colsample_bylevel':
         hp.uniform('feature_fraction', 0.5, 1),
         'subsample':
         hp.uniform('subsample', 0.5, 1),
         'bagging_temperature':
         hp.uniform('bagging_temperature', 0.0, 1.0),
         'min_data_in_leaf':
         hp.qloguniform('min_data_in_leaf', 0, 6, 1),
         'l2_leaf_reg':
         hp.choice('lambda_l2',
                   [0, hp.loguniform('lambda_l2_log', -16, 2)]),
         "grow_policy":
         "Lossguide",
         "counter_calc_method":
         "SkipTest",
         "boosting_type":
         "Plain"
     }
     if len(self.categorical_features) > 0:
         param_dict["one_hot_max_size"] = hp.randint('one_hot_max_size', 26)
         param_dict["max_ctr_complexity"] = hp.randint(
             'max_ctr_complexity', 6) + 1
     self.param_space = param_dict
     self.column_description_fname = feat_types_fname
def get_simple_feature_adder_wrapper_params(
        inner_model_params,
        max_features=None,
        pre_filter=None,
        features_indexes_getter=None,
        priority_getter=None,
        name='feature_adder_common'
    ):
    priority_getter = priority_getter if priority_getter is not None \
        else get_priority_getter_params(get_full_name(name, 'priority_getter'))
    pre_filter = pre_filter if pre_filter is not None \
        else get_min_size_prefilter_params(get_full_name(name, 'pre_filter'))
    features_indexes_getter = features_indexes_getter if features_indexes_getter is not None \
        else get_index_getter_params(get_full_name(name, 'indexes_getter'))
    max_features = max_features if max_features is not None \
        else hp.qloguniform(
                get_full_name(name, 'max_features'),
                -1, 10, 1,
            )
    extender_strategy = hp.choice(
        get_full_name(name, 'extender_strategy'),
        (
            scope.get_extender_strategy(
                max_features=max_features,
                priority_getter=priority_getter,
                pre_filter=pre_filter,
                simple_features_indexes_getter=features_indexes_getter,
            ),
            scope.get_nothing_doing_extender_strategy(),
        ),
    )
    return scope.get_complex_features_adder_wrapper(
        inner_model=inner_model_params,
        extender_strategy=extender_strategy,
    )
Exemple #27
0
def passive_aggressive(name,
    loss=None,
    C=None,
    fit_intercept=False,
    n_iter=None,
    n_jobs=1,
    shuffle=True,
    random_state=None,
    verbose=False):

    def _name(msg):
        return '%s.%s_%s' % (name, 'sgd', msg)

    rval = scope.sklearn_PassiveAggressiveClassifier(
        loss=hp.choice(
            _name('loss'),
            ['hinge', 'squared_hinge']) if loss is None else loss,
        C=hp.lognormal(
            _name('learning_rate'),
            np.log(0.01),
            np.log(10),
            ) if C is None else C,
        fit_intercept=fit_intercept,
        n_iter=scope.int(
            hp.qloguniform(
                _name('n_iter'),
                np.log(1),
                np.log(1000),
                q=1,
                )) if n_iter is None else n_iter,
        n_jobs=n_jobs,
        random_state=_random_state(_name('rstate'), random_state),
        verbose=verbose
        )
    return rval
Exemple #28
0
def linear_discriminant_analysis(name,
    solver=None,
    shrinkage=None,
    priors=None,
    n_components=None,
    store_covariance=False,
    tol=0.00001):

    def _name(msg):
        return '%s.%s_%s' % (name, 'lda', msg)

    solver_shrinkage = hp.choice(_name('solver_shrinkage_dual'),
                                     [('svd', None),
                                      ('lsqr', None),
                                      ('lsqr', 'auto'),
                                      ('eigen', None),
                                      ('eigen', 'auto')])

    rval = scope.sklearn_LinearDiscriminantAnalysis(
        solver=solver_shrinkage[0] if solver is None else solver,
        shrinkage=solver_shrinkage[1] if shrinkage is None else shrinkage,
        priors=priors,
        n_components=4 * scope.int(
            hp.qloguniform(
                _name('n_components'),
                low=np.log(0.51),
                high=np.log(30.5),
                q=1.0)) if n_components is None else n_components,
        store_covariance=store_covariance,
        tol=tol
        )
    return rval
Exemple #29
0
 def parse_search_space(self, learner_space):
     '''
     search space is dictionary
     {'n_estimators': ('uniform', 1, 1000, 'discrete')}
     '''
     search_space = dict()
     for k, v in learner_space.iteritems():
         if v[2] == 'samples':
             v = (v[0], v[1], min(100, self.X.shape[0]/len(self.kf)-1), v[3])
         if v[3] == 'discrete':
             search_space[k] = hp.quniform(k, v[1], v[2], 1)
         elif v[0] == 'uniform':
             search_space[k] = hp.uniform(k, v[1], v[2])
         elif v[0] == 'loguniform':
             search_space[k] = hp.loguniform(k, v[1], v[2])
         elif v[0] == 'normal':
             search_space[k] = hp.normal(k, v[1], v[2])
         elif v[0] == 'lognormal':
             search_space[k] = hp.lognormal(k, v[1], v[2])
         elif v[0] == 'quniform':
             search_space[k] = hp.quniform(k, v[1], v[2], v[3])
         elif v[0] == 'qloguniform':
             search_space[k] = hp.qloguniform(k, v[1], v[2], v[3])
         elif v[0] == 'qnormal':
             search_space[k] = hp.qnormal(k, v[1], v[2], v[3])
         elif v[0] == 'qlognormal':
             search_space[k] = hp.qlognormal(k, v[1], v[2], v[3])
     return search_space
def linear_discriminant_analysis(name,
    solver=None,
    shrinkage=None,
    priors=None,
    n_components=None,
    store_covariance=False,
    tol=0.00001):

    def _name(msg):
        return '%s.%s_%s' % (name, 'lda', msg)

    solver_shrinkage = hp.choice(_name('solver_shrinkage_dual'),
                                     [('svd', None),
                                      ('lsqr', None),
                                      ('lsqr', 'auto'),
                                      ('eigen', None),
                                      ('eigen', 'auto')])

    rval = scope.sklearn_LinearDiscriminantAnalysis(
        solver=solver_shrinkage[0] if solver is None else solver,
        shrinkage=solver_shrinkage[1] if shrinkage is None else shrinkage,
        priors=priors,
        n_components=4 * scope.int(
            hp.qloguniform(
                _name('n_components'),
                low=np.log(0.51),
                high=np.log(30.5),
                q=1.0)) if n_components is None else n_components,
        store_covariance=store_covariance,
        tol=tol
        )
    return rval
Exemple #31
0
class skKNeighborsReg(_KNeighborsRegressor, BaseWrapperReg, metaclass=MetaBaseWrapperRegWithABC):

    def __init__(self, n_neighbors=5, weights='uniform', algorithm='auto', leaf_size=30, p=2, metric='minkowski',
                 metric_params=None, n_jobs=1, **kwargs):
        n_jobs = 4
        _KNeighborsRegressor.__init__(
            self, n_neighbors, weights, algorithm, leaf_size, p, metric, metric_params, n_jobs, **kwargs)
        BaseWrapperReg.__init__(self)

    HyperOpt_space = {
        'n_neighbors': 1 + hp.randint('n_neighbors', 50),
        'leaf_size': hp.qloguniform('leaf_size', 2, 5, 1),
        'p': hp.uniform('p', 1, 2),

        # 'metric': 'minkowski',
        # 'metric_params': None,

        # 'n_jobs': 1,
        # 'weights': 'uniform',
        # 'algorithm': 'auto',
    }
    tuning_grid = {
        'n_neighbors': 5,
        'weights': 'uniform',
        'algorithm': 'auto',
        'leaf_size': 30,
        'p': 2,
        'metric': 'minkowski',
        'metric_params': None,
        'n_jobs': 1,
    }
def passive_aggressive(name,
    loss=None,
    C=None,
    fit_intercept=False,
    n_iter=None,
    n_jobs=1,
    shuffle=True,
    random_state=None,
    verbose=False):

    def _name(msg):
        return '%s.%s_%s' % (name, 'sgd', msg)

    rval = scope.sklearn_PassiveAggressiveClassifier(
        loss=hp.choice(
            _name('loss'),
            ['hinge', 'squared_hinge']) if loss is None else loss,
        C=hp.lognormal(
            _name('learning_rate'),
            np.log(0.01),
            np.log(10),
            ) if C is None else C,
        fit_intercept=fit_intercept,
        n_iter=scope.int(
            hp.qloguniform(
                _name('n_iter'),
                np.log(1),
                np.log(1000),
                q=1,
                )) if n_iter is None else n_iter,
        n_jobs=n_jobs,
        random_state=_random_state(_name('rstate'), random_state),
        verbose=verbose
        )
    return rval
Exemple #33
0
class skRidgeReg(_RidgeReg,
                 BaseWrapperReg,
                 metaclass=MetaBaseWrapperRegWithABC):
    def __init__(self,
                 alpha=1.0,
                 fit_intercept=True,
                 normalize=False,
                 copy_X=True,
                 max_iter=None,
                 tol=1e-3,
                 solver="auto",
                 random_state=None):
        _RidgeReg.__init__(self, alpha, fit_intercept, normalize, copy_X,
                           max_iter, tol, solver, random_state)
        BaseWrapperReg.__init__(self)

    HyperOpt_space = {
        'alpha': hp.loguniform('alpha', -5, 3),
        'max_iter': hp.qloguniform('max_iter', 4, 8, 1),
        'tol': hp.loguniform('tol', -8, 0),
    }
    tuning_grid = {
        'alpha': 1.0,
        'tol': 1e-3,

        # 'fit_intercept': True,
        # 'normalize': False,
        # 'max_iter': None,

        # 'copy_X': True,
        # 'solver': "auto",
        # 'random_state': None,
    }
    def set_finetune_space(self, config_file):
        ''' Given the original deep net architecture, and a set of pretrained weights
        and biases, define the configuration space to search for fintuning parameters '''

        # we know these fields won't change, so go ahead and set them as
        # defaults now
        model_params = nt.get_model_params(config_file)
        optim_params = nt.get_optim_params(config_file)
        default_finetune_model_params = {k: model_params[k] for k in ('num_hids', 'activs', 'd', 'k')}
        default_finetune_model_params['loss_terms'] = ['cross_entropy']
        default_finetune_optim_params = {k: optim_params[k] for k in ('optim_method', 'optim_type')}

        # define the space of hyperparameters we wish to
        search_finetune_model_params = {'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
                                        'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
        search_finetune_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
                                        'rho': hp.uniform('rho', 0, 1),
                                        'num_epochs': hp.qloguniform('num_epochs', log(10), log(5e3), 1),
                                        'batch_size': hp.quniform('batch_size', 128, 1024, 1),
                                        'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
                                        'scale_factor': hp.uniform('scale_factor', 0, 1)}

        # combine the default and search parameters into a dictionary to define the
        # full space - this is what will be passed into the objective function
        finetune_model_params = self.merge_default_search(
            default_finetune_model_params, search_finetune_model_params)
        finetune_optim_params = self.merge_default_search(
            default_finetune_optim_params, search_finetune_optim_params)

        finetune_hyperspace = {
            'finetune_model_params': finetune_model_params, 'finetune_optim_params': finetune_optim_params}

        return finetune_hyperspace
Exemple #35
0
    def visitSearchSpaceNumber(self, space: SearchSpaceNumber, path: str, counter=None):
        label = self.mk_label(path, counter)

        if space.pgo is not None:
            return scope.pgo_sample(
                space.pgo, hp.quniform(label, 0, len(space.pgo) - 1, 1)
            )

        dist = "uniform"
        if space.distribution:
            dist = space.distribution

        if space.maximum is None:
            raise SearchSpaceError(
                path, f"maximum not specified for a number with distribution {dist}"
            )
        max = space.getInclusiveMax()
        # if the maximum is not None, the inclusive maximum should not be none
        assert max is not None

        # These distributions need only a maximum
        if dist == "integer":
            if not space.discrete:
                raise SearchSpaceError(
                    path,
                    "integer distribution specified for a non discrete numeric type",
                )
            return hp.randint(label, max)

        if space.minimum is None:
            raise SearchSpaceError(
                path, f"minimum not specified for a number with distribution {dist}"
            )
        min = space.getInclusiveMin()
        # if the minimum is not None, the inclusive minimum should not be none
        assert min is not None

        if dist == "uniform":
            if space.discrete:
                return scope.int(hp.quniform(label, min, max, 1))
            else:
                return hp.uniform(label, min, max)
        elif dist == "loguniform":
            # for log distributions, hyperopt requires that we provide the log of the min/max
            if min <= 0:
                raise SearchSpaceError(
                    path,
                    f"minimum of 0 specified with a {dist} distribution.  This is not allowed; please set it (possibly using minimumForOptimizer) to be positive",
                )
            if min > 0:
                min = math.log(min)
            if max > 0:
                max = math.log(max)
            if space.discrete:
                return scope.int(hp.qloguniform(label, min, max, 1))
            else:
                return hp.loguniform(label, min, max)

        else:
            raise SearchSpaceError(path, f"Unknown distribution type: {dist}")
Exemple #36
0
def _validate_and_generate_hyperopt_search_space(search_space):
    if not type(search_space) == dict:
        raise Exception('Search space has to be type dict. Provided: {}'.format(type(search_space)))

    if not all([isinstance(k, str) for k in search_space.keys()]):
        raise Exception('Only string values are allowed for hyperparameter space keys.')

    if not all([isinstance(k, _HP) for k in search_space.values()]):
        raise Exception('All hyperparameter space values has to be of type cerebro.tune.base._HP.'
                        ' Nested search spaces are not supported yet')

    hyperopt_space = {}
    for k in search_space:
        if isinstance(search_space[k], _HPChoice):
            hyperopt_space[k] = hp.choice(k, search_space[k].options)
        elif isinstance(search_space[k], _HPUniform):
            hyperopt_space[k] = hp.uniform(k, search_space[k].min, search_space[k].max)
        elif isinstance(search_space[k], _HPQUniform):
            hyperopt_space[k] = hp.quniform(k, search_space[k].min, search_space[k].max, search_space[k].q)
        elif isinstance(search_space[k], _HPLogUniform):
            hyperopt_space[k] = hp.loguniform(k, search_space[k].min,  search_space[k].max)
        elif isinstance(search_space[k], _HPQLogUnifrom):
            hyperopt_space[k] = hp.qloguniform(k, search_space[k].min, search_space[k].max, search_space[k].q)
        else:
            raise Exception('Unsupported hyperparameter option type: {}'.format(type(search_space[k])))

    return hyperopt_space
Exemple #37
0
class skBaggingClf(BaseWrapperClf, _BaggingClassifier, metaclass=MetaBaseWrapperClfWithABC):
    def __init__(self, base_estimator=None, n_estimators=10, max_samples=1.0, max_features=1.0, bootstrap=True,
                 bootstrap_features=False, oob_score=False, warm_start=False, n_jobs=1, random_state=None, verbose=0):
        n_estimators = int(n_estimators)
        _BaggingClassifier.__init__(
            self, base_estimator, n_estimators, max_samples, max_features, bootstrap, bootstrap_features, oob_score,
            warm_start, n_jobs, random_state, verbose)
        BaseWrapperClf.__init__(self)

    HyperOpt_space = {
        'n_estimators': hp.qloguniform('n_estimators', 2, 5, 1),
    }

    tuning_grid = {

    }

    tuning_params = {
    }

    @property
    def feature_importances(self):
        return np.mean([
            tree.feature_importances_ for tree in self.estimators_
        ], axis=0)

    @property
    def feature_importances_pack(self):
        return {
            'mean': self.feature_importances,
            'std': np.std([
                tree.feature_importances_ for tree in self.estimators_
            ], axis=0)
        }
Exemple #38
0
    def __construct_hyperopt_space(self):
        # Function to contruct the space for fmin in hyperopt

        hyperopt_space = {}
        for param in self.space:
            param_details = self.space[param]
            label = param
            sampling_type = param_details[0]
            bounds = param_details[1]
            if sampling_type == 'choice':
                hyperopt_space[param] = hp.choice(label, bounds)
            elif sampling_type == 'uniform':
                hyperopt_space[param] = hp.uniform(label, bounds[0], bounds[1])
            elif sampling_type == 'randint':
                hyperopt_space[param] = hp.randint(label, bounds[0])
            elif sampling_type == 'quniform':
                hyperopt_space[param] = hp.quniform(label, bounds[0],
                                                    bounds[1], bounds[2])

            elif sampling_type == 'loguniform':
                hyperopt_space[param] = hp.loguniform(label, bounds[0],
                                                      bounds[1])

            elif sampling_type == 'qloguniform':
                hyperopt_space[param] = hp.qloguniform(label, bounds[0],
                                                       bounds[1], bounds[2])

        return hyperopt_space
    def set_old_space(self):
        ''' defines an old net from the 80s - simple sigmoid layers, nothing fancy'''

        min_layers = 1
        max_layers = 3

        # sets up the neural network
        nnets = [None] * (max_layers - min_layers + 1)

        for i, num_layers in enumerate(range(min_layers, max_layers + 1)):
            num_hids = [None] * num_layers
            for j in range(num_layers):
                num_hids[j] = hp.qloguniform(
                    'num_hid_%i%i' % (i, j), log(10), log(100), 1)

            nnets[i] = num_hids

        default_mln_model_params = {
            'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy']}

        search_mln_model_params = {
            'arch': hp.choice('arch', nnets),
            'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
            'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}

        default_mln_optim_params = {
            'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}

        search_mln_optim_params = {
            'learn_rate': hp.uniform('learn_rate', 0, 1),
            'rho': hp.uniform('rho', 0, 1),
            'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
            'batch_size': hp.quniform('batch_size', 128, 1024, 1),
            'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
            'scale_factor': hp.uniform('scale_factor', 0, 1)}

        # merge the default and search spaces
        mln_model_params = self.merge_default_search(
            default_mln_model_params, search_mln_model_params)
        mln_optim_params = self.merge_default_search(
            default_mln_optim_params, search_mln_optim_params)

        # define the hyperparamater space to search
        hyperspace = {'mln_model_params': mln_model_params,
                      'mln_optim_params': mln_optim_params}

        return hyperspace
def get_k_best_params(inner_model_params, name='k_best_selector'):
    return scope.get_k_best_wrapper(
        inner_model=inner_model_params,
        k_best=scope.get_k_best(
            k=hp.qloguniform(get_full_name(name, 'k'), 0, 5, 1),
            score_func=hp.choice(get_full_name(name, 'score'), (chi2, f_classif)),
        ),
    )
Exemple #41
0
def rbm(name,
        n_components=None,
        learning_rate=None,
        batch_size=None,
        n_iter=None,
        verbose=False,
        random_state=None):

    def _name(msg):
        return '%s.%s_%s' % (name, 'rbm', msg)

    rval = scope.sklearn_BernoulliRBM(
        n_components=scope.int(
            hp.qloguniform(
                name + '.n_components',
                low=np.log(0.51),
                high=np.log(999.5),
                q=1.0)) if n_components is None else n_components,
        learning_rate=hp.lognormal(
            name + '.learning_rate',
            np.log(0.01),
            np.log(10),
        ) if learning_rate is None else learning_rate,
        batch_size=scope.int(
            hp.qloguniform(
                name + '.batch_size',
                np.log(1),
                np.log(100),
                q=1,
            )) if batch_size is None else batch_size,
        n_iter=scope.int(
            hp.qloguniform(
                name + '.n_iter',
                np.log(1),
                np.log(1000),  # -- max sweeps over the *whole* train set
                q=1,
            )) if n_iter is None else n_iter,
        verbose=verbose,
        random_state=_random_state(_name('rstate'), random_state),
    )
    return rval
Exemple #42
0
def many_dists():
    a = hp.choice('a', [0, 1, 2])
    b = hp.randint('b', 10)
    c = hp.uniform('c', 4, 7)
    d = hp.loguniform('d', -2, 0)
    e = hp.quniform('e', 0, 10, 3)
    f = hp.qloguniform('f', 0, 3, 2)
    g = hp.normal('g', 4, 7)
    h = hp.lognormal('h', -2, 2)
    i = hp.qnormal('i', 0, 10, 2)
    j = hp.qlognormal('j', 0, 2, 1)
    k = hp.pchoice('k', [(.1, 0), (.9, 1)])
    z = a + b + c + d + e + f + g + h + i + j + k
    return {'loss': scope.float(scope.log(1e-12 + z ** 2)),
            'status': base.STATUS_OK}
Exemple #43
0
 def test_read_qloguniform(self):
     # 0 float
     # 1   hyperopt_param
     # 2     Literal{nhid1}
     # 3     qloguniform
     # 4       Literal{2.77258872224}
     # 5       Literal{6.9314718056}
     # 6      q =
     # 7       Literal{16}
     qloguniform = hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16). \
         inputs()[0].inputs()[1]
     ret = self.pyll_reader.read_qloguniform(qloguniform, 'nhid1')
     expected = configuration_space.UniformFloatHyperparameter(
         'nhid1', 16, 1024, q=16, base=np.e)
     self.assertEqual(expected, ret)
def pca(name, n_components=None, whiten=None, copy=True):
    rval = scope.sklearn_PCA(
        # -- qloguniform is missing a "scale" parameter so we
        #    lower the "high" parameter and multiply by 4 out front
        n_components=4 * scope.int(
            hp.qloguniform(
                name + '.n_components',
                low=np.log(0.51),
                high=np.log(30.5),
                q=1.0)) if n_components is None else n_components,
        whiten=hp_bool(
            name + '.whiten',
            ) if whiten is None else whiten,
        copy=copy,
        )
    return rval
    def _process_vw_argument(self, arg, value, algorithm):
        try:
            distr_part = self.distr_pattern.findall(value)[0]
        except IndexError:
            distr_part = ''
        range_part = self.range_pattern.findall(value)[0]
        is_continuous = '..' in range_part

        ocd = self.only_continuous.findall(value)
        if not is_continuous and len(ocd)> 0 and ocd[0] != '':
            raise ValueError(("Need a range instead of a list of discrete values to define "
                              "uniform or log-uniform distribution. "
                              "Please, use [min..max]%s form") % (distr_part))

        if is_continuous and arg == '-q':
            raise ValueError(("You must directly specify namespaces for quadratic features "
                              "as a list of values, not as a parametric distribution"))

        hp_choice_name = "_".join([algorithm, arg.replace('-', '')])

        try_omit_zero = 'O' in distr_part
        distr_part = distr_part.replace('O', '')

        if is_continuous:
            vmin, vmax = [float(i) for i in range_part.split('..')]

            if distr_part == 'L':
                distrib = hp.loguniform(hp_choice_name, log(vmin), log(vmax))
            elif distr_part == '':
                distrib = hp.uniform(hp_choice_name, vmin, vmax)
            elif distr_part == 'I':
                distrib = hp.quniform(hp_choice_name, vmin, vmax, 1)
            elif distr_part in {'LI', 'IL'}:
                distrib = hp.qloguniform(hp_choice_name, log(vmin), log(vmax), 1)
            else:
                raise ValueError("Cannot recognize distribution: %s" % (distr_part))
        else:
            possible_values = range_part.split(',')
            if arg == '-q':
                possible_values = [v.replace('+', ' -q ') for v in possible_values]
            distrib = hp.choice(hp_choice_name, possible_values)

        if try_omit_zero:
            hp_choice_name_outer = hp_choice_name + '_outer'
            distrib = hp.choice(hp_choice_name_outer, ['omit', distrib])

        return distrib
def colkmeans(name,
              n_clusters=None,
              init=None,
              n_init=None,
              max_iter=None,
              tol=None,
              precompute_distances=True,
              verbose=0,
              random_state=None,
              copy_x=True,
              n_jobs=1):
    rval = scope.sklearn_ColumnKMeans(
        n_clusters=scope.int(
            hp.qloguniform(
                name + '.n_clusters',
                low=np.log(1.51),
                high=np.log(19.5),
                q=1.0)) if n_clusters is None else n_clusters,
        init=hp.choice(
            name + '.init',
            ['k-means++', 'random'],
            ) if init is None else init,
        n_init=hp.choice(
            name + '.n_init',
            [1, 2, 10, 20],
            ) if n_init is None else n_init,
        max_iter=scope.int(
            hp.qlognormal(
                name + '.max_iter',
                np.log(300),
                np.log(10),
                q=1,
                )) if max_iter is None else max_iter,
        tol=hp.lognormal(
            name + '.tol',
            np.log(0.0001),
            np.log(10),
            ) if tol is None else tol,
        precompute_distances=precompute_distances,
        verbose=verbose,
        random_state=random_state,
        copy_x=copy_x,
        n_jobs=n_jobs,
        )
    return rval
Exemple #47
0
def nystrom(name, n_components=None, kernel=None, max_components=np.Inf, copy=True):
    
    def _name(msg):
      return '%s.%s_%s' % (name, 'nystrom', msg)
  
    rval = scope.sklearn_Nystrom(
            n_components=4 * scope.int(
            hp.qloguniform(
                name + '.n_components',
                low=np.log(0.51),
                high=np.log(min(max_components / 4, 30.5)),
                q=1.0)) if n_components is None else n_components,
            kernel=hp.pchoice(
            _name('kernel'),
            [ (0.35, 'sigmoid'),
              (0.35, 'rbf'),
              (0.30, 'poly')]) if kernel is None else kernel,
            gamma=_svc_gamma('gamma'),
            coef0=hp.uniform(_name('coef0'), 0.0, 1.0)
        )
    return rval
Exemple #48
0
def parameter_to_tpe(label, parameter):
    """returns the parameter in TPE format."""
    if parameter.is_int:
        if parameter.log_scale:
            return hp.qloguniform(label,
                                  log(parameter.min_val),
                                  log(parameter.max_val),
                                  1)
        else:
            return hp.quniform(label,
                               parameter.min_val,
                               parameter.max_val,
                               1)
    else:
        if parameter.log_scale:
            return hp.loguniform(label,
                                 log(parameter.min_val),
                                 log(parameter.max_val))
        else:
            return hp.uniform(label,
                                 parameter.min_val,
                                 parameter.max_val)
    def init2(self):
        Files.mkdir("../model/others")

        self.fpath = "../model/others/hopt_vw.txt"
        if config.test:
            self.max_evals = 1
        else:            
            self.max_evals = 25
            
        # TODO: read passes from hyperopt result
        self.space = {
            "l1": hp.qloguniform("l1", np.log(1e-9), np.log(1e-6), 1e-9),
            'interaction': "",
            "passes": 15
        }

        self.i_folds = [("B", 0)]

        self.output_items = []
        self.output_items += ["loss", "rsme"]
        self.output_items += ["loss{}".format(i) for i, i_fold in enumerate(self.i_folds)]
        self.output_items += ["rsme{}".format(i) for i, i_fold in enumerate(self.i_folds)]        

        self.columns = sorted(self.space.keys())
Exemple #50
0
"""
Cart-pole balancing with independent discretization
"""
from rlpy.Domains import InfCartPoleBalance
from rlpy.Agents import Q_Learning
from rlpy.Representations import *
from rlpy.Policies import eGreedy
from rlpy.Experiments import Experiment
import numpy as np
from hyperopt import hp

param_space = {
    "num_rbfs": hp.qloguniform("num_rbfs", np.log(1e1), np.log(1e4), 1),
    'resolution': hp.quniform("resolution", 3, 30, 1),
    'lambda_': hp.uniform("lambda_", 0., 1.),
    'boyan_N0': hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)),
    'initial_learn_rate': hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1))}


def make_experiment(
        exp_id=1, path="./Results/Temp/{domain}/{agent}/{representation}/",
        boyan_N0=753,
        initial_learn_rate=.7,
        resolution=25.,
        num_rbfs=206.,
        lambda_=0.75):
    opt = {}
    opt["path"] = path
    opt["exp_id"] = exp_id
    opt["max_steps"] = 10000
    opt["num_policy_checks"] = 20
Exemple #51
0
    'eta' : hp.quniform('eta', 0.3, 2, 0.2),
    'lambda' : hp.quniform('lambda', 0, 5, 0.05),
    'alpha' : hp.quniform('alpha', 0, 0.5, 0.005),
    'lambda_bias' : hp.quniform('lambda_bias', 0, 3, 0.1),
    'num_round' : hp.quniform('num_round', xgb_min_num_round, xgb_max_num_round, xgb_num_round_step),
    'nthread': xgb_nthread,
    'silent' : 1,
    'seed': xgb_random_seed,
    "max_evals": hyperopt_param["xgb_max_evals"],
}
#
param_space_reg_xgb_tree_1 = {
    'task': 'regression',
    'booster': 'gblinear',
    'objective': 'multi:softmax',
    'max_delta_step':hp.qloguniform('max_delta_step',1,10,1),
    'num_class':20,
    'eta': hp.quniform('eta', 0.01, 1, 0.05),
    'gamma': hp.quniform('gamma', 0.2, 4, 0.2),
    'min_child_weight': hp.quniform('min_child_weight', 1, 10,1),
    'max_depth': hp.quniform('max_depth', 4, 8, 1),
    'subsample': hp.quniform('subsample', 0.5, 1, 0.1),
    'colsample_bytree': hp.quniform('colsample_bytree', 0.1, 1, 0.01),
    'num_round': hp.quniform('num_round', xgb_min_num_round, xgb_max_num_round, xgb_num_round_step),
    'nthread': xgb_nthread,
    'silent': 1,
    'seed': xgb_random_seed,
    "max_evals": hyperopt_param["xgb_max_evals"],
}
param_space_reg_xgb_tree = {
    'task': 'regression',
	ridge = Ridge( alpha = ridge_alpha )
	
	elmr = pipeline.Pipeline( [( 'rl1', rl1 ), ( 'rl2', rl2 ), ( 'ridge', ridge )] )	
	
	elmr.fit( x_train, y_train )
	p = elmr.predict( x_test )

	rmse = RMSE( y_test, p )
	return rmse
	


###

space = ( 
	hp.qloguniform( 'n_hidden_1', log( 10 ), log( 1000 ), 1 ),
	hp.uniform( 'alpha_1', 0, 1 ),
	hp.loguniform( 'rbf_width_1', log( 1e-5 ), log( 100 )),
	
	hp.qloguniform( 'n_hidden_2', log( 10 ), log( 1000 ), 1 ),
	hp.uniform( 'alpha_2', 0, 1 ),
	hp.loguniform( 'rbf_width_2', log( 1e-5 ), log( 100 )),
	
	hp.loguniform( 'ridge_alpha', -15, 5 )
)

###

if __name__ == '__main__':

	headers = [ 'rmse', 'n_hidden_1', 'alpha_1', 'rbf_width_1',
Exemple #53
0
import numpy as np

from hyperopt import hp

import hpnnet.nips2011

space = {'preproc': hp.choice('preproc', [{
                    'preproc' : 0}, {
                    'preproc' : 1, # Column Normalization
                    'colnorm_thresh' : hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)),
                    }, {
                    'preproc' : 2, # PCA
                    'pca_energy' : hp.uniform('pca_energy', .5, 1),
                    }]),
        'nhid1': hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16),
        'dist1': hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal
        'scale_heur1': hp.choice('scale_heur1', [{
                       'scale_heur1' : 0,   # Old
                       'scale_mult1': hp.uniform('scale_mult1', .2, 2)}, {
                       'scale_heur1': 1}]), # Glorot
        'squash' : hp.choice('squash', [0, 1]), # ['tanh', 'logistic']
        'iseed': hp.choice('iseed', [0, 1, 2, 3]), # Are 5, 6, 7, 8
        'batch_size': hp.choice('batch_size', [0, 1]), # 20 or 100
        'lr': hp.lognormal('lr', np.log(.01), 3.),
        'lr_anneal_start': hp.qloguniform('lr_anneal_start', np.log(100), np.log(10000), q=1),
        'l2_penalty': hp.choice('l2_penalty', [{
        'l2_penalty' : 0}, { # Zero
        'l2_penalty' : 1,    # notzero
        'l2_penalty_nz': hp.lognormal('l2_penalty_nz', np.log(1.0e-6), 2.)}])
        }
	
	rl = GRBFRandomLayer( n_hidden = n_hidden, grbf_lambda = grbf_lambda )
	elmr = GenELMRegressor( hidden_layer = rl )
	
	elmr.fit( x_train, y_train )
	p = elmr.predict( x_test )

	rmse = RMSE( y_test, p )
	return rmse
	


###

space = ( 
	hp.qloguniform( 'n_hidden', log( 10 ), log( 1000 ), 1 ),
	hp.loguniform( 'grbf_lambda', -20, 0 )
)

###

if __name__ == '__main__':

	headers = [ 'rmse', 'n_hidden', 'grbf_lambda' ]
	o_f = open( output_file, 'wb' )
	writer = csv.writer( o_f )
	writer.writerow( headers )

	start_time = time()
	best = fmin( run_wrapper, space, algo = tpe.suggest, max_evals = max_evals )
	end_time = time()
    def learn_pretrain_settings(self, config_file):
        ''' automatically searches for the right settings to pre-train the model described in the configuration path '''

        # seperate out the default from the search space
        default_pretrain_model_params = {'corrupt_type': 'mask', 'activs': ['sigmoid', 'sigmoid'],
                                         'loss_terms': ['binary_cross_entropy', 'corruption']}

        default_pretrain_optim_params = {
            'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}

        default_last_model_params = {
            'activs': ['softmax'], 'loss_terms': ['cross_entropy'], 'num_hids': []}

        default_last_optim_params = {
            'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}

        # get the number of nodes per hidden layer from the original architecture so we know how to
        # initialize the autoencoders and the final softmax layer
        model_params = nt.get_model_params(config_file)
        pretrain_nodes = [model_params['d']] + model_params['num_hids']

        best_pretrain_settings = []
        self.curr_X = self.X

        print 'Starting pre-training...'

        for l1, l2 in zip(pretrain_nodes[:-1], pretrain_nodes[1:]):

             # we will reuse this hyperspace (denoising autoencoder) for every pair
            # of input-output layers
            search_pretrain_model_params = {'corrupt_p': hp.uniform('corrupt_p', 0, 1),
                                            'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
                                            'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}

            search_pretrain_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
                                            'rho': hp.uniform('rho', 0, 1),
                                            'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
                                            'batch_size': hp.quniform('batch_size', 128, 1024, 1),
                                            'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
                                            'scale_factor': hp.uniform('scale_factor', 0, 1)}

            # get the next pretraining space ready
            default_pretrain_model_params['d'] = l1
            default_pretrain_model_params['num_hids'] = [l2]

            curr_pretrain_model_params = self.merge_default_search(
                default_pretrain_model_params, search_pretrain_model_params)

            curr_pretrain_optim_params = self.merge_default_search(
                default_pretrain_optim_params, search_pretrain_optim_params)

            # combine the merged default model and optim parameters into a
            # single dictionary
            pretrain_hyperspace = {'pretrain_model_params': curr_pretrain_model_params,
                                   'pretrain_optim_params': curr_pretrain_optim_params}

            # search over the hyperparameters
            print 'Searching over the pretraining hyperspace...'
            best = fmin(self.compute_pretrain_objective, pretrain_hyperspace, algo=tpe.suggest,
                        max_evals=100)
            print 'Complete!'
            print 'Training this layer'

            best_pretrain_settings.append(best)

            # this updates curr_X for the next pre-training layer, and also stores the pre-trained
            # weights
            self.pretrain_layer_with_settings(
                best,
                search_pretrain_model_params,
                search_pretrain_optim_params,
                default_pretrain_model_params,
                default_pretrain_optim_params, layer_type='pretrain')

        print 'Pretraining the final layer..'
        # the last layer is not pretrained with an autoencoder...
        default_last_model_params['d'] = l2
        default_last_model_params['k'] = model_params['k']

        # ...it is trained normally, via simple softmax regression
        search_last_model_params = {'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
                                    'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
        search_last_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
                                    'rho': hp.uniform('rho', 0, 1),
                                    'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
                                    'batch_size': hp.quniform('batch_size', 128, 1024, 1),
                                    'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
                                    'scale_factor': hp.uniform('scale_factor', 0, 1)}

        last_model_params = self.merge_default_search(
            default_last_model_params, search_last_model_params)

        last_optim_params = self.merge_default_search(
            default_last_optim_params, search_last_optim_params)

        last_hyperspace = {
            'last_model_params': last_model_params, 'last_optim_params': last_optim_params}
        best = fmin(self.compute_last_objective, last_hyperspace, algo=tpe.suggest,
                    max_evals=100)
        best_pretrain_settings.append(best)

        self.pretrain_layer_with_settings(
            best,
            search_last_model_params,
            search_last_optim_params,
            default_last_model_params,
            default_last_optim_params, layer_type='last')

        print 'Complete!'
        return best_pretrain_settings
    def model(self):
        #cname = sys._getframe().f_code.co_name
        cname = 'lgb'
        train, y, test = self.train_, self.y_, self.test_
        train.drop('id', axis=1, inplace=True)
        test.drop('id', axis=1, inplace=True)
        from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
        dtrain = lgb.Dataset(train, label=y)
        def fix_params(params):
            for p in ['min_data_in_leaf', 'num_leaves', 'max_bin' ]:
                params[p] = int(params[p])
            params['num_leaves'] = max(params['num_leaves'], 2)
        def step_xgb(params):
            fix_params(params)
            cv = lgb.cv(params, dtrain,
                        num_boost_round=10000,
                        early_stopping_rounds=50,
                        nfold=6,
                        seed=params['seed'])
            rounds = np.argmin(cv['binary_logloss-mean'])
            score = np.min(cv['binary_logloss-mean'])
            print(cname, score, rounds, params, self.now())
            return dict(loss=score, status=STATUS_OK)
        space_lgb = dict(
                bagging_fraction = hp.quniform('bagging_fraction', 0.5, 1, 0.001),
                colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05),
                feature_fraction = hp.quniform('feature_fraction', 0.5, 1, 0.001),
                lambda_l1 = hp.choice('lambda_l1', [0, hp.loguniform('lambda_l1_positive', -16, 2)]),
                lambda_l2 = hp.choice('lambda_l2', [0, hp.loguniform('lambda_l2_positive', -16, 2)]),
                learning_rate = hp.loguniform('learning_rate', -7, 0),
                max_bin = hp.qloguniform('max_bin', 0, 20, 1),
                max_depth = hp.choice('max_depth', range(2, 9)),
                min_child_weight = hp.quniform('min_child_weight', 1, 6, 1),
                min_data_in_leaf = hp.qloguniform('min_data_in_leaf', 0, 6, 1),
                min_sum_hessian_in_leaf = hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
                num_leaves = hp.qloguniform('num_leaves', 2, 7, 1),
                reg_alpha = hp.quniform('reg_alpha', 0, 1, 0.001),
                subsample = hp.quniform('subsample', 0.6, 1, 0.05),

                bagging_freq = 1,
                objective = 'binary',
                metric = 'binary_logloss',
                seed = 1,
                #silent = 1,
            )
        trs = self.load('lightgbm_trials')
        if trs == None or self.debug_:
            tr = Trials()
        else:
            tr, _ = trs
        if len(tr.trials) > 0:
            print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_lgb, tr.argmin))
            best = tr.argmin
        while len(tr.trials) < self.max_ho_trials_:
            print(len(tr.trials), end=' ')
            #best = fmin(step_xgb, space_lgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr)
            best = fmin(step_xgb, space_lgb, algo=partial(tpe.suggest, n_startup_jobs=1), max_evals=len(tr.trials) + 1, trials = tr)
            self.save('lightgbm_trials', (tr, space_lgb))
        lgb_params = space_eval(space_lgb, best)
        fix_params(lgb_params)
        print(lgb_params)

        N_splits = self.num_splits_
        N_seeds = self.num_seeds_

        v, z = self.v_, self.z_
        skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True)
        cv = []
        for s in range(N_seeds):
            scores = []
            cname2 = cname + str(s)
            v[cname2], z[cname2] = 0, 0
            lgb_params['seed'] = s + self.base_seed_
            for n, (itrain, ival) in enumerate(skf.split(train, y)):
                dtrain = lgb.Dataset(train.ix[itrain], y[itrain])
                dvalid = lgb.Dataset(train.ix[ival], y[ival])
                clf = lgb.train(lgb_params, dtrain,
                                num_boost_round=10000,
                                valid_sets=[dtrain, dvalid],
                                valid_names=['train', 'valid'],
                                early_stopping_rounds=100, verbose_eval=False)

                p = clf.predict(train.ix[ival])
                v.loc[ival, cname2] += p
                score = metrics.log_loss(y[ival], p)
                z[cname2]  += clf.predict(test)
                print(cname, 'seed %d step %d of %d: '%(lgb_params['seed'], n+1, skf.n_splits), score, self.now())
                scores.append(score)
            z[cname2] /= N_splits
            cv.append(np.mean(scores))
            print('seed %d loss: '%(lgb_params['seed']), scores, np.mean(scores), np.std(scores))
            z['y'] = z[cname2]

        print('cv:', cv, np.mean(cv), np.std(cv))
        return cv, None
# TODO: this needs fixing


def hyperopt_obj_fn(args):
    ''' hyper-opt objective function '''
    num_hid, learn_rate, l1_decay, l2_decay, activs, scale_factor, init_method, num_epochs, rho = args

    # multilayer net parameters
    mln_params = {'d': d, 'k': k, 'num_hids': [num_hid], 'activs': activs,
                  'loss_terms': ['cross_entropy', 'regularization'], 'l2_decay': l2_decay, 'l1_decay': l1_decay}

    rmsprop_params = {'init_method': init_method, 'scale_factor': scale_factor, 'optim_type': 'minibatch',
                      'optim_method': 'RMSPROP', 'batch_size': 900, 'num_epochs': num_epochs, 'learn_rate': learn_rate, 'rho': rho}
    return compute_cv_loss(mln_params, rmsprop_params)

rmsprop_space = (
    hp.qloguniform('num_hid', log(10), log(1000), 1),
    hp.loguniform('learn_rate', log(1e-4), log(10)),
    hp.choice('l1_decay', [0, hp.loguniform('l1', log(1e-6), 2.)]),
    hp.choice('l2_decay', [0, hp.loguniform('l2', log(1e-6), 2.)]),
    hp.choice('activs', [['sigmoid', 'softmax'], ['reLU', 'softmax']]),
    hp.loguniform('scale_factor', log(1e-3), log(1)),
    hp.choice('init_method', ['gauss', 'fan-io']),
    hp.qloguniform('num_epochs', log(10), log(1e3), 1),
    hp.uniform('rho', 1e-2, 0.99)
)

best = fmin(hyperopt_obj_fn, rmsprop_space, algo=tpe.suggest, max_evals=100)
print best
Exemple #58
0
    'amp': 0.063,
    # 'tau_ref': 0.001,
    # 'tau_rc': 0.05,
    # 'alpha': 0.825,
    # 'tau_ref': hp.uniform('tau_ref', 0.001, 0.005),
    'tau_ref': 0.002,
    'tau_rc': hp.uniform('tau_rc', 0.01, 0.06),
    # 'alpha': hp.uniform('alpha', 0.1, 10.0),
    'alpha': 1.0,
    'sigma': 0.02,
    'noise': 10.,

    # learning rate params
    'epsW_schedule': hp.choice('epsW_schedule', ['linear', 'exp']),
    'epsW_base': hp.lognormal('epsW_base', np.log(1e-3), np.log(1e1)),
    'epsW_tgtFactor': hp.qloguniform('epsW_tgtFactor', np.log(1), np.log(1e4), 1),
    'epsB_schedule': hp.choice('epsB_schedule', ['linear', 'exp']),
    'epsB_base': hp.lognormal('epsB_base', np.log(1e-3), np.log(1e1)),
    'epsB_tgtFactor': hp.qloguniform('epsB_tgtFactor', np.log(1), np.log(1e4), 1),
    'momW': hp.uniform('momW', 0.001, 0.999),
    'momB': hp.uniform('momB', 0.001, 0.999),

    # initial weight params
    'initW1': hp.lognormal('initW1', np.log(1e-4), np.log(1e2)),
    'initW2': hp.lognormal('initW2', np.log(1e-2), np.log(1e1)),
    'initW3': hp.lognormal('initW3', np.log(1e-2), np.log(1e1)),
    'initW4': hp.lognormal('initW4', np.log(1e-2), np.log(1e1)),
    'initW5': hp.lognormal('initW5', np.log(1e-2), np.log(1e1)),

    # weight costs
    'wc1': hp.lognormal('wc1', np.log(1e-8), np.log(1e3)),
## sklearn
skl_random_seed = config.RANDOM_SEED
skl_n_jobs = config.NUM_CORES
skl_n_estimators_min = 100
skl_n_estimators_max = 1000
skl_n_estimators_step = 10


# ---------------------------- XGBoost ---------------------------------------
## regression with linear booster
param_space_reg_xgb_linear = {
    "booster": "gblinear",
    "objective": 'reg:linear',
    "base_score": config.BASE_SCORE,
    "n_estimators" : hp.quniform("n_estimators", xgb_n_estimators_min, xgb_n_estimators_max, xgb_n_estimators_step),
    "learning_rate" : hp.qloguniform("learning_rate", np.log(0.002), np.log(0.1), 0.002),
    "reg_alpha" : hp.loguniform("reg_alpha", np.log(1e-10), np.log(1e1)),
    "reg_lambda" : hp.loguniform("reg_lambda", np.log(1e-10), np.log(1e1)),
    "reg_lambda_bias" : hp.quniform("reg_lambda_bias", 0, 3, 0.1),
    "nthread": xgb_nthread,
    "seed": xgb_random_seed,
}

## regression with tree booster
param_space_reg_xgb_tree = {
    "booster": "gbtree",
    "objective": 'reg:linear',
    "base_score": config.BASE_SCORE,
    "n_estimators" : hp.quniform("n_estimators", xgb_n_estimators_min, xgb_n_estimators_max, xgb_n_estimators_step),
    "learning_rate" : hp.qloguniform("learning_rate", np.log(0.002), np.log(0.1), 0.002),
    "gamma": hp.loguniform("gamma", np.log(1e-10), np.log(1e1)),
Exemple #60
0
    assert lower < upper, "Lower bound of uniform greater than upper bound"
    return hp.loguniform(name, lower, upper)


def qloguniform(name, (lower, upper, q)):
    """
    Function to create hyperopt qloguniform variable
    Input
    ------------------
    name - Variable name
    (lower, upper, q) - Tuple of bounds and q value.

    Distribution looks like round(exp(uniform(lower, upper)) / q) * q
    """
    assert lower < upper, "Lower bound of uniform greater than upper bound"
    return hp.qloguniform(name, lower, upper, q)


def normal(name, (mu, sigma)):
    """
    Function to create hyperopt normal variable
    Input
    ------------------
    name - Variable name
    (mu, sigma) - Tuple of mean and standard deviation.
    """
    return hp.normal(name, mu, sigma)


def qnormal(name, (mu, sigma, q)):
    """