def __init__(self):
self.search_space = {
'learning_rate':
hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
'L1_flag':
hp.choice('L1_flag', [True, False]),
'hidden_size':
scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(256),
1)),
'batch_size':
scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),
1)),
'lmbda':
hp.loguniform('lmbda', np.log(0.00001), np.log(0.001)),
'optimizer':
hp.choice('optimizer', ["adam", "sgd", 'rms']),
'margin':
hp.uniform('margin', 0.5, 8.0),
'distance_measure':
hp.choice('distance_measure',
["kl_divergence", "expected_likelihood"]),
'cmax':
hp.loguniform('cmax', np.log(0.05), np.log(0.2)),
'cmin':
hp.loguniform('cmin', np.log(1), np.log(5)),
'epochs':
hp.choice('epochs', [10]) # always choose 10 training epochs.
}
class skLassoCVReg(_LassoCVReg, BaseWrapperReg, metaclass=MetaBaseWrapperRegWithABC):
def __init__(self, eps=1e-3, n_alphas=100, alphas=None, fit_intercept=True, normalize=False, precompute='auto',
max_iter=1000, tol=1e-4, copy_X=True, cv=None, verbose=False, n_jobs=1, positive=False,
random_state=None, selection='cyclic'):
n_alphas = int(n_alphas)
_LassoCVReg.__init__(
self, eps, n_alphas, alphas, fit_intercept, normalize, precompute, max_iter, tol, copy_X, cv,
verbose, n_jobs, positive, random_state, selection)
BaseWrapperReg.__init__(self)
HyperOpt_space = {
'n_alphas': hp.qloguniform('n_alphas', 2, 5, 1),
'eps': hp.loguniform('eps', -5, 0),
'max_iter': hp.qloguniform('max_iter', 4, 8, 1),
'tol': hp.loguniform('tol', -8, 0),
}
tuning_grid = {
'eps': 1e-3,
'n_alphas': 100,
'alphas': None,
'fit_intercept': True,
'normalize': False,
'precompute': 'auto',
'max_iter': 1000,
'tol': 1e-4,
'copy_X': True,
'cv': None,
'verbose': False,
'n_jobs': 1,
'positive': False,
'random_state': None,
'selection': 'cyclic',
}
def get_default_parameter_space():
"""
Return:
dict of DistributionWrappers
"""
return {
'n_estimators':
scope.int(hp.quniform('n_estimators', 5, 10, 5)),
'learning_rate':
hp.loguniform('learning_rate', -7, 0),
'num_leaves':
scope.int(hp.qloguniform('num_leaves', 1, 7, 1)),
'feature_fraction':
hp.uniform('feature_fraction', 0.5, 1),
'bagging_fraction':
hp.uniform('bagging_fraction', 0.5, 1),
'min_data_in_leaf':
scope.int(hp.qloguniform('min_data_in_leaf', 0, 6, 1)),
'min_sum_hessian_in_leaf':
hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
'lambda_l1':
hp.loguniform('lambda_l1', -16, 2),
'lambda_l2':
hp.loguniform('lambda_l2', -16, 2),
}
def _config_tuning_space(tuning_space_raw):
if tuning_space_raw is None:
return None
hyper_obj = {}
if "learning_rate" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"learning_rate": hp.loguniform('learning_rate', np.log(tuning_space_raw['learning_rate']['min']), np.log(tuning_space_raw['learning_rate']['max']))}}
if "hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"hidden_size": scope.int(hp.qloguniform('hidden_size', np.log(tuning_space_raw['hidden_size']['min']), np.log(tuning_space_raw['hidden_size']['max']), 1))}}
if "ent_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"ent_hidden_size": scope.int(hp.qloguniform("ent_hidden_size", np.log(tuning_space_raw['ent_hidden_size']['min']), np.log(tuning_space_raw['ent_hidden_size']['max']), 1))}}
if "rel_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"rel_hidden_size": scope.int(hp.qloguniform("rel_hidden_size", np.log(tuning_space_raw['rel_hidden_size']['min']), np.log(tuning_space_raw['rel_hidden_size']['max']), 1))}}
if "batch_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"batch_size": scope.int(hp.qloguniform("batch_size", np.log(tuning_space_raw['batch_size']['min']), np.log(tuning_space_raw['batch_size']['max']), 1))}}
if "margin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"margin": hp.uniform("margin", tuning_space_raw["margin"]["min"], tuning_space_raw["margin"]["max"])}}
if "lmbda" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"lmbda": hp.loguniform('lmbda', np.log(tuning_space_raw["lmbda"]["min"]), np.log(tuning_space_raw["lmbda"]["max"]))}}
if "distance_measure" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"distance_measure": hp.choice('distance_measure', tuning_space_raw["distance_measure"])}}
if "cmax" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmax": hp.loguniform('cmax', np.log(tuning_space_raw["cmax"]["min"]), np.log(tuning_space_raw["cmax"]["max"]))}}
if "cmin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmin": hp.loguniform('cmin', np.log(tuning_space_raw["cmin"]["min"]), np.log(tuning_space_raw["cmin"]["max"]))}}
if "optimizer" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"optimizer": hp.choice("optimizer", tuning_space_raw["optimizer"])}}
if "bilinear" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"bilinear": hp.choice('bilinear', tuning_space_raw["bilinear"])}}
if "epochs" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"epochs": hp.choice("epochs", tuning_space_raw["epochs"])}}
return hyper_obj
class skLassoLarsCVReg(_LassoLarsCV, BaseWrapperReg, metaclass=MetaBaseWrapperRegWithABC):
def __init__(self, fit_intercept=True, verbose=False, max_iter=500, normalize=True, precompute='auto', cv=None,
max_n_alphas=1000, n_jobs=1, eps=np.finfo(np.float).eps, copy_X=True, positive=False):
_LassoLarsCV.__init__(
self, fit_intercept, verbose, max_iter, normalize, precompute, cv, max_n_alphas, n_jobs, eps, copy_X,
positive)
BaseWrapperReg.__init__(self)
HyperOpt_space = {
'max_n_alphas': hp.qloguniform('max_n_alphas', 2, 5, 1),
'eps': hp.loguniform('eps', -5, 0),
'max_iter': hp.qloguniform('max_iter', 4, 8, 1),
}
tuning_grid = {
'fit_intercept': True,
'verbose': False,
'max_iter': 500,
'normalize': True,
'precompute': 'auto',
'cv': None,
'max_n_alphas': 1000,
'n_jobs': 1,
'eps': np.finfo(np.float).eps,
'copy_X': True,
'positive': False,
}
def bernoulli_rbm_hp_space(n_components=None,
learning_rate=None,
batch_size=None,
n_iter=None,
verbose=False,
random_state=None):
rval = dict(
n_components=scope.int(
hp.qloguniform(
'n_components', low=np.log(0.51), high=np.log(999.5), q=1.0))
if n_components is None else n_components,
learning_rate=hp.lognormal(
'learning_rate',
np.log(0.01),
np.log(10),
) if learning_rate is None else learning_rate,
batch_size=scope.int(
hp.qloguniform(
'.batch_size',
np.log(1),
np.log(100),
q=1,
)) if batch_size is None else batch_size,
n_iter=scope.int(
hp.qloguniform(
'n_iter',
np.log(1),
np.log(1000), # -- max sweeps over the *whole* train set
q=1,
)) if n_iter is None else n_iter,
verbose=verbose,
random_state=_random_state('rstate', random_state),
)
return rval
def __init__(
self,
learning_task,
n_estimators=100,
max_hyperopt_evals=50,
categorical_features=None,
early_stopping_round=5,
random_state=0,
use_gpu=False,
output_folder_path="./",
):
Experiment.__init__(
self,
learning_task,
"lgb",
n_estimators,
max_hyperopt_evals,
categorical_features,
early_stopping_round,
random_state,
output_folder_path,
use_gpu,
)
# hard-coded params search space here
self.space = {
"learning_rate":
hp.loguniform("learning_rate", -7, 0),
"num_leaves":
hp.qloguniform("num_leaves", 0, 7, 1),
"feature_fraction":
hp.uniform("feature_fraction", 0.5, 1),
"bagging_fraction":
hp.uniform("bagging_fraction", 0.5, 1),
"min_data_in_leaf":
hp.qloguniform("min_data_in_leaf", 0, 6, 1),
"min_sum_hessian_in_leaf":
hp.loguniform("min_sum_hessian_in_leaf", -16, 5),
"lambda_l1":
hp.choice("lambda_l1",
[0, hp.loguniform("lambda_l1_positive", -16, 2)]),
"lambda_l2":
hp.choice("lambda_l2",
[0, hp.loguniform("lambda_l2_positive", -16, 2)]),
}
# hard-coded default params here
self.default_params = {
"learning_rate": 0.1,
"num_leaves": 127,
"feature_fraction": 1.0,
"bagging_fraction": 1.0,
"min_data_in_leaf": 20,
"min_sum_hessian_in_leaf": 1e-3,
"lambda_l1": 0,
"lambda_l2": 0,
}
self.default_params = self.preprocess_params(self.default_params)
self.title = "LightGBM"
def hyper_GAN_3_5(P_args):
P_search = P_args.copy()
P_search.set_keys(
name = 'Hyper_GAN_3.5',
dataset = 'SHL_ext',
epochs = 100,
runs = 5,
batch_size = 512,
FX_num = 150,
FX_sel = 'all',
cross_val = 'combined',
sample_no = 512,
undersampling = False,
oversampling = False,
C_aco_func = 'gumbel',
D_optim = 'SGD',
G_optim = 'SGD',
)
param_space={
'GD_ratio' : hp.uniform('GD_ratio', 0, 0.9),
'CLR' : hp.loguniform('CLR', np.log(0.00001), np.log(0.1)),
'CB1' : hp.loguniform('CB1', np.log(0.001), np.log(0.99)),
'C_ac_func' : hp.choice('C_ac_func',['relu','leaky','leaky20','sig']),
'C_hidden' : scope.int(hp.qloguniform('C_hidden', np.log(16), np.log(4096), q=1)),
'C_hidden_no' : scope.int(hp.quniform('C_hidden_no', 1, 8, q=1)),
'C_optim' : hp.choice('C_optim',['AdamW','SGD']),
'C_tau' : hp.loguniform('C_tau', np.log(0.01), np.log(10.)),
'GLR' : hp.loguniform('GLR', np.log(0.00001), np.log(0.1)),
'GB1' : hp.loguniform('GB1', np.log(0.001), np.log(0.99)),
'DLR' : hp.loguniform('DLR', np.log(0.00001), np.log(0.1)),
'DB1' : hp.loguniform('DB1', np.log(0.001), np.log(0.99)),
'G_ac_func' : hp.choice('G_ac_func',['relu','leaky','leaky20','sig']),
'G_hidden' : scope.int(hp.qloguniform('G_hidden', np.log(16), np.log(4096), q=1)),
'G_hidden_no' : scope.int(hp.quniform('G_hidden_no', 1, 8, q=1)),
'D_ac_func' : hp.choice('D_ac_func',['relu','leaky','leaky20','sig']),
'D_hidden' : scope.int(hp.qloguniform('D_hidden', np.log(16), np.log(4096), q=1)),
'D_hidden_no' : scope.int(hp.quniform('D_hidden_no', 1, 8, q=1)),
}
hyperopt_GAN(P_search,param_space,eval_step=5,max_evals=None,P_val=P_search.copy().set_keys(
sample_no = None,
undersampling = False,
oversampling = False,))
def __init__(self):
self.search_space = {
'learning_rate': hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
'hidden_size': scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(256),1)),
'batch_size': scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),1)),
'lmbda': hp.loguniform('lmbda', np.log(0.00001), np.log(0.001)),
'optimizer': hp.choice('optimizer', ["adam", "sgd", 'rms']),
'epochs': hp.choice('epochs', [10]) # always choose 10 training epochs.
}
def define_search_space(objective, starting_params):
if objective == "rmse":
prediction = starting_params["collaborative_params"][
"prediction_network_params"]
rmse_space = {
'lr':
hp.qlognormal("lr", np.log(prediction["lr"]),
0.5 * prediction["lr"], 0.05 * prediction["lr"]),
'epochs':
hp.quniform('epochs', prediction["epochs"] - 20,
prediction["epochs"] + 20, 5),
'kernel_l2':
hp.choice('kernel_l2', [
0.0,
hp.qloguniform('kernel_l2_choice', np.log(1e-9), np.log(1e-5),
5e-9)
]),
'batch_size':
hp.qloguniform('batch_size', np.log(512), np.log(1023), 512),
# 'batch_size': hp.choice('batch_size', [512]),
'conv_depth':
hp.quniform('conv_depth', 1, prediction["conv_depth"] + 1, 1),
'gaussian_noise':
hp.qlognormal('gaussian_noise',
np.log(prediction["gaussian_noise"]),
0.5 * prediction["gaussian_noise"], 0.005),
'network_depth':
hp.quniform('network_depth', 1, prediction['network_depth'] + 1,
1),
'n_dims':
hp.quniform('n_dims', starting_params["n_dims"] - 32,
starting_params["n_dims"] + 64, 16),
}
return rmse_space
if objective == "ndcg":
embedding = starting_params["collaborative_params"]["user_item_params"]
ndcg_space = {
'gcn_lr':
hp.qlognormal("gcn_lr", np.log(embedding["gcn_lr"]),
0.5 * embedding["gcn_lr"],
0.05 * embedding["gcn_lr"]),
'gcn_epochs':
hp.quniform('gcn_epochs', embedding["gcn_epochs"],
embedding["gcn_epochs"] + 20, 5),
'gaussian_noise':
hp.qlognormal('gaussian_noise',
np.log(embedding["gaussian_noise"]),
1.0 * embedding["gaussian_noise"], 0.005),
'margin':
hp.quniform('margin', 0.8, 1.8, 0.2),
'n_dims':
hp.quniform('n_dims', starting_params["n_dims"],
starting_params["n_dims"] + 96, 16),
}
return ndcg_space
def __call__(self):
print_exp(self.exp_name)
if self.only_q:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-4)),
q=(18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
30),
epochs=(250, ),
batch=(32, ))
elif self.only_batch:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-3)),
q=(23, ),
epochs=(250, ),
batch=(16, 32, 50, 100, 128, 200))
elif self.only_epochs:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-4)),
q=(23, ),
epochs=(100, 120, 150, 170, 200, 250, 300, 400,
500),
batch=(32, ))
elif self.only_lr:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-3)),
q=(23, ),
epochs=(250, ),
batch=(32, ))
else:
pv = ParamValues(
lr=hp.loguniform("lr", np.log(1e-4), np.log(1e-3)),
q=scope.int(hp.qloguniform("q", np.log(10), np.log(100), 1)),
#q=(18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,35),
epochs=scope.int(
hp.qloguniform("epochs", np.log(10), np.log(500), 10)),
#epochs=(100, 125, 150, 175, 200, 250, 300, 400, 500, 700),
batch=(4, 8, 16, 32, 64, 128, 256))
hyper_params = {
"data": {},
"model": {
"lr": pv.lr,
#"encoding_dim": hp.choice("encoding_dim", pv.q), ##
"encoding_dim": pv.q,
},
"fit": {
#"epochs": hp.choice("epochs", pv.epochs), #
"epochs": pv.epochs,
"batch_size": hp.choice("batch_size", pv.batch)
}
}
run = RunFN(self.metric, hyper_params, pv, self.data_path, self.sep,
self.run_on_mongodb, self.start_mongodb, self.db_name,
self.exp_name, self.ip, self.port, self.nr_of_trials,
self.nr_of_workers)
run()
def get_hp_space():
space = (hp.qloguniform('n_hidden', log(100), log(5000), 10),
hp.qloguniform('temporal_order', log(1), log(30), 1),
hp.loguniform('learning_rate', log(0.0001), log(1)),
hp.quniform('batch_size', 10, 500, 10)
# hp.choice('activation_func', ['tanh', 'sigmoid']),
# hp.choice('sampling_positive', ['true', 'false'])
# gibbs_sampling_step_test ???
)
return space
def __init__(self):
self.search_space = {
'learning_rate': hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
'L1_flag': hp.choice('L1_flag', [True, False]),
'hidden_size': scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(128),1)),
'batch_size': scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),1)),
'margin': hp.uniform('margin', 0.0, 2.0),
'optimizer': hp.choice('optimizer', ["adam", "sgd", 'rms']),
'epochs': hp.choice('epochs', [10]) # always choose 10 training epochs.
}
def tpe_configspace(self):
from hyperopt import hp
import numpy as np
space = {
'l_rate': hp.loguniform('l_rate', np.log(1e-6), np.log(1e-1)),
'burn_in': hp.uniform('burn_in', 0, .8),
'n_units_1': hp.qloguniform('n_units_1', np.log(16), np.log(512), 1),
'n_units_2': hp.qloguniform('n_units_2', np.log(16), np.log(512), 1),
'mdecay': hp.uniform('mdecay', 0, 1)
}
return(space)
class GradientBoostingModel(TreeBasedModel):
@staticmethod
def build_estimator(args, train_data=None):
return GradientBoostingRegressor(random_state=RANDOM_STATE,
presort=True,
**args)
loss_alpha = hp.choice('loss_alpha',
[('ls', 0.9), ('lad', 0.9),
('huber', hp.uniform('gbr_alpha', 0.85, 0.95)),
('quantile', 0.5)])
hp_space = {
'n_estimators':
scope.int(
hp.qloguniform('n_estimators', np.log(10.5), np.log(1000.5), 1)),
'learning_rate':
hp.lognormal('learning_rate', np.log(0.01), np.log(10.0)),
'criterion':
hp.choice('criterion', ['mse', 'friedman_mse', 'mae']),
'max_depth':
hp.pchoice('max_depth', [(0.2, 2), (0.5, 3), (0.2, 4), (0.1, 5)]),
'min_samples_leaf':
hp.choice(
'min_samples_leaf_enabled',
[
1, # most common choice.
scope.int(
hp.qloguniform('min_samples_leaf', np.log(1.5),
np.log(50.5), 1))
]),
'subsample':
hp.pchoice(
'subsample_enabled',
[
(0.2, 1.0), # default choice.
(0.8, hp.uniform('subsample', 0.5, 1.0)
) # stochastic grad boosting.
]),
'max_features':
hp.pchoice(
'max_features_str',
[
(0.1, 'sqrt'), # most common choice.
(0.2, 'log2'), # less common choice.
(0.1, None), # all features, less common choice.
(0.6, hp.uniform('max_features_str_frac', 0., 1.))
]),
'loss':
loss_alpha[0],
'alpha':
loss_alpha[1]
}
def define_search_space(objective, starting_params):
prediction = starting_params["collaborative_params"][
"prediction_network_params"]
space = {
'lr':
hp.qlognormal("lr", np.log(prediction["lr"]), 0.5 * prediction["lr"],
0.05 * prediction["lr"]),
'epochs':
hp.quniform('epochs', prediction["epochs"] - 10,
prediction["epochs"] + 20, 5),
'kernel_l2':
hp.choice('kernel_l2', [
0.0,
hp.qloguniform('kernel_l2_choice', np.log(1e-9), np.log(1e-5),
5e-9)
]),
'batch_size':
hp.qloguniform('batch_size', np.log(1024), np.log(4096), 1024),
'conv_depth':
hp.quniform('conv_depth', 1, prediction["conv_depth"] + 2, 1),
'gcn_layers':
hp.quniform('gcn_layers', 1, prediction["gcn_layers"] + 1, 1),
'ncf_layers':
hp.quniform('ncf_layers', 1, prediction["ncf_layers"] + 1, 1),
'ps_proportion':
hp.choice('ps_proportion', [
0.0,
hp.qloguniform('ps_proportion_choice', np.log(0.1),
np.log(prediction["ps_proportion"] + 1.0), 0.05)
]),
'ns_proportion':
hp.quniform('ns_proportion', 0.0, prediction["ns_proportion"] + 2.0,
0.1),
'nsh':
hp.quniform('nsh', 0.0, prediction["nsh"] + 2.0, 0.1),
# 'gaussian_noise': hp.qlognormal('gaussian_noise', np.log(prediction["gaussian_noise"]),
# 0.5 * prediction["gaussian_noise"], 0.005),
'gaussian_noise':
hp.choice('gaussian_noise', [
0.0,
hp.qloguniform('gaussian_noise_choice', np.log(1e-3), np.log(0.5),
1e-3)
]),
'margin':
hp.choice('margin', [
0.0,
hp.qloguniform('margin_choice', np.log(1e-4), np.log(0.05), 5e-4)
]),
'n_dims':
hp.quniform('n_dims', starting_params["n_dims"] - 16,
starting_params["n_dims"] + 64, 16),
}
return space
class GradientBoostingModel(TreeBasedModel):
@staticmethod
def build_estimator(args, train_data=None):
return GradientBoostingRegressor(
random_state=RANDOM_STATE, presort=True, **args
)
loss_alpha = hp.choice(
"loss_alpha",
[
("ls", 0.9),
("lad", 0.9),
("huber", hp.uniform("gbr_alpha", 0.85, 0.95)),
("quantile", 0.5),
],
)
hp_space = {
"n_estimators": scope.int(
hp.qloguniform("n_estimators", np.log(10.5), np.log(1000.5), 1)
),
"learning_rate": hp.lognormal("learning_rate", np.log(0.01), np.log(10.0)),
"criterion": hp.choice("criterion", ["mse", "friedman_mse", "mae"]),
"max_depth": hp.pchoice("max_depth", [(0.2, 2), (0.5, 3), (0.2, 4), (0.1, 5)]),
"min_samples_leaf": hp.choice(
"min_samples_leaf_enabled",
[
1, # most common choice.
scope.int(
hp.qloguniform("min_samples_leaf", np.log(1.5), np.log(50.5), 1)
),
],
),
"subsample": hp.pchoice(
"subsample_enabled",
[
(0.2, 1.0), # default choice.
(0.8, hp.uniform("subsample", 0.5, 1.0)), # stochastic grad boosting.
],
),
"max_features": hp.pchoice(
"max_features_str",
[
(0.1, "sqrt"), # most common choice.
(0.2, "log2"), # less common choice.
(0.1, None), # all features, less common choice.
(0.6, hp.uniform("max_features_str_frac", 0.0, 1.0)),
],
),
"loss": loss_alpha[0],
"alpha": loss_alpha[1],
}
def optimize(max_evals, max_ppn):
space = hp.choice('schedule', [
('static', hp.qloguniform('chunk_s', 2, 11, 10),
hp.randint('ppn_s', max_ppn)),
('dynamic', hp.qloguniform('chunk_d', 2, 11, 10),
hp.randint('ppn_d', max_ppn)),
('guided', hp.qloguniform('chunk_g', 2, 11, 10),
hp.randint('ppn_g', max_ppn)),
])
trials = Trials()
best = fmin(function, space=space, algo=tpe.suggest,
max_evals=max_evals, trials=trials)
return best, trials
def get_hp_space():
space = (hp.qloguniform('n_hidden', log(100), log(5000), 10),
hp.qloguniform('n_hidden_recurrent', log(100), log(5000), 1),
hp.qloguniform('temporal_order', log(10), log(100), 10),
hp.loguniform('learning_rate_RBM', log(0.0001), log(1)),
hp.loguniform('learning_rate_RNN', log(0.0001), log(1)),
hp.qloguniform('K', log(2), log(20), 1)
# Different learninf rate for RBM and RNN
# hp.choice('activation_func', ['tanh', 'sigmoid']),
# hp.choice('sampling_positive', ['true', 'false'])
# gibbs_sampling_step_test ???
)
return space
def __init__(self, train_fname, test_fname, feat_types_fname, cat_count_fname, work_dir, task,\
max_evals, n_cv_folds, num_trees, num_threads, time_log_fname, train_query_fname=None, test_query_fname=None):
super().__init__(train_fname, test_fname, feat_types_fname, work_dir, task,\
max_evals, n_cv_folds, time_log_fname, train_query_fname=train_query_fname, test_query_fname=test_query_fname)
print("using LightGBM in " + str(lgb.__file__))
if self.task == "ranking":
self.objective = "lambdarank"
else:
self.objective = self.task
self.num_threads = num_threads
self.num_trees = num_trees
param_dict = {
'learning_rate':
hp.loguniform('learning_rate', -7, 0),
'num_leaves':
hp.qloguniform('num_leaves', 1, 7, 1),
'max_depth':
0,
'max_bin':
2**(hp.randint('max_bin_minus_4', 7) + 4),
'feature_fraction':
hp.uniform('feature_fraction', 0.5, 1),
'bagging_fraction':
hp.uniform('bagging_fraction', 0.5, 1),
'min_data_in_leaf':
hp.qloguniform('min_data_in_leaf', 0, 6, 1),
'min_sum_hessian_in_leaf':
hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
'lambda_l1':
hp.choice('lambda_l1',
[0, hp.loguniform('lambda_l1_log', -16, 2)]),
'lambda_l2':
hp.choice('lambda_l2',
[0, hp.loguniform('lambda_l2_log', -16, 2)]),
"tree_learner":
"serial"
}
if len(self.categorical_features) > 0:
max_cat_count = 0
with open(cat_count_fname, "r") as in_file:
for line in in_file:
_, cat_count = line.strip().split("\t")
cat_count = int(cat_count)
if cat_count > max_cat_count:
max_cat_count = cat_count
param_dict["cat_smooth"] = hp.qloguniform('cat_smooth', 0, 8, 1)
param_dict["cat_l2"] = hp.qloguniform('cat_l2', 0, 6, 1)
param_dict["max_cat_threshold"] = hp.randint(
'max_cat_threshold', max_cat_count // 2) + 1
self.param_space = param_dict
self.metric = None
def set_multilayer_dropout_space(self):
''' defines a hyperspace for a "modern" neural networks: at least two layers with dropout + reLU '''
# Force at least 2 layers, cuz we're modern
min_layers = 2
max_layers = 3
# sets up the neural network
nnets = [None] * (max_layers - min_layers + 1)
for i, num_layers in enumerate(range(min_layers, max_layers + 1)):
num_hids = [None] * num_layers
for j in range(num_layers):
num_hids[j] = hp.qloguniform(
'num_hid_%i%i' % (i, j), log(100), log(1000), 1)
nnets[i] = num_hids
default_mln_model_params = {
'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy', 'dropout']}
search_mln_model_params = {
'arch': hp.choice('arch', nnets),
'input_p': hp.uniform('ip', 0, 1),
'hidden_p': hp.uniform('hp', 0, 1),
'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
default_mln_optim_params = {
'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}
search_mln_optim_params = {
'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# merge the default and search spaces
mln_model_params = self.merge_default_search(
default_mln_model_params, search_mln_model_params)
mln_optim_params = self.merge_default_search(
default_mln_optim_params, search_mln_optim_params)
# define the hyperparamater space to search
hyperspace = {'mln_model_params': mln_model_params,
'mln_optim_params': mln_optim_params}
return hyperspace
def _config_tuning_space(tuning_space_raw):
if tuning_space_raw is None:
return None
hyper_obj = {}
if "learning_rate" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"learning_rate": hp.loguniform('learning_rate', np.log(tuning_space_raw['learning_rate']['min']), np.log(tuning_space_raw['learning_rate']['max']))}}
if "hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"hidden_size": scope.int(hp.qloguniform('hidden_size', np.log(tuning_space_raw['hidden_size']['min']), np.log(tuning_space_raw['hidden_size']['max']), 1))}}
if "ent_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"ent_hidden_size": scope.int(hp.qloguniform("ent_hidden_size", np.log(tuning_space_raw['ent_hidden_size']['min']), np.log(tuning_space_raw['ent_hidden_size']['max']), 1))}}
if "rel_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"rel_hidden_size": scope.int(hp.qloguniform("rel_hidden_size", np.log(tuning_space_raw['rel_hidden_size']['min']), np.log(tuning_space_raw['rel_hidden_size']['max']), 1))}}
if "batch_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"batch_size": scope.int(hp.qloguniform("batch_size", np.log(tuning_space_raw['batch_size']['min']), np.log(tuning_space_raw['batch_size']['max']), 1))}}
if "margin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"margin": hp.uniform("margin", tuning_space_raw["margin"]["min"], tuning_space_raw["margin"]["max"])}}
if "lmbda" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"lmbda": hp.loguniform('lmbda', np.log(tuning_space_raw["lmbda"]["min"]), np.log(tuning_space_raw["lmbda"]["max"]))}}
if "distance_measure" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"distance_measure": hp.choice('distance_measure', tuning_space_raw["distance_measure"])}}
if "cmax" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmax": hp.loguniform('cmax', np.log(tuning_space_raw["cmax"]["min"]), np.log(tuning_space_raw["cmax"]["max"]))}}
if "cmin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmin": hp.loguniform('cmin', np.log(tuning_space_raw["cmin"]["min"]), np.log(tuning_space_raw["cmin"]["max"]))}}
if "optimizer" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"optimizer": hp.choice("optimizer", tuning_space_raw["optimizer"])}}
if "bilinear" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"bilinear": hp.choice('bilinear', tuning_space_raw["bilinear"])}}
if "epochs" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"epochs": hp.choice("epochs", tuning_space_raw["epochs"])}}
if "feature_map_dropout" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"feature_map_dropout": hp.choice('feature_map_dropout', tuning_space_raw["feature_map_dropout"])}}
if "input_dropout" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"input_dropout": hp.choice('input_dropout', tuning_space_raw["input_dropout"])}}
if "hidden_dropout" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"hidden_dropout": hp.choice('hidden_dropout', tuning_space_raw["hidden_dropout"])}}
if "use_bias" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"use_bias": hp.choice('use_bias', tuning_space_raw["use_bias"])}}
if "label_smoothing" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"label_smoothing": hp.choice('label_smoothing', tuning_space_raw["label_smoothing"])}}
if "lr_decay" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"lr_decay": hp.choice('lr_decay', tuning_space_raw["lr_decay"])}}
if "l1_flag" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"l1_flag": hp.choice('l1_flag', tuning_space_raw["l1_flag"])}}
if "sampling" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"sampling": hp.choice('sampling', tuning_space_raw["sampling"])}}
return hyper_obj
def __init__(self,
learning_task,
n_estimators=5000,
max_hyperopt_evals=50,
counters_sort_col=None,
holdout_size=0,
train_path=None,
test_path=None,
cd_path=None,
num_class=2,
output_folder_path='./'):
Experiment.__init__(self, learning_task, 'lgb', n_estimators,
max_hyperopt_evals, True, counters_sort_col,
holdout_size, train_path, test_path, cd_path,
num_class, output_folder_path)
self.space = {
'learning_rate':
hp.loguniform('learning_rate', -7, 0),
'num_leaves':
hp.qloguniform('num_leaves', 0, 7, 1),
'feature_fraction':
hp.uniform('feature_fraction', 0.5, 1),
'bagging_fraction':
hp.uniform('bagging_fraction', 0.5, 1),
'min_data_in_leaf':
hp.qloguniform('min_data_in_leaf', 0, 6, 1),
'min_sum_hessian_in_leaf':
hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
'lambda_l1':
hp.choice('lambda_l1',
[0, hp.loguniform('lambda_l1_positive', -16, 2)]),
'lambda_l2':
hp.choice('lambda_l2',
[0, hp.loguniform('lambda_l2_positive', -16, 2)]),
}
self.default_params = {
'learning_rate': 0.1,
'num_leaves': 127,
'feature_fraction': 1.0,
'bagging_fraction': 1.0,
'min_data_in_leaf': 100,
'min_sum_hessian_in_leaf': 10,
'lambda_l1': 0,
'lambda_l2': 0
}
self.default_params = self.preprocess_params(self.default_params)
self.title = 'LightGBM'
def tpe_configspace(self):
import numpy as np
from hyperopt import hp
space = {
'learning_rate': hp.loguniform('learning_rate', np.log(1e-7), np.log(1e-1)),
'batch_size': hp.qloguniform('batch_size', np.log(8), np.log(256), 1),
'n_units_1': hp.qloguniform('n_units_1', np.log(8), np.log(128), 1),
'n_units_2': hp.qloguniform('n_units_2', np.log(8), np.log(128), 1),
'discount': hp.uniform('discount', 0, 1),
'likelihood_ratio_clipping': hp.uniform('likelihood_ratio_clipping', 0, 1),
'entropy_regularization': hp.uniform('entropy_regularization', 0, 1)
}
return(space)
def __init__(self, train_fname, test_fname, feat_types_fname, work_dir, task, max_evals,\
n_cv_folds, num_trees, num_threads, time_log_fname, train_query_fname=None, test_query_fname=None):
super().__init__(train_fname, test_fname, feat_types_fname, work_dir, task, max_evals,\
n_cv_folds, time_log_fname, train_query_fname=train_query_fname, test_query_fname=test_query_fname)
print("using CatBoost in " + str(cat.__file__))
if self.task == "ranking":
self.objective = "YetiRank"
elif self.task == "binary":
self.objective = "Logloss"
elif self.task == "regression":
self.objective = "RMSE"
else:
raise NotImplementedError("Unknown task " + self.task)
self.num_threads = num_threads
self.num_trees = num_trees
param_dict = {
'learning_rate':
hp.loguniform('learning_rate', -7, 0),
'num_leaves':
hp.qloguniform('num_leaves', 1, 7, 1),
'border_count':
2**(hp.randint('max_bin_minus_4', 7) + 4),
'random_strength':
hp.randint('random_strength', 20) + 1,
'colsample_bylevel':
hp.uniform('feature_fraction', 0.5, 1),
'subsample':
hp.uniform('subsample', 0.5, 1),
'bagging_temperature':
hp.uniform('bagging_temperature', 0.0, 1.0),
'min_data_in_leaf':
hp.qloguniform('min_data_in_leaf', 0, 6, 1),
'l2_leaf_reg':
hp.choice('lambda_l2',
[0, hp.loguniform('lambda_l2_log', -16, 2)]),
"grow_policy":
"Lossguide",
"counter_calc_method":
"SkipTest",
"boosting_type":
"Plain"
}
if len(self.categorical_features) > 0:
param_dict["one_hot_max_size"] = hp.randint('one_hot_max_size', 26)
param_dict["max_ctr_complexity"] = hp.randint(
'max_ctr_complexity', 6) + 1
self.param_space = param_dict
self.column_description_fname = feat_types_fname
def get_simple_feature_adder_wrapper_params(
inner_model_params,
max_features=None,
pre_filter=None,
features_indexes_getter=None,
priority_getter=None,
name='feature_adder_common'
):
priority_getter = priority_getter if priority_getter is not None \
else get_priority_getter_params(get_full_name(name, 'priority_getter'))
pre_filter = pre_filter if pre_filter is not None \
else get_min_size_prefilter_params(get_full_name(name, 'pre_filter'))
features_indexes_getter = features_indexes_getter if features_indexes_getter is not None \
else get_index_getter_params(get_full_name(name, 'indexes_getter'))
max_features = max_features if max_features is not None \
else hp.qloguniform(
get_full_name(name, 'max_features'),
-1, 10, 1,
)
extender_strategy = hp.choice(
get_full_name(name, 'extender_strategy'),
(
scope.get_extender_strategy(
max_features=max_features,
priority_getter=priority_getter,
pre_filter=pre_filter,
simple_features_indexes_getter=features_indexes_getter,
),
scope.get_nothing_doing_extender_strategy(),
),
)
return scope.get_complex_features_adder_wrapper(
inner_model=inner_model_params,
extender_strategy=extender_strategy,
)
def passive_aggressive(name,
loss=None,
C=None,
fit_intercept=False,
n_iter=None,
n_jobs=1,
shuffle=True,
random_state=None,
verbose=False):
def _name(msg):
return '%s.%s_%s' % (name, 'sgd', msg)
rval = scope.sklearn_PassiveAggressiveClassifier(
loss=hp.choice(
_name('loss'),
['hinge', 'squared_hinge']) if loss is None else loss,
C=hp.lognormal(
_name('learning_rate'),
np.log(0.01),
np.log(10),
) if C is None else C,
fit_intercept=fit_intercept,
n_iter=scope.int(
hp.qloguniform(
_name('n_iter'),
np.log(1),
np.log(1000),
q=1,
)) if n_iter is None else n_iter,
n_jobs=n_jobs,
random_state=_random_state(_name('rstate'), random_state),
verbose=verbose
)
return rval
def linear_discriminant_analysis(name,
solver=None,
shrinkage=None,
priors=None,
n_components=None,
store_covariance=False,
tol=0.00001):
def _name(msg):
return '%s.%s_%s' % (name, 'lda', msg)
solver_shrinkage = hp.choice(_name('solver_shrinkage_dual'),
[('svd', None),
('lsqr', None),
('lsqr', 'auto'),
('eigen', None),
('eigen', 'auto')])
rval = scope.sklearn_LinearDiscriminantAnalysis(
solver=solver_shrinkage[0] if solver is None else solver,
shrinkage=solver_shrinkage[1] if shrinkage is None else shrinkage,
priors=priors,
n_components=4 * scope.int(
hp.qloguniform(
_name('n_components'),
low=np.log(0.51),
high=np.log(30.5),
q=1.0)) if n_components is None else n_components,
store_covariance=store_covariance,
tol=tol
)
return rval
def parse_search_space(self, learner_space):
'''
search space is dictionary
{'n_estimators': ('uniform', 1, 1000, 'discrete')}
'''
search_space = dict()
for k, v in learner_space.iteritems():
if v[2] == 'samples':
v = (v[0], v[1], min(100, self.X.shape[0]/len(self.kf)-1), v[3])
if v[3] == 'discrete':
search_space[k] = hp.quniform(k, v[1], v[2], 1)
elif v[0] == 'uniform':
search_space[k] = hp.uniform(k, v[1], v[2])
elif v[0] == 'loguniform':
search_space[k] = hp.loguniform(k, v[1], v[2])
elif v[0] == 'normal':
search_space[k] = hp.normal(k, v[1], v[2])
elif v[0] == 'lognormal':
search_space[k] = hp.lognormal(k, v[1], v[2])
elif v[0] == 'quniform':
search_space[k] = hp.quniform(k, v[1], v[2], v[3])
elif v[0] == 'qloguniform':
search_space[k] = hp.qloguniform(k, v[1], v[2], v[3])
elif v[0] == 'qnormal':
search_space[k] = hp.qnormal(k, v[1], v[2], v[3])
elif v[0] == 'qlognormal':
search_space[k] = hp.qlognormal(k, v[1], v[2], v[3])
return search_space
def linear_discriminant_analysis(name,
solver=None,
shrinkage=None,
priors=None,
n_components=None,
store_covariance=False,
tol=0.00001):
def _name(msg):
return '%s.%s_%s' % (name, 'lda', msg)
solver_shrinkage = hp.choice(_name('solver_shrinkage_dual'),
[('svd', None),
('lsqr', None),
('lsqr', 'auto'),
('eigen', None),
('eigen', 'auto')])
rval = scope.sklearn_LinearDiscriminantAnalysis(
solver=solver_shrinkage[0] if solver is None else solver,
shrinkage=solver_shrinkage[1] if shrinkage is None else shrinkage,
priors=priors,
n_components=4 * scope.int(
hp.qloguniform(
_name('n_components'),
low=np.log(0.51),
high=np.log(30.5),
q=1.0)) if n_components is None else n_components,
store_covariance=store_covariance,
tol=tol
)
return rval
class skKNeighborsReg(_KNeighborsRegressor, BaseWrapperReg, metaclass=MetaBaseWrapperRegWithABC):
def __init__(self, n_neighbors=5, weights='uniform', algorithm='auto', leaf_size=30, p=2, metric='minkowski',
metric_params=None, n_jobs=1, **kwargs):
n_jobs = 4
_KNeighborsRegressor.__init__(
self, n_neighbors, weights, algorithm, leaf_size, p, metric, metric_params, n_jobs, **kwargs)
BaseWrapperReg.__init__(self)
HyperOpt_space = {
'n_neighbors': 1 + hp.randint('n_neighbors', 50),
'leaf_size': hp.qloguniform('leaf_size', 2, 5, 1),
'p': hp.uniform('p', 1, 2),
# 'metric': 'minkowski',
# 'metric_params': None,
# 'n_jobs': 1,
# 'weights': 'uniform',
# 'algorithm': 'auto',
}
tuning_grid = {
'n_neighbors': 5,
'weights': 'uniform',
'algorithm': 'auto',
'leaf_size': 30,
'p': 2,
'metric': 'minkowski',
'metric_params': None,
'n_jobs': 1,
}
def passive_aggressive(name,
loss=None,
C=None,
fit_intercept=False,
n_iter=None,
n_jobs=1,
shuffle=True,
random_state=None,
verbose=False):
def _name(msg):
return '%s.%s_%s' % (name, 'sgd', msg)
rval = scope.sklearn_PassiveAggressiveClassifier(
loss=hp.choice(
_name('loss'),
['hinge', 'squared_hinge']) if loss is None else loss,
C=hp.lognormal(
_name('learning_rate'),
np.log(0.01),
np.log(10),
) if C is None else C,
fit_intercept=fit_intercept,
n_iter=scope.int(
hp.qloguniform(
_name('n_iter'),
np.log(1),
np.log(1000),
q=1,
)) if n_iter is None else n_iter,
n_jobs=n_jobs,
random_state=_random_state(_name('rstate'), random_state),
verbose=verbose
)
return rval
class skRidgeReg(_RidgeReg,
BaseWrapperReg,
metaclass=MetaBaseWrapperRegWithABC):
def __init__(self,
alpha=1.0,
fit_intercept=True,
normalize=False,
copy_X=True,
max_iter=None,
tol=1e-3,
solver="auto",
random_state=None):
_RidgeReg.__init__(self, alpha, fit_intercept, normalize, copy_X,
max_iter, tol, solver, random_state)
BaseWrapperReg.__init__(self)
HyperOpt_space = {
'alpha': hp.loguniform('alpha', -5, 3),
'max_iter': hp.qloguniform('max_iter', 4, 8, 1),
'tol': hp.loguniform('tol', -8, 0),
}
tuning_grid = {
'alpha': 1.0,
'tol': 1e-3,
# 'fit_intercept': True,
# 'normalize': False,
# 'max_iter': None,
# 'copy_X': True,
# 'solver': "auto",
# 'random_state': None,
}
def set_finetune_space(self, config_file):
''' Given the original deep net architecture, and a set of pretrained weights
and biases, define the configuration space to search for fintuning parameters '''
# we know these fields won't change, so go ahead and set them as
# defaults now
model_params = nt.get_model_params(config_file)
optim_params = nt.get_optim_params(config_file)
default_finetune_model_params = {k: model_params[k] for k in ('num_hids', 'activs', 'd', 'k')}
default_finetune_model_params['loss_terms'] = ['cross_entropy']
default_finetune_optim_params = {k: optim_params[k] for k in ('optim_method', 'optim_type')}
# define the space of hyperparameters we wish to
search_finetune_model_params = {'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
search_finetune_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(10), log(5e3), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# combine the default and search parameters into a dictionary to define the
# full space - this is what will be passed into the objective function
finetune_model_params = self.merge_default_search(
default_finetune_model_params, search_finetune_model_params)
finetune_optim_params = self.merge_default_search(
default_finetune_optim_params, search_finetune_optim_params)
finetune_hyperspace = {
'finetune_model_params': finetune_model_params, 'finetune_optim_params': finetune_optim_params}
return finetune_hyperspace
def visitSearchSpaceNumber(self, space: SearchSpaceNumber, path: str, counter=None):
label = self.mk_label(path, counter)
if space.pgo is not None:
return scope.pgo_sample(
space.pgo, hp.quniform(label, 0, len(space.pgo) - 1, 1)
)
dist = "uniform"
if space.distribution:
dist = space.distribution
if space.maximum is None:
raise SearchSpaceError(
path, f"maximum not specified for a number with distribution {dist}"
)
max = space.getInclusiveMax()
# if the maximum is not None, the inclusive maximum should not be none
assert max is not None
# These distributions need only a maximum
if dist == "integer":
if not space.discrete:
raise SearchSpaceError(
path,
"integer distribution specified for a non discrete numeric type",
)
return hp.randint(label, max)
if space.minimum is None:
raise SearchSpaceError(
path, f"minimum not specified for a number with distribution {dist}"
)
min = space.getInclusiveMin()
# if the minimum is not None, the inclusive minimum should not be none
assert min is not None
if dist == "uniform":
if space.discrete:
return scope.int(hp.quniform(label, min, max, 1))
else:
return hp.uniform(label, min, max)
elif dist == "loguniform":
# for log distributions, hyperopt requires that we provide the log of the min/max
if min <= 0:
raise SearchSpaceError(
path,
f"minimum of 0 specified with a {dist} distribution. This is not allowed; please set it (possibly using minimumForOptimizer) to be positive",
)
if min > 0:
min = math.log(min)
if max > 0:
max = math.log(max)
if space.discrete:
return scope.int(hp.qloguniform(label, min, max, 1))
else:
return hp.loguniform(label, min, max)
else:
raise SearchSpaceError(path, f"Unknown distribution type: {dist}")
def _validate_and_generate_hyperopt_search_space(search_space):
if not type(search_space) == dict:
raise Exception('Search space has to be type dict. Provided: {}'.format(type(search_space)))
if not all([isinstance(k, str) for k in search_space.keys()]):
raise Exception('Only string values are allowed for hyperparameter space keys.')
if not all([isinstance(k, _HP) for k in search_space.values()]):
raise Exception('All hyperparameter space values has to be of type cerebro.tune.base._HP.'
' Nested search spaces are not supported yet')
hyperopt_space = {}
for k in search_space:
if isinstance(search_space[k], _HPChoice):
hyperopt_space[k] = hp.choice(k, search_space[k].options)
elif isinstance(search_space[k], _HPUniform):
hyperopt_space[k] = hp.uniform(k, search_space[k].min, search_space[k].max)
elif isinstance(search_space[k], _HPQUniform):
hyperopt_space[k] = hp.quniform(k, search_space[k].min, search_space[k].max, search_space[k].q)
elif isinstance(search_space[k], _HPLogUniform):
hyperopt_space[k] = hp.loguniform(k, search_space[k].min, search_space[k].max)
elif isinstance(search_space[k], _HPQLogUnifrom):
hyperopt_space[k] = hp.qloguniform(k, search_space[k].min, search_space[k].max, search_space[k].q)
else:
raise Exception('Unsupported hyperparameter option type: {}'.format(type(search_space[k])))
return hyperopt_space
class skBaggingClf(BaseWrapperClf, _BaggingClassifier, metaclass=MetaBaseWrapperClfWithABC):
def __init__(self, base_estimator=None, n_estimators=10, max_samples=1.0, max_features=1.0, bootstrap=True,
bootstrap_features=False, oob_score=False, warm_start=False, n_jobs=1, random_state=None, verbose=0):
n_estimators = int(n_estimators)
_BaggingClassifier.__init__(
self, base_estimator, n_estimators, max_samples, max_features, bootstrap, bootstrap_features, oob_score,
warm_start, n_jobs, random_state, verbose)
BaseWrapperClf.__init__(self)
HyperOpt_space = {
'n_estimators': hp.qloguniform('n_estimators', 2, 5, 1),
}
tuning_grid = {
}
tuning_params = {
}
@property
def feature_importances(self):
return np.mean([
tree.feature_importances_ for tree in self.estimators_
], axis=0)
@property
def feature_importances_pack(self):
return {
'mean': self.feature_importances,
'std': np.std([
tree.feature_importances_ for tree in self.estimators_
], axis=0)
}
def __construct_hyperopt_space(self):
# Function to contruct the space for fmin in hyperopt
hyperopt_space = {}
for param in self.space:
param_details = self.space[param]
label = param
sampling_type = param_details[0]
bounds = param_details[1]
if sampling_type == 'choice':
hyperopt_space[param] = hp.choice(label, bounds)
elif sampling_type == 'uniform':
hyperopt_space[param] = hp.uniform(label, bounds[0], bounds[1])
elif sampling_type == 'randint':
hyperopt_space[param] = hp.randint(label, bounds[0])
elif sampling_type == 'quniform':
hyperopt_space[param] = hp.quniform(label, bounds[0],
bounds[1], bounds[2])
elif sampling_type == 'loguniform':
hyperopt_space[param] = hp.loguniform(label, bounds[0],
bounds[1])
elif sampling_type == 'qloguniform':
hyperopt_space[param] = hp.qloguniform(label, bounds[0],
bounds[1], bounds[2])
return hyperopt_space
def set_old_space(self):
''' defines an old net from the 80s - simple sigmoid layers, nothing fancy'''
min_layers = 1
max_layers = 3
# sets up the neural network
nnets = [None] * (max_layers - min_layers + 1)
for i, num_layers in enumerate(range(min_layers, max_layers + 1)):
num_hids = [None] * num_layers
for j in range(num_layers):
num_hids[j] = hp.qloguniform(
'num_hid_%i%i' % (i, j), log(10), log(100), 1)
nnets[i] = num_hids
default_mln_model_params = {
'd': self.d, 'k': self.k, 'loss_terms': ['cross_entropy']}
search_mln_model_params = {
'arch': hp.choice('arch', nnets),
'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
default_mln_optim_params = {
'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}
search_mln_optim_params = {
'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# merge the default and search spaces
mln_model_params = self.merge_default_search(
default_mln_model_params, search_mln_model_params)
mln_optim_params = self.merge_default_search(
default_mln_optim_params, search_mln_optim_params)
# define the hyperparamater space to search
hyperspace = {'mln_model_params': mln_model_params,
'mln_optim_params': mln_optim_params}
return hyperspace
def get_k_best_params(inner_model_params, name='k_best_selector'):
return scope.get_k_best_wrapper(
inner_model=inner_model_params,
k_best=scope.get_k_best(
k=hp.qloguniform(get_full_name(name, 'k'), 0, 5, 1),
score_func=hp.choice(get_full_name(name, 'score'), (chi2, f_classif)),
),
)
def rbm(name,
n_components=None,
learning_rate=None,
batch_size=None,
n_iter=None,
verbose=False,
random_state=None):
def _name(msg):
return '%s.%s_%s' % (name, 'rbm', msg)
rval = scope.sklearn_BernoulliRBM(
n_components=scope.int(
hp.qloguniform(
name + '.n_components',
low=np.log(0.51),
high=np.log(999.5),
q=1.0)) if n_components is None else n_components,
learning_rate=hp.lognormal(
name + '.learning_rate',
np.log(0.01),
np.log(10),
) if learning_rate is None else learning_rate,
batch_size=scope.int(
hp.qloguniform(
name + '.batch_size',
np.log(1),
np.log(100),
q=1,
)) if batch_size is None else batch_size,
n_iter=scope.int(
hp.qloguniform(
name + '.n_iter',
np.log(1),
np.log(1000), # -- max sweeps over the *whole* train set
q=1,
)) if n_iter is None else n_iter,
verbose=verbose,
random_state=_random_state(_name('rstate'), random_state),
)
return rval
def many_dists():
a = hp.choice('a', [0, 1, 2])
b = hp.randint('b', 10)
c = hp.uniform('c', 4, 7)
d = hp.loguniform('d', -2, 0)
e = hp.quniform('e', 0, 10, 3)
f = hp.qloguniform('f', 0, 3, 2)
g = hp.normal('g', 4, 7)
h = hp.lognormal('h', -2, 2)
i = hp.qnormal('i', 0, 10, 2)
j = hp.qlognormal('j', 0, 2, 1)
k = hp.pchoice('k', [(.1, 0), (.9, 1)])
z = a + b + c + d + e + f + g + h + i + j + k
return {'loss': scope.float(scope.log(1e-12 + z ** 2)),
'status': base.STATUS_OK}
def test_read_qloguniform(self):
# 0 float
# 1 hyperopt_param
# 2 Literal{nhid1}
# 3 qloguniform
# 4 Literal{2.77258872224}
# 5 Literal{6.9314718056}
# 6 q =
# 7 Literal{16}
qloguniform = hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16). \
inputs()[0].inputs()[1]
ret = self.pyll_reader.read_qloguniform(qloguniform, 'nhid1')
expected = configuration_space.UniformFloatHyperparameter(
'nhid1', 16, 1024, q=16, base=np.e)
self.assertEqual(expected, ret)
def pca(name, n_components=None, whiten=None, copy=True):
rval = scope.sklearn_PCA(
# -- qloguniform is missing a "scale" parameter so we
# lower the "high" parameter and multiply by 4 out front
n_components=4 * scope.int(
hp.qloguniform(
name + '.n_components',
low=np.log(0.51),
high=np.log(30.5),
q=1.0)) if n_components is None else n_components,
whiten=hp_bool(
name + '.whiten',
) if whiten is None else whiten,
copy=copy,
)
return rval
def _process_vw_argument(self, arg, value, algorithm):
try:
distr_part = self.distr_pattern.findall(value)[0]
except IndexError:
distr_part = ''
range_part = self.range_pattern.findall(value)[0]
is_continuous = '..' in range_part
ocd = self.only_continuous.findall(value)
if not is_continuous and len(ocd)> 0 and ocd[0] != '':
raise ValueError(("Need a range instead of a list of discrete values to define "
"uniform or log-uniform distribution. "
"Please, use [min..max]%s form") % (distr_part))
if is_continuous and arg == '-q':
raise ValueError(("You must directly specify namespaces for quadratic features "
"as a list of values, not as a parametric distribution"))
hp_choice_name = "_".join([algorithm, arg.replace('-', '')])
try_omit_zero = 'O' in distr_part
distr_part = distr_part.replace('O', '')
if is_continuous:
vmin, vmax = [float(i) for i in range_part.split('..')]
if distr_part == 'L':
distrib = hp.loguniform(hp_choice_name, log(vmin), log(vmax))
elif distr_part == '':
distrib = hp.uniform(hp_choice_name, vmin, vmax)
elif distr_part == 'I':
distrib = hp.quniform(hp_choice_name, vmin, vmax, 1)
elif distr_part in {'LI', 'IL'}:
distrib = hp.qloguniform(hp_choice_name, log(vmin), log(vmax), 1)
else:
raise ValueError("Cannot recognize distribution: %s" % (distr_part))
else:
possible_values = range_part.split(',')
if arg == '-q':
possible_values = [v.replace('+', ' -q ') for v in possible_values]
distrib = hp.choice(hp_choice_name, possible_values)
if try_omit_zero:
hp_choice_name_outer = hp_choice_name + '_outer'
distrib = hp.choice(hp_choice_name_outer, ['omit', distrib])
return distrib
def colkmeans(name,
n_clusters=None,
init=None,
n_init=None,
max_iter=None,
tol=None,
precompute_distances=True,
verbose=0,
random_state=None,
copy_x=True,
n_jobs=1):
rval = scope.sklearn_ColumnKMeans(
n_clusters=scope.int(
hp.qloguniform(
name + '.n_clusters',
low=np.log(1.51),
high=np.log(19.5),
q=1.0)) if n_clusters is None else n_clusters,
init=hp.choice(
name + '.init',
['k-means++', 'random'],
) if init is None else init,
n_init=hp.choice(
name + '.n_init',
[1, 2, 10, 20],
) if n_init is None else n_init,
max_iter=scope.int(
hp.qlognormal(
name + '.max_iter',
np.log(300),
np.log(10),
q=1,
)) if max_iter is None else max_iter,
tol=hp.lognormal(
name + '.tol',
np.log(0.0001),
np.log(10),
) if tol is None else tol,
precompute_distances=precompute_distances,
verbose=verbose,
random_state=random_state,
copy_x=copy_x,
n_jobs=n_jobs,
)
return rval
def nystrom(name, n_components=None, kernel=None, max_components=np.Inf, copy=True):
def _name(msg):
return '%s.%s_%s' % (name, 'nystrom', msg)
rval = scope.sklearn_Nystrom(
n_components=4 * scope.int(
hp.qloguniform(
name + '.n_components',
low=np.log(0.51),
high=np.log(min(max_components / 4, 30.5)),
q=1.0)) if n_components is None else n_components,
kernel=hp.pchoice(
_name('kernel'),
[ (0.35, 'sigmoid'),
(0.35, 'rbf'),
(0.30, 'poly')]) if kernel is None else kernel,
gamma=_svc_gamma('gamma'),
coef0=hp.uniform(_name('coef0'), 0.0, 1.0)
)
return rval
def parameter_to_tpe(label, parameter):
"""returns the parameter in TPE format."""
if parameter.is_int:
if parameter.log_scale:
return hp.qloguniform(label,
log(parameter.min_val),
log(parameter.max_val),
1)
else:
return hp.quniform(label,
parameter.min_val,
parameter.max_val,
1)
else:
if parameter.log_scale:
return hp.loguniform(label,
log(parameter.min_val),
log(parameter.max_val))
else:
return hp.uniform(label,
parameter.min_val,
parameter.max_val)
def init2(self):
Files.mkdir("../model/others")
self.fpath = "../model/others/hopt_vw.txt"
if config.test:
self.max_evals = 1
else:
self.max_evals = 25
# TODO: read passes from hyperopt result
self.space = {
"l1": hp.qloguniform("l1", np.log(1e-9), np.log(1e-6), 1e-9),
'interaction': "",
"passes": 15
}
self.i_folds = [("B", 0)]
self.output_items = []
self.output_items += ["loss", "rsme"]
self.output_items += ["loss{}".format(i) for i, i_fold in enumerate(self.i_folds)]
self.output_items += ["rsme{}".format(i) for i, i_fold in enumerate(self.i_folds)]
self.columns = sorted(self.space.keys())
"""
Cart-pole balancing with independent discretization
"""
from rlpy.Domains import InfCartPoleBalance
from rlpy.Agents import Q_Learning
from rlpy.Representations import *
from rlpy.Policies import eGreedy
from rlpy.Experiments import Experiment
import numpy as np
from hyperopt import hp
param_space = {
"num_rbfs": hp.qloguniform("num_rbfs", np.log(1e1), np.log(1e4), 1),
'resolution': hp.quniform("resolution", 3, 30, 1),
'lambda_': hp.uniform("lambda_", 0., 1.),
'boyan_N0': hp.loguniform("boyan_N0", np.log(1e1), np.log(1e5)),
'initial_learn_rate': hp.loguniform("initial_learn_rate", np.log(5e-2), np.log(1))}
def make_experiment(
exp_id=1, path="./Results/Temp/{domain}/{agent}/{representation}/",
boyan_N0=753,
initial_learn_rate=.7,
resolution=25.,
num_rbfs=206.,
lambda_=0.75):
opt = {}
opt["path"] = path
opt["exp_id"] = exp_id
opt["max_steps"] = 10000
opt["num_policy_checks"] = 20
'eta' : hp.quniform('eta', 0.3, 2, 0.2),
'lambda' : hp.quniform('lambda', 0, 5, 0.05),
'alpha' : hp.quniform('alpha', 0, 0.5, 0.005),
'lambda_bias' : hp.quniform('lambda_bias', 0, 3, 0.1),
'num_round' : hp.quniform('num_round', xgb_min_num_round, xgb_max_num_round, xgb_num_round_step),
'nthread': xgb_nthread,
'silent' : 1,
'seed': xgb_random_seed,
"max_evals": hyperopt_param["xgb_max_evals"],
}
#
param_space_reg_xgb_tree_1 = {
'task': 'regression',
'booster': 'gblinear',
'objective': 'multi:softmax',
'max_delta_step':hp.qloguniform('max_delta_step',1,10,1),
'num_class':20,
'eta': hp.quniform('eta', 0.01, 1, 0.05),
'gamma': hp.quniform('gamma', 0.2, 4, 0.2),
'min_child_weight': hp.quniform('min_child_weight', 1, 10,1),
'max_depth': hp.quniform('max_depth', 4, 8, 1),
'subsample': hp.quniform('subsample', 0.5, 1, 0.1),
'colsample_bytree': hp.quniform('colsample_bytree', 0.1, 1, 0.01),
'num_round': hp.quniform('num_round', xgb_min_num_round, xgb_max_num_round, xgb_num_round_step),
'nthread': xgb_nthread,
'silent': 1,
'seed': xgb_random_seed,
"max_evals": hyperopt_param["xgb_max_evals"],
}
param_space_reg_xgb_tree = {
'task': 'regression',
ridge = Ridge( alpha = ridge_alpha ) elmr = pipeline.Pipeline( [( 'rl1', rl1 ), ( 'rl2', rl2 ), ( 'ridge', ridge )] ) elmr.fit( x_train, y_train ) p = elmr.predict( x_test ) rmse = RMSE( y_test, p ) return rmse ### space = ( hp.qloguniform( 'n_hidden_1', log( 10 ), log( 1000 ), 1 ), hp.uniform( 'alpha_1', 0, 1 ), hp.loguniform( 'rbf_width_1', log( 1e-5 ), log( 100 )), hp.qloguniform( 'n_hidden_2', log( 10 ), log( 1000 ), 1 ), hp.uniform( 'alpha_2', 0, 1 ), hp.loguniform( 'rbf_width_2', log( 1e-5 ), log( 100 )), hp.loguniform( 'ridge_alpha', -15, 5 ) ) ### if __name__ == '__main__': headers = [ 'rmse', 'n_hidden_1', 'alpha_1', 'rbf_width_1',
import numpy as np
from hyperopt import hp
import hpnnet.nips2011
space = {'preproc': hp.choice('preproc', [{
'preproc' : 0}, {
'preproc' : 1, # Column Normalization
'colnorm_thresh' : hp.loguniform('colnorm_thresh', np.log(1e-9), np.log(1e-3)),
}, {
'preproc' : 2, # PCA
'pca_energy' : hp.uniform('pca_energy', .5, 1),
}]),
'nhid1': hp.qloguniform('nhid1', np.log(16), np.log(1024), q=16),
'dist1': hp.choice('dist1', [0, 1]), # 0 = Uniform, 1 = Normal
'scale_heur1': hp.choice('scale_heur1', [{
'scale_heur1' : 0, # Old
'scale_mult1': hp.uniform('scale_mult1', .2, 2)}, {
'scale_heur1': 1}]), # Glorot
'squash' : hp.choice('squash', [0, 1]), # ['tanh', 'logistic']
'iseed': hp.choice('iseed', [0, 1, 2, 3]), # Are 5, 6, 7, 8
'batch_size': hp.choice('batch_size', [0, 1]), # 20 or 100
'lr': hp.lognormal('lr', np.log(.01), 3.),
'lr_anneal_start': hp.qloguniform('lr_anneal_start', np.log(100), np.log(10000), q=1),
'l2_penalty': hp.choice('l2_penalty', [{
'l2_penalty' : 0}, { # Zero
'l2_penalty' : 1, # notzero
'l2_penalty_nz': hp.lognormal('l2_penalty_nz', np.log(1.0e-6), 2.)}])
}
rl = GRBFRandomLayer( n_hidden = n_hidden, grbf_lambda = grbf_lambda ) elmr = GenELMRegressor( hidden_layer = rl ) elmr.fit( x_train, y_train ) p = elmr.predict( x_test ) rmse = RMSE( y_test, p ) return rmse ### space = ( hp.qloguniform( 'n_hidden', log( 10 ), log( 1000 ), 1 ), hp.loguniform( 'grbf_lambda', -20, 0 ) ) ### if __name__ == '__main__': headers = [ 'rmse', 'n_hidden', 'grbf_lambda' ] o_f = open( output_file, 'wb' ) writer = csv.writer( o_f ) writer.writerow( headers ) start_time = time() best = fmin( run_wrapper, space, algo = tpe.suggest, max_evals = max_evals ) end_time = time()
def learn_pretrain_settings(self, config_file):
''' automatically searches for the right settings to pre-train the model described in the configuration path '''
# seperate out the default from the search space
default_pretrain_model_params = {'corrupt_type': 'mask', 'activs': ['sigmoid', 'sigmoid'],
'loss_terms': ['binary_cross_entropy', 'corruption']}
default_pretrain_optim_params = {
'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}
default_last_model_params = {
'activs': ['softmax'], 'loss_terms': ['cross_entropy'], 'num_hids': []}
default_last_optim_params = {
'optim_type': 'minibatch', 'optim_method': 'RMSPROP'}
# get the number of nodes per hidden layer from the original architecture so we know how to
# initialize the autoencoders and the final softmax layer
model_params = nt.get_model_params(config_file)
pretrain_nodes = [model_params['d']] + model_params['num_hids']
best_pretrain_settings = []
self.curr_X = self.X
print 'Starting pre-training...'
for l1, l2 in zip(pretrain_nodes[:-1], pretrain_nodes[1:]):
# we will reuse this hyperspace (denoising autoencoder) for every pair
# of input-output layers
search_pretrain_model_params = {'corrupt_p': hp.uniform('corrupt_p', 0, 1),
'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
search_pretrain_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
# get the next pretraining space ready
default_pretrain_model_params['d'] = l1
default_pretrain_model_params['num_hids'] = [l2]
curr_pretrain_model_params = self.merge_default_search(
default_pretrain_model_params, search_pretrain_model_params)
curr_pretrain_optim_params = self.merge_default_search(
default_pretrain_optim_params, search_pretrain_optim_params)
# combine the merged default model and optim parameters into a
# single dictionary
pretrain_hyperspace = {'pretrain_model_params': curr_pretrain_model_params,
'pretrain_optim_params': curr_pretrain_optim_params}
# search over the hyperparameters
print 'Searching over the pretraining hyperspace...'
best = fmin(self.compute_pretrain_objective, pretrain_hyperspace, algo=tpe.suggest,
max_evals=100)
print 'Complete!'
print 'Training this layer'
best_pretrain_settings.append(best)
# this updates curr_X for the next pre-training layer, and also stores the pre-trained
# weights
self.pretrain_layer_with_settings(
best,
search_pretrain_model_params,
search_pretrain_optim_params,
default_pretrain_model_params,
default_pretrain_optim_params, layer_type='pretrain')
print 'Pretraining the final layer..'
# the last layer is not pretrained with an autoencoder...
default_last_model_params['d'] = l2
default_last_model_params['k'] = model_params['k']
# ...it is trained normally, via simple softmax regression
search_last_model_params = {'l1_reg': hp.choice('l1_reg', [None, hp.loguniform('l1_decay', log(1e-5), log(10))]),
'l2_reg': hp.choice('l2_reg', [None, hp.loguniform('l2_decay', log(1e-5), log(10))])}
search_last_optim_params = {'learn_rate': hp.uniform('learn_rate', 0, 1),
'rho': hp.uniform('rho', 0, 1),
'num_epochs': hp.qloguniform('num_epochs', log(1e2), log(2000), 1),
'batch_size': hp.quniform('batch_size', 128, 1024, 1),
'init_method': hp.choice('init_method', ['gauss', 'fan-io']),
'scale_factor': hp.uniform('scale_factor', 0, 1)}
last_model_params = self.merge_default_search(
default_last_model_params, search_last_model_params)
last_optim_params = self.merge_default_search(
default_last_optim_params, search_last_optim_params)
last_hyperspace = {
'last_model_params': last_model_params, 'last_optim_params': last_optim_params}
best = fmin(self.compute_last_objective, last_hyperspace, algo=tpe.suggest,
max_evals=100)
best_pretrain_settings.append(best)
self.pretrain_layer_with_settings(
best,
search_last_model_params,
search_last_optim_params,
default_last_model_params,
default_last_optim_params, layer_type='last')
print 'Complete!'
return best_pretrain_settings
def model(self):
#cname = sys._getframe().f_code.co_name
cname = 'lgb'
train, y, test = self.train_, self.y_, self.test_
train.drop('id', axis=1, inplace=True)
test.drop('id', axis=1, inplace=True)
from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval
dtrain = lgb.Dataset(train, label=y)
def fix_params(params):
for p in ['min_data_in_leaf', 'num_leaves', 'max_bin' ]:
params[p] = int(params[p])
params['num_leaves'] = max(params['num_leaves'], 2)
def step_xgb(params):
fix_params(params)
cv = lgb.cv(params, dtrain,
num_boost_round=10000,
early_stopping_rounds=50,
nfold=6,
seed=params['seed'])
rounds = np.argmin(cv['binary_logloss-mean'])
score = np.min(cv['binary_logloss-mean'])
print(cname, score, rounds, params, self.now())
return dict(loss=score, status=STATUS_OK)
space_lgb = dict(
bagging_fraction = hp.quniform('bagging_fraction', 0.5, 1, 0.001),
colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05),
feature_fraction = hp.quniform('feature_fraction', 0.5, 1, 0.001),
lambda_l1 = hp.choice('lambda_l1', [0, hp.loguniform('lambda_l1_positive', -16, 2)]),
lambda_l2 = hp.choice('lambda_l2', [0, hp.loguniform('lambda_l2_positive', -16, 2)]),
learning_rate = hp.loguniform('learning_rate', -7, 0),
max_bin = hp.qloguniform('max_bin', 0, 20, 1),
max_depth = hp.choice('max_depth', range(2, 9)),
min_child_weight = hp.quniform('min_child_weight', 1, 6, 1),
min_data_in_leaf = hp.qloguniform('min_data_in_leaf', 0, 6, 1),
min_sum_hessian_in_leaf = hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
num_leaves = hp.qloguniform('num_leaves', 2, 7, 1),
reg_alpha = hp.quniform('reg_alpha', 0, 1, 0.001),
subsample = hp.quniform('subsample', 0.6, 1, 0.05),
bagging_freq = 1,
objective = 'binary',
metric = 'binary_logloss',
seed = 1,
#silent = 1,
)
trs = self.load('lightgbm_trials')
if trs == None or self.debug_:
tr = Trials()
else:
tr, _ = trs
if len(tr.trials) > 0:
print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_lgb, tr.argmin))
best = tr.argmin
while len(tr.trials) < self.max_ho_trials_:
print(len(tr.trials), end=' ')
#best = fmin(step_xgb, space_lgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr)
best = fmin(step_xgb, space_lgb, algo=partial(tpe.suggest, n_startup_jobs=1), max_evals=len(tr.trials) + 1, trials = tr)
self.save('lightgbm_trials', (tr, space_lgb))
lgb_params = space_eval(space_lgb, best)
fix_params(lgb_params)
print(lgb_params)
N_splits = self.num_splits_
N_seeds = self.num_seeds_
v, z = self.v_, self.z_
skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True)
cv = []
for s in range(N_seeds):
scores = []
cname2 = cname + str(s)
v[cname2], z[cname2] = 0, 0
lgb_params['seed'] = s + self.base_seed_
for n, (itrain, ival) in enumerate(skf.split(train, y)):
dtrain = lgb.Dataset(train.ix[itrain], y[itrain])
dvalid = lgb.Dataset(train.ix[ival], y[ival])
clf = lgb.train(lgb_params, dtrain,
num_boost_round=10000,
valid_sets=[dtrain, dvalid],
valid_names=['train', 'valid'],
early_stopping_rounds=100, verbose_eval=False)
p = clf.predict(train.ix[ival])
v.loc[ival, cname2] += p
score = metrics.log_loss(y[ival], p)
z[cname2] += clf.predict(test)
print(cname, 'seed %d step %d of %d: '%(lgb_params['seed'], n+1, skf.n_splits), score, self.now())
scores.append(score)
z[cname2] /= N_splits
cv.append(np.mean(scores))
print('seed %d loss: '%(lgb_params['seed']), scores, np.mean(scores), np.std(scores))
z['y'] = z[cname2]
print('cv:', cv, np.mean(cv), np.std(cv))
return cv, None
# TODO: this needs fixing
def hyperopt_obj_fn(args):
''' hyper-opt objective function '''
num_hid, learn_rate, l1_decay, l2_decay, activs, scale_factor, init_method, num_epochs, rho = args
# multilayer net parameters
mln_params = {'d': d, 'k': k, 'num_hids': [num_hid], 'activs': activs,
'loss_terms': ['cross_entropy', 'regularization'], 'l2_decay': l2_decay, 'l1_decay': l1_decay}
rmsprop_params = {'init_method': init_method, 'scale_factor': scale_factor, 'optim_type': 'minibatch',
'optim_method': 'RMSPROP', 'batch_size': 900, 'num_epochs': num_epochs, 'learn_rate': learn_rate, 'rho': rho}
return compute_cv_loss(mln_params, rmsprop_params)
rmsprop_space = (
hp.qloguniform('num_hid', log(10), log(1000), 1),
hp.loguniform('learn_rate', log(1e-4), log(10)),
hp.choice('l1_decay', [0, hp.loguniform('l1', log(1e-6), 2.)]),
hp.choice('l2_decay', [0, hp.loguniform('l2', log(1e-6), 2.)]),
hp.choice('activs', [['sigmoid', 'softmax'], ['reLU', 'softmax']]),
hp.loguniform('scale_factor', log(1e-3), log(1)),
hp.choice('init_method', ['gauss', 'fan-io']),
hp.qloguniform('num_epochs', log(10), log(1e3), 1),
hp.uniform('rho', 1e-2, 0.99)
)
best = fmin(hyperopt_obj_fn, rmsprop_space, algo=tpe.suggest, max_evals=100)
print best
'amp': 0.063,
# 'tau_ref': 0.001,
# 'tau_rc': 0.05,
# 'alpha': 0.825,
# 'tau_ref': hp.uniform('tau_ref', 0.001, 0.005),
'tau_ref': 0.002,
'tau_rc': hp.uniform('tau_rc', 0.01, 0.06),
# 'alpha': hp.uniform('alpha', 0.1, 10.0),
'alpha': 1.0,
'sigma': 0.02,
'noise': 10.,
# learning rate params
'epsW_schedule': hp.choice('epsW_schedule', ['linear', 'exp']),
'epsW_base': hp.lognormal('epsW_base', np.log(1e-3), np.log(1e1)),
'epsW_tgtFactor': hp.qloguniform('epsW_tgtFactor', np.log(1), np.log(1e4), 1),
'epsB_schedule': hp.choice('epsB_schedule', ['linear', 'exp']),
'epsB_base': hp.lognormal('epsB_base', np.log(1e-3), np.log(1e1)),
'epsB_tgtFactor': hp.qloguniform('epsB_tgtFactor', np.log(1), np.log(1e4), 1),
'momW': hp.uniform('momW', 0.001, 0.999),
'momB': hp.uniform('momB', 0.001, 0.999),
# initial weight params
'initW1': hp.lognormal('initW1', np.log(1e-4), np.log(1e2)),
'initW2': hp.lognormal('initW2', np.log(1e-2), np.log(1e1)),
'initW3': hp.lognormal('initW3', np.log(1e-2), np.log(1e1)),
'initW4': hp.lognormal('initW4', np.log(1e-2), np.log(1e1)),
'initW5': hp.lognormal('initW5', np.log(1e-2), np.log(1e1)),
# weight costs
'wc1': hp.lognormal('wc1', np.log(1e-8), np.log(1e3)),
## sklearn
skl_random_seed = config.RANDOM_SEED
skl_n_jobs = config.NUM_CORES
skl_n_estimators_min = 100
skl_n_estimators_max = 1000
skl_n_estimators_step = 10
# ---------------------------- XGBoost ---------------------------------------
## regression with linear booster
param_space_reg_xgb_linear = {
"booster": "gblinear",
"objective": 'reg:linear',
"base_score": config.BASE_SCORE,
"n_estimators" : hp.quniform("n_estimators", xgb_n_estimators_min, xgb_n_estimators_max, xgb_n_estimators_step),
"learning_rate" : hp.qloguniform("learning_rate", np.log(0.002), np.log(0.1), 0.002),
"reg_alpha" : hp.loguniform("reg_alpha", np.log(1e-10), np.log(1e1)),
"reg_lambda" : hp.loguniform("reg_lambda", np.log(1e-10), np.log(1e1)),
"reg_lambda_bias" : hp.quniform("reg_lambda_bias", 0, 3, 0.1),
"nthread": xgb_nthread,
"seed": xgb_random_seed,
}
## regression with tree booster
param_space_reg_xgb_tree = {
"booster": "gbtree",
"objective": 'reg:linear',
"base_score": config.BASE_SCORE,
"n_estimators" : hp.quniform("n_estimators", xgb_n_estimators_min, xgb_n_estimators_max, xgb_n_estimators_step),
"learning_rate" : hp.qloguniform("learning_rate", np.log(0.002), np.log(0.1), 0.002),
"gamma": hp.loguniform("gamma", np.log(1e-10), np.log(1e1)),
assert lower < upper, "Lower bound of uniform greater than upper bound"
return hp.loguniform(name, lower, upper)
def qloguniform(name, (lower, upper, q)):
"""
Function to create hyperopt qloguniform variable
Input
------------------
name - Variable name
(lower, upper, q) - Tuple of bounds and q value.
Distribution looks like round(exp(uniform(lower, upper)) / q) * q
"""
assert lower < upper, "Lower bound of uniform greater than upper bound"
return hp.qloguniform(name, lower, upper, q)
def normal(name, (mu, sigma)):
"""
Function to create hyperopt normal variable
Input
------------------
name - Variable name
(mu, sigma) - Tuple of mean and standard deviation.
"""
return hp.normal(name, mu, sigma)
def qnormal(name, (mu, sigma, q)):
"""