def main(args, max_evals):
# Hyperparameters space
model_path = f"./checkpoint/{args.experiment_id}_ckpt.pth"
trials_path = f"./results/{args.experiment_id}_trials.p"
display_step = args.iterations//100 if args.iterations > 1000 else 10 # relative display steps
space = {#------------------------------------- Architecture -------------------------------------#
'experiment_id': hp.choice(label='experiment_id', options=[args.experiment_id]),
'input_size': hp.choice(label='input_size', options=[SIZE]),
'n_classes': hp.choice(label='n_classes', options=[4_000]),
#------------------------------ Optimization Regularization -----------------------------#
'iterations': hp.choice(label='iterations', options=[args.iterations]),
'display_step': scope.int(hp.choice(label='display_step', options=[display_step])),
'batch_size': scope.int(hp.choice(label='batch_size', options=[512])),
#'initial_lr': hp.loguniform(label='lr', low=np.log(5e-3), high=np.log(0.1)),
'initial_lr': scope.float(hp.choice(label='initial_lr', options=[0.1])),
'lr_decay': scope.float(hp.choice(label='lr_decay', options=[0.5])),
'adjust_lr_step': hp.choice(label='adjust_lr_step', options=[300_000//3]),
'weight_decay': hp.choice(label='weight_decay', options=[5e-4]),
'with_center_loss': hp.choice(label='with_center_loss', options=[bool(args.with_center_loss)]),
'initial_clr': hp.choice(label='initial_clr', options=[0.01, 0.05, 0.1, 0.5]),
'alpha': hp.choice(label='alpha', options=[0.1, 0.01]),
#'display_step': scope.int(hp.choice(label='eval_epochs', options=[3_000])),
#-------------------------------------- Others --------------------------------------#
'path': hp.choice(label='path', options=[model_path]),
'trials_path': hp.choice(label='trials_path', options=[trials_path]),
'random_seed': scope.int(hp.quniform('random_seed', 1, 10, 1))}
# Hyperparameters search
trials = Trials()
fmin_objective = partial(fit_and_log, trials=trials, verbose=True)
best_model = fmin(fmin_objective, space=space, algo=tpe.suggest, max_evals=max_evals, trials=trials)
# Save output
with open(trials_path, "wb") as f:
pickle.dump(trials, f)
def bayes_tune(plot_results):
space = {
"hidden_layers":
hp.choice("options", [{
"hidden_layers":
1,
"network": (X_train_std.shape[1],
scope.int(hp.quniform("1_hidden_1", 5, 32, 1)), 1)
}, {
"hidden_layers":
2,
"network": (X_train_std.shape[1],
scope.int(hp.quniform("2_hidden_1", 5, 32, 1)),
scope.int(hp.quniform("2_hidden_2", 5, 32, 1)), 1)
}]),
"learning_rate":
hp.loguniform("learning_rate", np.log(0.0001), np.log(0.2)),
"iterations":
scope.int(hp.quniform("iterations", 500, 5000, 1)),
"reg_param":
hp.loguniform("reg_param", np.log(0.001), np.log(1))
}
bayes_trials = Trials()
best = fmin(objective,
space,
algo=tpe.suggest,
max_evals=250,
trials=bayes_trials)
if plot_results:
dataplot.plot_results_bayes(bayes_trials)
def tune_hyperparameters(self, X, y):
space = {
'num_iterations':
scope.int(hp.quniform('num_iterations', 100, 2000, 1)),
'n_estimators':
scope.int(hp.quniform('n_estimators', 100, 1000, 1)),
'learning_rate':
hp.choice('learning_rate', [0.005, 0.01, 0.05]),
'num_leaves':
hp.choice('num_leaves', [15, 31, 60, 90]),
'max_depth':
hp.choice('max_depth', range(2, 20, 2)),
'bagging_fraction':
hp.quniform('bagging_fraction', 0.5, 0.9, 1),
'min_split_gain':
hp.quniform('min_split_gain', 0.4, 0.9, 1),
# 'min_data_in_leaf': hp.quniform('min_data_in_leaf', range(4, 80, 10)),
}
ts = Trials()
best = fmin(lambda params: self.objective(params, X, y),
space,
algo=partial(tpe.suggest, n_startup_jobs=15),
max_evals=self.TUNIING_PARAMS["max_evals"],
verbose=0,
trials=ts)
self.best_param = best
self.is_tuned = True
def _classification_incremental_sgd_classifier() -> dict:
return {
'loss':
hp.choice('loss', [
'hinge', 'log', 'modified_huber', 'squared_hinge', 'perceptron',
'squared_loss', 'huber', 'epsilon_insensitive',
'squared_epsilon_insensitive'
]),
'penalty':
hp.choice('penalty', [None, 'l1', 'l2', 'elasticnet']),
'alpha':
hp.uniform('alpha', 0.0001, 0.5),
'l1_ratio':
hp.uniform('l1_ratio', 0.15, 1.0),
'fit_intercept':
hp.choice('fit_intercept', [True, False]),
'tol':
hp.uniform('tol', 1e-3, 0.5),
'epsilon':
hp.uniform('epsilon', 1e-3, 0.5),
'learning_rate':
hp.choice('learning_rate',
['constant', 'optimal', 'invscaling', 'adaptive']),
'eta0':
scope.int(hp.quniform('eta0', 4, 30, 1)),
'power_t':
hp.uniform('power_t', 0.3, 0.7),
# 'early_stopping': hp.choice('early_stopping', [True, False]), #needs to be false with partial_fit
'n_iter_no_change':
scope.int(hp.quniform('n_iter_no_change', 5, 30, 5)),
'validation_fraction':
0.1,
'average':
hp.choice('average', [True, False])
}
def get_parameters(train_data,
kFold,
iterations,
save=False,
filepath='./result/loss_time_tpe.csv'):
def objective(parameters):
if save:
loss, timepoint = object_function.cv_method(
parameters, train_data, kFold, start)
timepoint_dic.append(timepoint)
loss_dic.append(loss)
else:
loss = object_function.cv_method(parameters, train_data, kFold)
return loss
configspace = {
'boosting_type': hp.choice('boosting_type', ['gbdt']),
'objective': hp.choice('objective', ['regression_l2']),
# hp.quniform('max_depth', -10, 10, 1),
'num_leaves': scope.int(hp.quniform('num_leaves', 10, 35, 1)),
'min_data_in_leaf': scope.int(hp.quniform('min_data_in_leaf', 1, 12,
1)),
'max_bin': scope.int(hp.quniform('max_bin', 20, 255, 10)),
'lambda_l2': scope.int(hp.quniform('lambda_l2', 0, 70, 5)),
'feature_fraction': hp.uniform('feature_fraction', 0.01, 1),
'min_gain_to_split': hp.uniform('min_gain_to_split', 0.0, 1),
'feature_fraction': hp.uniform('feature_fraction', 0.005, 0.5),
'verbose': hp.choice('verbose', [-1])
}
if save:
trials = Trials()
start = time.time()
timepoint_dic = []
loss_dic = []
best = fmin(objective,
configspace,
algo=tpe.suggest,
max_evals=iterations,
trials=trials)
best_parameters = space_eval(configspace, best)
best_loss = trials.best_trial['result']['loss']
save_to_csv.save(filepath, timepoint_dic, loss_dic)
else:
trials = Trials()
best = fmin(objective,
configspace,
algo=tpe.suggest,
max_evals=iterations,
trials=trials)
best_parameters = space_eval(configspace, best)
best_loss = trials.best_trial['result']['loss']
return best_parameters, best_loss
def __init__(self):
self.search_space = {
'learning_rate':
hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
'L1_flag':
hp.choice('L1_flag', [True, False]),
'hidden_size':
scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(256),
1)),
'batch_size':
scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),
1)),
'lmbda':
hp.loguniform('lmbda', np.log(0.00001), np.log(0.001)),
'optimizer':
hp.choice('optimizer', ["adam", "sgd", 'rms']),
'margin':
hp.uniform('margin', 0.5, 8.0),
'distance_measure':
hp.choice('distance_measure',
["kl_divergence", "expected_likelihood"]),
'cmax':
hp.loguniform('cmax', np.log(0.05), np.log(0.2)),
'cmin':
hp.loguniform('cmin', np.log(1), np.log(5)),
'epochs':
hp.choice('epochs', [10]) # always choose 10 training epochs.
}
def flat_get_hyperspace(combination):
# reate hyperspace for optimisation.
param_hyperopt = {}
if combination == 'DT':
hyper = {
'DT_criterion': hp.choice('DT_criterion', ['gini', 'entropy']),
'DT_max_depth': scope.int(hp.quniform('DT_max_depth', 5, 15, 1))
}
elif combination == 'RF':
hyper = {
'RF_max_depth': scope.int(hp.quniform('RF_max_depth', 5, 15, 1)),
'RF_n_estimators':
scope.int(hp.quniform('RF_n_estimators', 10, 50, 5))
}
elif combination == 'NN':
hyper = {
'NN_dropout': hp.uniform('NN_dropout', 0, 0.5),
'NN_nodes': scope.int(hp.quniform('NN_nodes', 5, 50, 5)),
'NN_layers': scope.int(hp.quniform('NN_layers', 1, 2, 1))
}
elif combination == 'LR':
hyper = {'LR_penalty': hp.choice('LR_penalty', ['l1', 'l2'])}
param_hyperopt = {**param_hyperopt, **hyper}
return param_hyperopt
def _config_tuning_space(tuning_space_raw):
if tuning_space_raw is None:
return None
hyper_obj = {}
if "learning_rate" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"learning_rate": hp.loguniform('learning_rate', np.log(tuning_space_raw['learning_rate']['min']), np.log(tuning_space_raw['learning_rate']['max']))}}
if "hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"hidden_size": scope.int(hp.qloguniform('hidden_size', np.log(tuning_space_raw['hidden_size']['min']), np.log(tuning_space_raw['hidden_size']['max']), 1))}}
if "ent_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"ent_hidden_size": scope.int(hp.qloguniform("ent_hidden_size", np.log(tuning_space_raw['ent_hidden_size']['min']), np.log(tuning_space_raw['ent_hidden_size']['max']), 1))}}
if "rel_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"rel_hidden_size": scope.int(hp.qloguniform("rel_hidden_size", np.log(tuning_space_raw['rel_hidden_size']['min']), np.log(tuning_space_raw['rel_hidden_size']['max']), 1))}}
if "batch_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"batch_size": scope.int(hp.qloguniform("batch_size", np.log(tuning_space_raw['batch_size']['min']), np.log(tuning_space_raw['batch_size']['max']), 1))}}
if "margin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"margin": hp.uniform("margin", tuning_space_raw["margin"]["min"], tuning_space_raw["margin"]["max"])}}
if "lmbda" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"lmbda": hp.loguniform('lmbda', np.log(tuning_space_raw["lmbda"]["min"]), np.log(tuning_space_raw["lmbda"]["max"]))}}
if "distance_measure" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"distance_measure": hp.choice('distance_measure', tuning_space_raw["distance_measure"])}}
if "cmax" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmax": hp.loguniform('cmax', np.log(tuning_space_raw["cmax"]["min"]), np.log(tuning_space_raw["cmax"]["max"]))}}
if "cmin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmin": hp.loguniform('cmin', np.log(tuning_space_raw["cmin"]["min"]), np.log(tuning_space_raw["cmin"]["max"]))}}
if "optimizer" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"optimizer": hp.choice("optimizer", tuning_space_raw["optimizer"])}}
if "bilinear" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"bilinear": hp.choice('bilinear', tuning_space_raw["bilinear"])}}
if "epochs" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"epochs": hp.choice("epochs", tuning_space_raw["epochs"])}}
return hyper_obj
def get_default_parameter_space():
"""
Return:
dict of DistributionWrappers
"""
return {
'n_estimators':
scope.int(hp.quniform('n_estimators', 5, 10, 5)),
'learning_rate':
hp.loguniform('learning_rate', -7, 0),
'num_leaves':
scope.int(hp.qloguniform('num_leaves', 1, 7, 1)),
'feature_fraction':
hp.uniform('feature_fraction', 0.5, 1),
'bagging_fraction':
hp.uniform('bagging_fraction', 0.5, 1),
'min_data_in_leaf':
scope.int(hp.qloguniform('min_data_in_leaf', 0, 6, 1)),
'min_sum_hessian_in_leaf':
hp.loguniform('min_sum_hessian_in_leaf', -16, 5),
'lambda_l1':
hp.loguniform('lambda_l1', -16, 2),
'lambda_l2':
hp.loguniform('lambda_l2', -16, 2),
}
def __init__(self):
self.search_space = {
'learning_rate': hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
'hidden_size': scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(256),1)),
'batch_size': scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),1)),
'lmbda': hp.loguniform('lmbda', np.log(0.00001), np.log(0.001)),
'optimizer': hp.choice('optimizer', ["adam", "sgd", 'rms']),
'epochs': hp.choice('epochs', [10]) # always choose 10 training epochs.
}
def __call__(self):
print_exp(self.exp_name)
if self.only_q:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-4)),
q=(18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29,
30),
epochs=(250, ),
batch=(32, ))
elif self.only_batch:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-3)),
q=(23, ),
epochs=(250, ),
batch=(16, 32, 50, 100, 128, 200))
elif self.only_epochs:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-4)),
q=(23, ),
epochs=(100, 120, 150, 170, 200, 250, 300, 400,
500),
batch=(32, ))
elif self.only_lr:
pv = ParamValues(lr=hp.loguniform("lr", np.log(1e-4),
np.log(1e-3)),
q=(23, ),
epochs=(250, ),
batch=(32, ))
else:
pv = ParamValues(
lr=hp.loguniform("lr", np.log(1e-4), np.log(1e-3)),
q=scope.int(hp.qloguniform("q", np.log(10), np.log(100), 1)),
#q=(18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30,35),
epochs=scope.int(
hp.qloguniform("epochs", np.log(10), np.log(500), 10)),
#epochs=(100, 125, 150, 175, 200, 250, 300, 400, 500, 700),
batch=(4, 8, 16, 32, 64, 128, 256))
hyper_params = {
"data": {},
"model": {
"lr": pv.lr,
#"encoding_dim": hp.choice("encoding_dim", pv.q), ##
"encoding_dim": pv.q,
},
"fit": {
#"epochs": hp.choice("epochs", pv.epochs), #
"epochs": pv.epochs,
"batch_size": hp.choice("batch_size", pv.batch)
}
}
run = RunFN(self.metric, hyper_params, pv, self.data_path, self.sep,
self.run_on_mongodb, self.start_mongodb, self.db_name,
self.exp_name, self.ip, self.port, self.nr_of_trials,
self.nr_of_workers)
run()
def __init__(self):
self.search_space = {
'learning_rate': hp.loguniform('learning_rate', np.log(0.00001), np.log(0.1)),
'L1_flag': hp.choice('L1_flag', [True, False]),
'hidden_size': scope.int(hp.qloguniform('hidden_size', np.log(8), np.log(128),1)),
'batch_size': scope.int(hp.qloguniform('batch_size', np.log(8), np.log(4096),1)),
'margin': hp.uniform('margin', 0.0, 2.0),
'optimizer': hp.choice('optimizer', ["adam", "sgd", 'rms']),
'epochs': hp.choice('epochs', [10]) # always choose 10 training epochs.
}
def main():
base_name = 'stg2/01_lgb'
seed = 201_000
args = parse_args()
cmd = CmdFactory(_CV)
if not os.path.exists('artifacts/stg1'):
os.makedirs('artifacts/stg1')
X, y = load_train_data()
X_test, _ = load_test_data()
if args.type == 'hpt':
space = {
'learning_rate': hp.uniform('learning_rate', 0.01, 0.5),
'max_depth': scope.int(hp.quniform("max_depth", 1, 7, 1)),
'num_leaves': scope.int(hp.quniform('num_leaves', 2, 20, 1)),
'feature_fraction': hp.uniform('feature_fraction', 0.4, 0.8),
'bagging_fraction': hp.uniform('bagging_fraction', 0.4, 1.0),
'max_bin': scope.int(hp.quniform("max_bin", 400, 700, 1)),
}
cmd.hpt(
train_data=(X, y),
space=space,
seed=seed,
trials_file='artifacts/{}_trials.pickle'.format(base_name),
max_iter=args.max_iter,
steps=args.trial_steps,
n_class=10,
)
elif args.type == 'pred':
params = {
'learning_rate': 0.029191104512834937,
'max_depth': 4,
'num_leaves': 9,
'feature_fraction': 0.7115094613220588,
'bagging_fraction': 0.4897073688073762,
'max_bin': 639,
}
cmd.pred(
train_data=(X, y),
test_data=X_test,
params=params,
seed=seed,
out_tr='artifacts/{}_train.npy'.format(base_name),
out_test='artifacts/{}_test.npy'.format(base_name),
n_bags=args.n_bags,
n_class=10,
)
else:
raise ValueError('type must be hpt or pred.')
def _classification_incremental_perceptron() -> dict:
return {
'penalty': hp.choice('penalty', [None, 'l1', 'l2', 'elasticnet']),
'alpha': hp.uniform('alpha', 0.0001, 0.5),
'fit_intercept': hp.choice('fit_intercept', [True, False]),
'tol': hp.uniform('tol', 1e-3, 0.5),
'shuffle': hp.choice('shuffle', [True, False]),
'eta0': scope.int(hp.quniform('eta0', 4, 30, 1)),
# 'early_stopping': hp.choice('early_stopping', [True, False]), #needs to be false with partial_fit
'validation_fraction': 0.1,
'n_iter_no_change': scope.int(hp.quniform('n_iter_no_change', 5, 30,
5))
}
def run(self):
"""
Hyper-parameter optimization with hyperopt.
"""
if self.pp['net']:
space = {
# Qlearnnet
'net_lr':
hp.loguniform('net_lr', np.log(5e-7), np.log(1e-4)),
'net_lr_decay':
hp.loguniform('net_lr_decay', np.log(0.90), np.log(0.99)),
# Singh
# 'net_lr': hp.loguniform('net_lr', np.log(1e-7), np.log(5e-4)),
'beta':
hp.uniform('beta', 16, 30),
# Double
'net_copy_iter':
hp.loguniform('net_copy_iter', np.log(5), np.log(150)),
'net_creep_tau':
hp.loguniform('net_creep_tau', np.log(0.01), np.log(0.7)),
# Exp. replay
'batch_size':
scope.int(hp.uniform('batch_size', 8, 16)),
'buffer_size':
scope.int(hp.uniform('buffer_size', 2000, 10000)),
# N-step
'n_step':
scope.int(hp.uniform('n_step', 3, 40)),
# Policy
'vf_coeff':
hp.uniform('vf_coeff', 0.005, 0.5),
'entropy_coeff':
hp.uniform('entropy_coeff', 1.0, 100.0)
}
else:
space = {
'beta': hp.uniform('beta', 7, 23),
'alpha': hp.uniform('alpha', 0.0001, 0.4),
'alpha_decay': hp.uniform('alpha_decay', 0.9999, 0.9999999),
'epsilon': hp.loguniform('epsilon', np.log(0.2), np.log(0.8)),
'epsilon_decay': hp.uniform('epsilon_decay', 0.9995,
0.9999999),
'gamma': hp.uniform('gamma', 0.7, 0.90),
'lambda': hp.uniform('lambda', 0.0, 1.0)
}
# Only optimize parameters specified in args
space = {param: space[param] for param in self.pp['hopt']}
if self.pp['hopt_fname'].startswith('mongo:'):
self._hopt_mongo(space)
else:
self._hopt_pickle(space)
def get_default_parameter_space():
"""
Return:
dict of DistributionWrappers
"""
return {
'iterations': scope.int(hp.quniform('iterations', 5, 10, 5)),
'depth': scope.int(hp.quniform('depth', 1, 11, 1)),
'learning_rate': hp.loguniform('learning_rate', -5, -1),
'rsm': hp.uniform('rsm', 0, 1),
'leaf_estimation_method': hp.choice('leaf_estimation_method', ['Newton', 'Gradient']),
'l2_leaf_reg': scope.int(hp.quniform('l2_leaf_reg', 1, 10, 1)),
'bagging_temperature': hp.uniform('bagging_temperature', 0, 2),
}
def _config_tuning_space(tuning_space_raw):
if tuning_space_raw is None:
return None
hyper_obj = {}
if "learning_rate" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"learning_rate": hp.loguniform('learning_rate', np.log(tuning_space_raw['learning_rate']['min']), np.log(tuning_space_raw['learning_rate']['max']))}}
if "hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"hidden_size": scope.int(hp.qloguniform('hidden_size', np.log(tuning_space_raw['hidden_size']['min']), np.log(tuning_space_raw['hidden_size']['max']), 1))}}
if "ent_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"ent_hidden_size": scope.int(hp.qloguniform("ent_hidden_size", np.log(tuning_space_raw['ent_hidden_size']['min']), np.log(tuning_space_raw['ent_hidden_size']['max']), 1))}}
if "rel_hidden_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"rel_hidden_size": scope.int(hp.qloguniform("rel_hidden_size", np.log(tuning_space_raw['rel_hidden_size']['min']), np.log(tuning_space_raw['rel_hidden_size']['max']), 1))}}
if "batch_size" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"batch_size": scope.int(hp.qloguniform("batch_size", np.log(tuning_space_raw['batch_size']['min']), np.log(tuning_space_raw['batch_size']['max']), 1))}}
if "margin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"margin": hp.uniform("margin", tuning_space_raw["margin"]["min"], tuning_space_raw["margin"]["max"])}}
if "lmbda" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"lmbda": hp.loguniform('lmbda', np.log(tuning_space_raw["lmbda"]["min"]), np.log(tuning_space_raw["lmbda"]["max"]))}}
if "distance_measure" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"distance_measure": hp.choice('distance_measure', tuning_space_raw["distance_measure"])}}
if "cmax" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmax": hp.loguniform('cmax', np.log(tuning_space_raw["cmax"]["min"]), np.log(tuning_space_raw["cmax"]["max"]))}}
if "cmin" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"cmin": hp.loguniform('cmin', np.log(tuning_space_raw["cmin"]["min"]), np.log(tuning_space_raw["cmin"]["max"]))}}
if "optimizer" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"optimizer": hp.choice("optimizer", tuning_space_raw["optimizer"])}}
if "bilinear" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"bilinear": hp.choice('bilinear', tuning_space_raw["bilinear"])}}
if "epochs" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"epochs": hp.choice("epochs", tuning_space_raw["epochs"])}}
if "feature_map_dropout" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"feature_map_dropout": hp.choice('feature_map_dropout', tuning_space_raw["feature_map_dropout"])}}
if "input_dropout" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"input_dropout": hp.choice('input_dropout', tuning_space_raw["input_dropout"])}}
if "hidden_dropout" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"hidden_dropout": hp.choice('hidden_dropout', tuning_space_raw["hidden_dropout"])}}
if "use_bias" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"use_bias": hp.choice('use_bias', tuning_space_raw["use_bias"])}}
if "label_smoothing" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"label_smoothing": hp.choice('label_smoothing', tuning_space_raw["label_smoothing"])}}
if "lr_decay" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"lr_decay": hp.choice('lr_decay', tuning_space_raw["lr_decay"])}}
if "l1_flag" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"l1_flag": hp.choice('l1_flag', tuning_space_raw["l1_flag"])}}
if "sampling" in tuning_space_raw:
hyper_obj = {**hyper_obj, **{"sampling": hp.choice('sampling', tuning_space_raw["sampling"])}}
return hyper_obj
def create_hyperopt_space(string_space):
"""Convert the parameters passed from the command line into the corresponding hyperopt space definition.
WARNING: Uses eval() to convert strings to names in the namespace.
For example,
{'C': 'float:hp.lognormal:[0, 1]', 'kernel': 'rbf', 'gamma': 'float:hp.lognormal:[0, 1]'}
gets converted to
{'C': float(hp.lognormal('C', 0, 1)), 'kernel': 'rbf', 'gamma': float(hp.lognormal('gamma', 0, 1))}
Args:
string_space (dict): dictionary of space definitions taken from the raw JSON.
Returns:
(dict): space dictionary ready to be used by hyperopt.
"""
space = {}
for k, v in string_space.items():
if ':' in v: # contains specifications that need to be converted to hyperopt ranges
p_type, p_func, p_params = v.split(':')
p_params = ast.literal_eval(p_params)
if p_type == 'int':
# convert to integer after evaluation
space[k] = scope.int(eval(p_func)(k, *p_params)) # pylint: disable=eval-used
elif p_type == 'float':
# no need to convert because hyperopt natively returns floats
space[k] = eval(p_func)(k, *p_params) # pylint: disable=eval-used
else:
raise TypeError("parameter types must be int or float")
else: # is a constant string
space[k] = v
return space
class Profile:
LGBM_ORIGINAL_NAME = 'LGBM_ORIGINAL'
LGBM_ORIGINAL = {
'fixed_hyperparameters': {
'learning_rate': 0.01,
'num_leaves': 60,
'feature_fraction': 0.6,
'bagging_fraction': 0.6,
'bagging_freq': 2,
'num_iterations': 600,
'min_data_in_leaf': 20,
'boosting_type': 'gbdt',
'objective': 'binary',
'boost_from_average': False
},
'search_space': {
'learning_rate':
hp.loguniform('learning_rate', np.log(0.001), np.log(0.01)),
'num_leaves':
scope.int(hp.quniform('num_leaves', 10, 80, 5)),
'feature_fraction':
hp.loguniform('feature_fraction', np.log(0.6), np.log(0.9)),
'bagging_fraction':
hp.loguniform('bagging_fraction', np.log(0.6), np.log(0.9)),
'bagging_freq':
scope.int(hp.quniform('bagging_freq', 2, 10, 1)),
'num_iterations':
scope.int(hp.quniform('num_iterations', 100, 800, 50)),
'min_data_in_leaf':
scope.int(hp.quniform('min_data_in_leaf', 20, 50, 10)),
'boosting_type':
'gbdt',
'objective':
'binary',
'boost_from_average':
False
}
}
NAME_PROFILE_MAP = {LGBM_ORIGINAL_NAME: LGBM_ORIGINAL}
@staticmethod
def parse_profile(profile_name):
profile = Profile.NAME_PROFILE_MAP[profile_name]
return profile['fixed_hyperparameters'], profile['search_space']
def _regression_random_forest() -> dict:
return {
'n_estimators': hp.choice('n_estimators',
np.arange(150, 1000, dtype=int)),
'max_features': hp.choice('max_features',
['sqrt', 'log2', 'auto', None]),
'max_depth': scope.int(hp.quniform('max_depth', 4, 30, 1)),
}
def get_default_parameter_space():
"""
Return:
dict of DistributionWrappers
"""
return {
'n_estimators': scope.int(hp.quniform('n_estimators', 5, 10, 5)),
'eta': hp.loguniform('eta', -7, 0),
'max_depth': scope.int(hp.quniform('max_depth', 2, 10, 1)),
'subsample': hp.uniform('subsample', 0.5, 1),
'colsample_bytree': hp.uniform('colsample_bytree', 0.5, 1),
'colsample_bylevel': hp.uniform('colsample_bylevel', 0.5, 1),
'min_child_weight': hp.loguniform('min_child_weight', -16, 5),
'gamma': hp.loguniform('gamma', -16, 2),
'lambdax': hp.loguniform('lambdax', -16, 2),
'alpha': hp.loguniform('alpha', -16, 2)
}
def get_search_space():
return {
'batch_size': scope.int(hp.uniform('batch_size', 30, 40)),
'epochs': hp.choice('epochs', [2, 5]),
'activation': hp.choice('activation', ['relu', 'elu']),
'optimizer': hp.choice('optimizer', ['adam', 'sgd']),
'dropout': hp.uniform('dropout', 0, 1)
}
def BayesianOptOnCV(self):
"""
We tried Bayesian Optimization with k-fold cross validation
Here it is implemented as a mehtod
"""
skf = StratifiedKFold(n_splits=5)
train = self.train
y = self.y
param_hyperopt= {
'learning_rate': hp.loguniform('learning_rate', np.log(0.001), np.log(1)),
'max_depth': scope.int(hp.quniform('max_depth', 3, 5, 1)),
'n_estimators': 80,
'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0),
'reg_lambda': hp.quniform('reg_lambda', 700, 900, 10),
'gamma' : hp.quniform('gamma', 0, 20, 1),
'tree_method' : "gpu_hist"
}
cross_val_ndgc = []
def objective_function(params):
model = xgb.XGBClassifier(**params)
for train_index, validation_index in skf.split(X, y):
X_train = train.iloc[train_index]
y_train = y.iloc[train_index]
X_test = X.iloc[validation_index]
y_test = y.iloc[validation_index].to_numpy()
self.model.fit(X_train, y_train)
validation_probs = self.model.predict_proba(X_test)
validation_preds = create_preds(validation_probs)
cross_val_ndgc.append(ndgc(validation_preds, y_test))
score = np.mean(cross_val_ndgc)
return {'loss': -score, 'status': STATUS_OK}
def bayesian_optimization():
trials = Trials()
best_param = fmin(objective_function,
param_hyperopt,
algo=tpe.suggest,
max_evals=20,
trials=trials,
rstate= np.random.RandomState(1))
loss = [x['result']['loss'] for x in trials.trials]
best_param_values = [x for x in best_param.values()]
return best_param
best_param = bayesian_optimization()
def _classification_decision_tree() -> dict:
return {
'max_depth':
scope.int(hp.quniform('max_depth', 4, 30, 1)),
'min_samples_split':
hp.choice('min_samples_split', np.arange(2, 10, dtype=int)),
'min_samples_leaf':
hp.choice('min_samples_leaf', np.arange(1, 10, dtype=int)),
}
def get_shift_space(indi, prefix):
space = {
'start': 1,
'stop':
scope.int(hp.quniform(indi + prefix + '__shift__stop', 2, 15, 1)),
'step':
1 # scope.int(hp.quniform(indi+prefix+'__shift__step', 1, 5, 1))
}
return hp.choice(indi + prefix + '__shift__params', [None, space])
def hyperopt_bayesian_optimization(X, y):
param_space = {
'max_depth': scope.int(hp.quniform('max_depth', 3, 15, 1)),
'n_estimators': scope.int(hp.quniform('n_estimators', 100, 600, 1)),
'criterion': hp.choice('criterion', ['gini', 'entropy']),
'max_features': hp.uniform('max_features', 0.01, 1)
}
optimization_function = partial(optimize, x=X, y=y)
trials = Trials()
result = fmin(fn=optimization_function,
space=param_space,
algo=tpe.suggest,
max_evals=15,
trials=trials)
print(result)
def test_hyperopt2skopt_space():
hyperopt_space = {
'int_uniform': scope.int(hp.uniform('l_int_uniform', 1, 7)),
'randint': hp.randint('l_randint', 7),
'uniform': hp.uniform('l_uniform', -3, 3),
'uniform_named': hp.uniform('l_uniform_named', low=1, high=10),
'uniform_part_named': hp.uniform('l_uniform_part_named', 1, high=10),
'unsupported': hp.loguniform('l_unsupported', -1, 5),
'choice': hp.choice('choice', ['a', 'b', 4]),
'random_param': 'just_one_val',
}
space, ind2names = hyperopt2skopt_space(hyperopt_space, sample_size=100)
assert len(space) == len(ind2names)
named_space = {ind2names[i]: space[i] for i in range(len(space))}
int_uniform = named_space['int_uniform']
assert isinstance(int_uniform, Integer)
assert int_uniform.low == 1
assert int_uniform.high == 7
randint = named_space['randint']
assert isinstance(randint, Integer)
assert randint.low == 0
assert randint.high == 7
uniform = named_space['uniform']
assert isinstance(uniform, Real)
assert uniform.low == -3
assert uniform.high == 3
uniform_named = named_space['uniform_named']
assert isinstance(uniform_named, Real)
assert uniform_named.low == 1
assert uniform_named.high == 10
uniform_part_named = named_space['uniform_part_named']
assert isinstance(uniform_part_named, Real)
assert uniform_part_named.low == 1
assert uniform_part_named.high == 10
unsupported = named_space['unsupported']
assert isinstance(unsupported, Categorical)
assert len(unsupported.categories) == 100
assert all([np.exp(-1) <= x <= np.exp(5) for x in unsupported.categories])
choice = named_space['choice']
assert isinstance(choice, Categorical)
assert set(choice.categories) == {'a', 'b', 4}
random_param = named_space['random_param']
assert isinstance(random_param, Categorical)
assert set(random_param.categories) == {'just_one_val'}
def get_default_parameter_space():
"""
Return:
dict of DistributionWrappers
"""
return {
'n_estimators':
scope.int(hp.quniform('n_estimators', 100, 2000, 200)),
'max_samples': hp.uniform('max_samples', 0.1, 1),
'contamination': hp.loguniform('contamination', -5, 0),
}
def stage_2_optimization_guidedlda(
data_vect,
seed_topics,
dictionary,
feature_names,
corpus,
param_dict={}, # add **params_optimal_stage_1 in deploy
metric_to_optimize='coherence_consistent',
stage_2_params={
'n_topics_min': 14,
'n_topics_max': 16,
'n_iter_param': 2000,
'refresh_param': 200,
'seed_confidence': 0,
'max_eval_param': 1000,
'timeout_param': 86400,
},
random_seed=420):
'''
This function
'''
# Define some values of Stage 2 optimization
# Param space for Stage 2.
param_space_stage2 = {
'n_topics':
scope.int(
hp.quniform('n_topics', stage_2_params['n_topics_min'],
stage_2_params['n_topics_max'], 1)),
}
# Stage 2 optimization with fixed optimal params from Stage 1.
result_stage_2, trials_stage_2 = optimization_hyper_params_guidedlda(
data_vect=data_vect,
seed_topics=seed_topics,
dictionary=dictionary,
feature_names=feature_names,
corpus=corpus,
param_space=param_space_stage2,
param_dict=param_dict,
n_iter_param=stage_2_params['n_iter_param'],
refresh_param=stage_2_params['refresh_param'],
seed_confidence=stage_2_params['seed_confidence'],
random_seed=random_seed,
max_eval_param=stage_2_params['max_eval_param'],
timeout_param=stage_2_params['timeout_param'],
metric_to_optimize=metric_to_optimize)
# Define optimal params values as joined dictionary from both Stages
optimal_params_result = {**param_dict, **result_stage_2}
return optimal_params_result, trials_stage_2
def get_hyperspace(combination):
# create hyperspace for optimisation (currently supports DT, RF, NN and LR).
param_hyperopt = {}
for node, clf in enumerate(combination):
if clf == 'DT':
hyper = {
'DT_criterion_' + str(node):
hp.choice('DT_criterion_' + str(node), ['gini', 'entropy']),
'DT_max_depth_' + str(node):
scope.int(hp.quniform('DT_max_depth_' + str(node), 5, 15, 1))
}
elif clf == 'RF':
hyper = {
'RF_max_depth_' + str(node):
scope.int(hp.quniform('RF_max_depth_' + str(node), 5, 15, 1)),
'RF_n_estimators_' + str(node):
scope.int(
hp.quniform('RF_n_estimators_' + str(node), 10, 50, 5))
}
elif clf == 'NN':
hyper = {
'NN_dropout_' + str(node):
hp.uniform('NN_dropout_' + str(node), 0, 0.5),
'NN_nodes_' + str(node):
scope.int(hp.quniform('NN_nodes_' + str(node), 5, 50, 5)),
'NN_layers_' + str(node):
scope.int(hp.quniform('NN_layers_' + str(node), 1, 2, 1))
}
elif clf == 'LR':
hyper = {
'LR_penalty_' + str(node):
hp.choice('LR_penalty_' + str(node), ['l1', 'l2'])
}
param_hyperopt = {**param_hyperopt, **hyper}
return param_hyperopt
