def fmin_thread_fn(space, trials, max_evals, seed):
fmin(fn=passthrough,
space=space,
algo=rand.suggest,
trials=trials,
rstate=np.random.RandomState(seed),
max_evals=max_evals,
return_argmin=False)
def fmin_thread_fn(space, trials, max_evals, seed):
fmin(
fn=passthrough,
space=space,
algo=rand.suggest,
trials=trials,
rstate=np.random.RandomState(seed),
max_evals=max_evals,
return_argmin=False)
def fmin_thread_fn(space, mongo_trials: MongoTrials, max_evals: int):
fmin(
fn=objective_with_attachments,
space=space,
algo=rand.suggest,
trials=mongo_trials,
rstate=np.random.RandomState(),
max_evals=max_evals,
return_argmin=False,
)
def test_basic(self):
bandit = self._bandit_cls()
#print 'bandit params', bandit.params, bandit
#print 'algo params', algo.vh.params
trials = Trials()
fmin(lambda x: x, bandit.expr,
trials=trials,
algo=suggest,
max_evals=self._n_steps)
assert trials.average_best_error(bandit) - bandit.loss_target < .2
def fit(self, X_train, y_train):
if self.best_score_ is None:
self.best_score_ = -float('inf')
self.X_train = X_train
self.y_train = y_train
self.estimator = sklearn.base.clone(self.estimator)
best_params = fmin(fn=self.__objective,
space=self.search_space,
algo=tpe.suggest,
max_evals=self.n_iter,
rstate=RandomState(42))
evaluated = space_eval(self.search_space, best_params)
if self.verbose:
print('best params: ', evaluated)
casted_best_params = self.__cast_params(evaluated)
self.best_params_ = casted_best_params
self.estimator.set_params(**casted_best_params)
self.estimator.fit(self.X_train, self.y_train)
return self
def fit(self, X, y, indicator):
'''
indicator=1 means we intend to do just sampling and one-time fitting
for evaluating a fixed set of hyper-parameters,
0 means run hyperopt to search in the neighborhood of the seed
hyper-parameters to see if model quality is improving.
'''
XFull = X
yFull = y
self.Xe_train, self.Xe_test, self.ys_train, self.ys_test = \
train_test_split(XFull, yFull.ravel(),test_size = self.test_size, random_state=self.seed,shuffle=True)
if indicator == 1:
## just fit lightgbm once to obtain the AUC w.r.t a fixed set of hyper-parameters ##
model = LGBMClassifier(random_state=self.seed,
min_data=1,
min_data_in_bin=1)
model.set_params(**self.param_space)
model.fit(self.Xe_train, self.ys_train)
mypreds = model.predict_proba(self.Xe_test)[:, 1]
auc = auc_metric(self.ys_test.reshape(-1, 1),
mypreds.reshape(-1, 1))
return auc
else:
trials = Trials()
best = fmin(fn=self.gbc_objective,
space=self.param_space,
algo=tpe.suggest,
trials=trials,
max_evals=self.max_evaluations)
params = space_eval(self.param_space, best)
self.best_params = params
return params, 1 - np.min([x['loss'] for x in trials.results])
def search(self):
"""
Search the parameter grid
"""
self.__start_now = datetime.datetime.now()
self.__reset()
if self.verbose > 0:
print("Starting HyperOpt Parameter Search")
best = fmin(fn=self.__objective,
space=self.__space,
algo=self.__algo,
max_evals=self.iters,
trials=self.__trial,
verbose=self.verbose,
timeout=self.time_to_search)
if self.verbose > 0:
print("Finished HyperOpt Parameter Search")
if self.best_score > self.__init_score:
self.best_params = self.__trial.best_trial["result"]["params"]
else:
self.best_params = self.__initparams
def nu_simple_fmin(hpo_project_key, objective, rseed=1337, full_model_string=None, notebook_name=None, verbose=True, stack=3, keep_temp=False, data_args=None):
# db에서 가져오기
db_info = asyncio.run(Requests().get_action(parameter1 = hpo_project_key, parameter2 = "null", url = hpo_url))[0]
hpo_project_id = db_info["hpoProjectId"]
algo, space = __transform_db_to_function(method = db_info["method"], config = db_info["config"])
trials = Trials()
best = fmin(objective, space, algo=algo, max_evals=50, trials=trials, rstate=np.random.RandomState(rseed), return_argmin=True)
importances = calculate_importance(trials)
# 저장 api
all_info = dict()
all_info["best_result"] = trials.best_trial['result']
all_info["best_hp"] = best
all_info["trial_result"] = trials.results
all_info["trial_hp"] = trials.vals
# json int64 때문에 작업
all_info = __to_int(all_info)
all_info["hpo_project_key"] = hpo_project_key
tmp_importance = list()
for i in range(len(importances)):
for key1, value1 in importances[i].items():
for key2, value2 in value1.items():
tmp_importance.append(value2)
break
all_info["importances"] = tmp_importance
asyncio.run(Requests().post_action(request_datas = all_info, url = hpo_url))
return best, trials
def nu_fmin(hpo_project_key, objective, space, algo, max_evals, trials, rseed=1337, full_model_string=None, notebook_name=None, verbose=True, stack=3, keep_temp=False, data_args=None):
best = fmin(objective, space, algo=algo, max_evals=max_evals, trials=trials, rstate=np.random.RandomState(rseed), return_argmin=True)
importances = calculate_importance(trials)
all_info = dict()
all_info["best_result"] = trials.best_trial['result']
all_info["best_hp"] = best
all_info["trial_result"] = trials.results
all_info["trial_hp"] = trials.vals
# json int64 때문에 작업
all_info = __to_int(all_info)
all_info["hpo_project_key"] = hpo_project_key
method, config = __transform_function_to_db(algo, space)
all_info["method"] = method
all_info["config"] = config
tmp_importance = list()
for i in range(len(importances)):
for key1, value1 in importances[i].items():
for key2, value2 in value1.items():
tmp_importance.append(value2)
break
all_info["importances"] = tmp_importance
asyncio.run(Requests().post_action(request_datas = all_info, url = hpo_url))
return best
def optimize(self, trials):
space = {
'n_estimators':
500,
'boosting_type':
'dart',
'num_leaves':
hp.choice('num_leaves', np.arange(20, 60, dtype=int)),
'max_depth':
hp.choice('max_depth', np.arange(1, 30, dtype=int)),
'min_data':
hp.choice('min_data', np.arange(5, 40, dtype=int)),
'max_bin':
hp.choice('max_bin', np.arange(200, 300, dtype=int)),
'rate_drop':
hp.uniform('rate_drop', 0, 1),
'skip_drop':
hp.uniform('skip_drop', 0, 1),
# 'eta': hp.quniform('eta', 0.025, 0.5, 0.025),
'eta':
hp.loguniform('eta', -3.5, -1.5),
'min_child_weight':
hp.quniform('min_child_weight', 1, 6, 1),
# 'subsample': hp.quniform('subsample', 0.5, 1, 0.05),
'subsample':
hp.uniform('subsample', 0.5, 1),
'alpha':
hp.uniform('alpha', 0, 0.5),
'lambda':
hp.uniform('lambda', 1e-4, 1),
'scale_pos_weight':
hp.choice('scale_pos_weight',
[0.1, 1., 10, 100, 1e3, 1e4, 1e5, 1e6]),
'colsample_bytree':
hp.quniform('colsample_bytree', 0.5, 1, 0.05),
# 'num_class': 2,
'objective':
'binary',
'metric':
'auc',
'nthread':
0,
'verbose':
1
}
best = fmin(self.scorer,
space,
algo=tpe.suggest,
trials=trials,
max_evals=100)
best = space_eval(space, best)
for k, v in best.items():
if k in ['max_depth', 'num_leaves', 'min_data', 'max_bin']:
v = int(v)
space[k] = v
return space
def optimize(self, trials):
space = {
'n_estimators':
1000,
'booster':
'dart',
'sample_type':
hp.choice('sample_type', ['uniform', 'weighted']),
'normalize_type':
hp.choice('normalize_type', ['tree', 'forest']),
'rate_drop':
hp.uniform('rate_drop', 0, 1),
'skip_drop':
hp.uniform('skip_drop', 0, 1),
# 'eta': hp.quniform('eta', 0.025, 0.5, 0.025),
'eta':
hp.loguniform('eta', -3.5, -1.5),
'max_depth':
hp.choice('max_depth', np.arange(1, 30, dtype=int)),
'min_child_weight':
hp.quniform('min_child_weight', 1, 6, 1),
# 'subsample': hp.quniform('subsample', 0.5, 1, 0.05),
'subsample':
hp.uniform('subsample', 0.5, 1),
'alpha':
hp.uniform('alpha', 0, 0.5),
'lambda':
hp.uniform('lambda', 1e-4, 1),
'scale_pos_weight_multiplier':
hp.choice('scale_pos_weight_multiplier',
[0, 0.1, 1., 10, 100, 1e3, 1e4, 1e5, 1e6]),
'gamma':
hp.quniform('gamma', 0.5, 1, 0.05),
'colsample_bytree':
hp.quniform('colsample_bytree', 0.5, 1, 0.05),
# 'num_class': 2,
'objective':
'binary:logistic',
'eval_metric':
'auc',
'nthread':
0,
'silent':
1
}
best = fmin(self.scorer,
space,
algo=tpe.suggest,
trials=trials,
max_evals=100)
best = space_eval(space, best)
for k, v in best.items():
if k == 'max_depth':
v = int(v)
space[k] = v
return space
def search(self):
best = fmin(fn=self.objective_model,
space=self.space,
algo=tpe.suggest,
max_evals=self.max_evals)
#Changing the Python data type of dict values
datatypes = {x[1]['label']: x[2] for x in self.params}
for key, value in best.items():
best[key] = eval(datatypes[key])(value)
return self.cv_params['estimator'].set_params(**best)
def tpe_xi(self):
"""
Tree-structured Parzen Estimator Bayesian optimisation technique
to find optimal value of xi
"""
best = fmin(
fn=lambda xi: dixons_coles(df_past).optimise_xi(xi),
space=hp.uniform(
'xi', 0.0, 1.0
), #does this create a dictionary that causes problems further down the line
algo=tpe.suggest,
max_evals=1)
return best
def main():
space = {
'ltr':
hp.choice('ltr', [True]),
'shuffle':
hp.choice('shuffle', [False]),
'num_leaves':
hp.choice('num_leaves', list(np.arange(8, 256, 2, dtype=int))),
'max_depth':
hp.choice('max_depth', list(np.arange(4, 64, 2, dtype=int))),
'max_bin':
hp.choice('max_bin', list(np.arange(255, 255 * 4, 5, dtype=int))),
'min_data_in_leaf':
hp.choice('min_data_in_leaf', list(np.arange(5, 100, 5, dtype=int))),
'learning_rate':
hp.uniform('learning_rate', 0.01, 0.3),
'bagging_fraction':
hp.uniform('bagging_fraction', 0.2, 1.0),
'feature_fraction':
hp.uniform('feature_fraction', 0.2, 1.0),
'early_stopping':
hp.uniform('test_size', 100, 1000),
}
trials_step = 1 # how many additional trials to do after loading saved trials. 1 = save after iteration
max_trials = 1 # initial max_trials. put something small to not have to wait
try: # try to load an already saved trials object, and increase the max
trials = pickle.load(
open(BASE_PATH + SET + TRAILKEY + '.hyperopt', "rb"))
print("Found saved Trials! Loading...")
max_trials = len(trials.trials) + trials_step
print("Rerunning from {} trials to {} (+{}) trials".format(
len(trials.trials), max_trials, trials_step))
except: # create a new trials object and start searching
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=max_trials)
print("Best:", best)
print("Num:", max_trials)
# save the trials object
with open(BASE_PATH + SET + TRAILKEY + ".hyperopt", "wb") as f:
pickle.dump(trials, f)
def fit(self, X, y, indicator):
'''
indicator=1 means we intend to do just sampling and one-time fitting
for evaluating a fixed set of hyper-parameters,
0 means run hyperopt to search in the neighborhood of the seed
hyper-parameters to see if model quality is improving.
'''
num_samples = len(X)
print('AutoGBT[AutoHyperOptimizer]:Total samples passed for'\
'hyperparameter tuning:',num_samples)
if num_samples > self.max_samples:
removeperc = 1.0 - (float(self.max_samples) / num_samples)
print ('AutoGBT[AutoHyperOptimizer]:Need to downsample for managing time:,'\
'I will remove data percentage',removeperc)
XFull, yFull = Utils.random_sample_in_order(
X, y.reshape(-1, 1), removeperc)
print('AutoGBT[AutoHyperOptimizer]:downsampled data length',
len(XFull))
else:
XFull = X
yFull = y
self.Xe_train, self.Xe_test, self.ys_train, self.ys_test = \
train_test_split(XFull, yFull.ravel(),test_size = self.test_size, random_state=self.seed,shuffle=True)
if indicator == 1:
## just fit lightgbm once to obtain the AUC w.r.t a fixed set of hyper-parameters ##
model = lgbm.LGBMClassifier(random_state=self.seed,
min_data=1,
min_data_in_bin=1)
model.set_params(**self.param_space)
model.fit(self.Xe_train, self.ys_train)
mypreds = model.predict_proba(self.Xe_test)[:, 1]
auc = auc_metric(self.ys_test.reshape(-1, 1),
mypreds.reshape(-1, 1))
return auc
else:
trials = Trials()
best = fmin(fn=self.gbc_objective,
space=self.param_space,
algo=tpe.suggest,
trials=trials,
max_evals=self.max_evaluations)
params = space_eval(self.param_space, best)
print('AutoGBT[AutoHyperOptimizer]:Best hyper-parameters', params)
self.best_params = params
return params, 1 - np.min([x['loss'] for x in trials.results])
def fit(self, X, y, indicator):
num_samples = len(X)
print('Total samples passed for' \
'hyperparameter tuning:', num_samples)
if num_samples > self.max_samples:
removeperc = 1.0 - (float(self.max_samples) / num_samples)
print ('Need to downsample for managing time:,' \
'I will remove data percentage', removeperc)
XFull, yFull = self.random_sample_in_order(X, y.reshape(-1, 1), removeperc)
print('downsampled data length', len(XFull))
else:
XFull = X
yFull = y
Xe_train, self.Xe_test, ys_train, self.ys_test = \
train_test_split(XFull, yFull.ravel(), test_size=self.test_size, random_state=self.seed, shuffle=True)
self.lgb_train = lgb.Dataset(Xe_train, ys_train)#, free_raw_data=True)
self.lgb_val = lgb.Dataset(self.Xe_test, self.ys_test)#, free_raw_data=True, reference=self.lgb_train)
del X
del y
del Xe_train, ys_train
gc.collect()
if indicator == 1:
## just fit lightgbm once to obtain the AUC w.r.t a fixed set of hyper-parameters ##
model = lgb.train(self.param_space, self.lgb_train, valid_sets=self.lgb_val, valid_names='eval',
verbose_eval=False, early_stopping_rounds=30, num_boost_round=100)
fpr, tpr, thresholds = metrics.roc_curve(
self.ys_test, model.predict(self.Xe_test, num_iteration=model.best_iteration))
auc = metrics.auc(fpr, tpr)
print("FIX AUC is", auc)
return auc
else:
trials = Trials()
best = fmin(fn=self.gbc_objective, space=self.param_space, algo=tpe.suggest, trials=trials,
max_evals=self.max_evaluations)
params = space_eval(self.param_space, best)
print('Best hyper-parameters', params)
self.best_params = params
model = lgb.train(params, self.lgb_train, valid_sets=self.lgb_val, valid_names='eval',
verbose_eval=False, early_stopping_rounds=30, num_boost_round=100)
fpr, tpr, thresholds = metrics.roc_curve(
self.ys_test, model.predict(self.Xe_test, num_iteration=model.best_iteration))
auc = metrics.auc(fpr, tpr)
print("Tune AUC is", auc)
return params, auc
def hyperopt_lightgbm(self, df, drop_name, params):
#X = df[]
df = df[df.label.isnull() == False][:20000]
feature_name = [i for i in df.columns if i not in (drop_name + ['label']) and 'target' not in i]
X = df[feature_name]
y = df[['label']]
X_train, X_val, y_train, y_val = self.data_split(X, y, test_size=0.5)
train_data = lgb.Dataset(X_train, label=y_train)
valid_data = lgb.Dataset(X_val, label=y_val)
space = {
#"learning_rate": hp.loguniform("learning_rate", np.log(0.03), np.log(0.1)),
"max_depth": hp.choice("max_depth", [-1, 4, 5, 6]),
"num_leaves": hp.choice("num_leaves", np.linspace(32, 128, 32, dtype=int)),
"feature_fraction": hp.quniform("feature_fraction", 0.5, 0.8, 0.1),
"bagging_fraction": hp.quniform("bagging_fraction", 0.5, 0.8, 0.1),
#"bagging_freq": hp.choice("bagging_freq", np.linspace(0, 50, 10, dtype=int)),
"reg_alpha": hp.uniform("reg_alpha", 0, 2),
"reg_lambda": hp.uniform("reg_lambda", 0, 2),
#"min_child_weight": hp.uniform('min_child_weight', 0.5, 10),
}
def objective(hyperparams):
model = lgb.train({**params, **hyperparams}, train_data, 30,
valid_data, early_stopping_rounds=30, verbose_eval=0)
score = model.best_score["valid_0"][params["metric"]]
# in classification, less is better
return {'loss': -score, 'status': STATUS_OK}
trials = Trials()
best = fmin(fn=objective, space=space, trials=trials,
algo=tpe.suggest, max_evals=20, verbose=1,
rstate=np.random.RandomState(1))
hyperparams = space_eval(space, best)
print(f"auc = {-trials.best_trial['result']['loss']:0.4f} {hyperparams}")
return hyperparams
def hypersearch(self, params, **kwargs):
# set additonal parameters which are the parameters already tunned and
# removing those that are going to be tunned
add_params = self.params.copy()
for k, v in params.items():
add_params.pop(k)
# overwrite if something is passed as kwargs (only applicable to
# add_params)
for k, v in params.items():
if k in kwargs.keys(): add_params[k] = kwargs[k]
partial_objective = lambda params: lgb_objective(
params, self.data, additional_params=add_params)
lgb_objective.i = 0
best = fmin(fn=partial_objective,
space=params,
algo=tpe.suggest,
max_evals=self.n_evals)
return best
def search_hyperparams():
start_t = time.time()
print('\n')
print('=' * 50)
space = {
'alpha': hp.uniform('alpha', 0.01, 1.0),
'epsilon': hp.uniform('epsilon', 0.01, 1.0),
}
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=N_ITER_SEARCH)
run_t_min = (time.time() - start_t) / 60.0
print(f'\nRunning time: {run_t_min:.2f} min.')
print(f'\nBest sln: {best}.')
print('=' * 50)
print('\n')
def train(time_allowed=20, trials=None, output="model.pickle"):
import xgboost as xgb
import numpy
from hyperopt import hp, tpe, Trials
from hyperopt.fmin import fmin
Y, *R, _ = get_files()
X, tf, u = gen_features(*R)
del R
dtrain = xgb.DMatrix(X, Y)
if trials is None:
trials = Trials()
param = {
'max_depth': 2,
'eta': .3,
'verbosity': 0,
'objective': 'binary:logistic',
'tree_method': 'gpu_hist',
"predictor": 'gpu_predictor'
}
num_round = 100
nfold = 7
def objective(params):
params['max_depth'] = int(params['max_depth'])
params.update(param)
result = xgb.cv(params,
dtrain,
num_round,
nfold=nfold,
metrics={'auc'},
seed=0)
score = max(result['test-auc-mean'] - result['test-auc-std'] / 2)
return -score
space = {
'max_depth': hp.quniform('max_depth', 2, 12, 1),
'colsample_bytree': hp.uniform('colsample_bytree', 0.3, 1.0),
'gamma': hp.uniform('gamma', 0.01, 0.5),
'subsample': hp.uniform('subsample', .3, 1),
'scale_pos_weight': hp.uniform('scale_pos_weight', .8, 20.0),
'eta': hp.uniform('eta', .01, .4),
}
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
trials=trials,
max_evals=len(trials.trials) + 25)
param.update(best)
param['max_depth'] = int(param['max_depth']) #fixme
print("re-cv to find early stopping")
cv = xgb.cv(param, dtrain, num_round, nfold=nfold, metrics={'auc'}, seed=0)
#num_round = (r['test-auc-mean'] - r['test-auc-std']/2).idxmax()
num_round = (
cv['test-auc-mean'] - cv['test-auc-std'] / 2 -
numpy.linspace(0, cv.shape[0] * .0003, num=cv.shape[0])).idxmax()
print("final train")
model = xgb.train(param, dtrain, num_round)
MODEL = (model, param, trials, tf, u)
print("pickle to s3")
client = boto3.client('s3')
client.put_object(Body=gzip.compress(pickle.dumps(MODEL)),
Bucket=BUCKET,
Key=output,
ContentType='application/python-pickle',
ContentEncoding='gzip')
#with open("model.pickle", "wb") as f:
# pickle.dump(MODEL, f)
return MODEL
def find_goodModel(train, target):
train = preProcessData(train)
def objective(space):
# clf = xgb.XGBRegressor(n_estimators = space['n_estimators'],
# max_depth = space['max_depth'],
# min_child_weight = space['min_child_weight'],
# subsample = space['subsample'],
# learning_rate = space['learning_rate'],
# gamma = space['gamma'],
# colsample_bytree = space['colsample_bytree'],
# objective='reg:linear'
# )
clf = RandomForestClassifier(
max_depth=space['max_depth'],
min_samples_split=space['min_samples_split'],
min_samples_leaf=space['min_samples_leaf'],
bootstrap=space['bootstrap'],
criterion=space['criterion'])
# score = A*SameLabel + B*Features +
clf.fit(train, target)
cross_mean_score = cross_val_score(estimator=clf,
X=train,
y=target,
scoring='precision_macro',
cv=3,
n_jobs=-1).mean()
result = {'loss': cross_mean_score, 'status': STATUS_OK}
print " result is ", result
return result
col_train = train.columns
bootStrapArr = [True, False]
criterionArr = ["gini", "entropy"]
space = {
'max_depth':
hp.choice('max_depth', np.arange(10, 30, dtype=int)),
'min_samples_split':
hp.choice('min_samples_split', np.arange(8, 15, dtype=int)),
'min_samples_leaf':
hp.choice('min_samples_leaf', np.arange(5, 15, dtype=int)),
'bootstrap':
hp.choice('bootstrap', bootStrapArr),
'criterion':
hp.choice('criterion', criterionArr)
}
trials = Trials()
best = fmin(
fn=objective,
space=space,
algo=tpe.suggest,
max_evals=3, # change
trials=trials)
print(best)
best['bootstrap'] = bootStrapArr[best['bootstrap']]
best['criterion'] = criterionArr[best['criterion']]
obj = {
'predictions': makePredictions(best, train, train, target),
'params': best,
'STATUS': 'OK'
}
return obj
def lgb_hyperopt(data, labels, num_evals=1000, n_folds=5, diagnostic=False):
"""
Function to turn parameters for Lightgbm
"""
LGBM_MAX_LEAVES = 2**11 #maximum number of leaves per tree for LightGBM
LGBM_MAX_DEPTH = 25 #maximum tree depth for LightGBM
EVAL_METRIC_LGBM_CLASS = 'f1'
def lgb_f1_score(y_hat, data):
y_true = data.get_label()
y_hat = np.round(y_hat)
return 'f1', f1_score(y_true, y_hat), True
print('Running {} rounds of LightGBM parameter optimisation:'.format(
num_evals))
#clear space
integer_params = [
'max_depth', 'num_leaves', 'max_bin', 'min_data_in_leaf',
'min_data_in_bin'
]
def objective(space_params):
#cast integer params from float to int
for param in integer_params:
space_params[param] = int(space_params[param])
#extract nested conditional parameters
if space_params['boosting']['boosting'] == 'goss':
top_rate = space_params['boosting'].get('top_rate')
other_rate = space_params['boosting'].get('other_rate')
#0 <= top_rate + other_rate <= 1
top_rate = max(top_rate, 0)
top_rate = min(top_rate, 0.5)
other_rate = max(other_rate, 0)
other_rate = min(other_rate, 0.5)
space_params['top_rate'] = top_rate
space_params['other_rate'] = other_rate
subsample = space_params['boosting'].get('subsample', 1.0)
space_params['boosting'] = space_params['boosting']['boosting']
space_params['subsample'] = subsample
cv_results = lgb.cv(space_params,
train,
nfold=n_folds,
stratified=True,
early_stopping_rounds=100,
seed=42,
feval=lgb_f1_score)
best_loss = -cv_results['f1-mean'][-1]
return {'loss': best_loss, 'status': STATUS_OK}
train = lgb.Dataset(data, labels)
#integer and string parameters, used with hp.choice()
boosting_list = [{
'boosting': 'gbdt',
'subsample': hp.uniform('subsample', 0.5, 1)
}, {
'boosting': 'goss',
'subsample': 1.0,
'top_rate': hp.uniform('top_rate', 0, 0.5),
'other_rate': hp.uniform('other_rate', 0, 0.5)
}] #if including 'dart', make sure to set 'n_estimators'
objective_list_reg = ['huber', 'gamma', 'fair', 'tweedie']
objective_list_class = ['binary', 'cross_entropy']
objective_list = objective_list_class
is_unbalance_list = [True]
space = {
'boosting': hp.choice('boosting', boosting_list),
'num_leaves': hp.quniform('num_leaves', 2, LGBM_MAX_LEAVES, 1),
'max_depth': hp.quniform('max_depth', 2, LGBM_MAX_DEPTH, 1),
'max_bin': hp.quniform('max_bin', 32, 255, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 256, 1),
'min_data_in_bin': hp.quniform('min_data_in_bin', 1, 256, 1),
'min_gain_to_split': hp.quniform('min_gain_to_split', 0.1, 5, 0.01),
'lambda_l1': hp.uniform('lambda_l1', 0, 5),
'lambda_l2': hp.uniform('lambda_l2', 0, 5),
'learning_rate': hp.loguniform('learning_rate', np.log(0.005),
np.log(0.2)),
'metric': None,
'objective': hp.choice('objective', objective_list),
'feature_fraction': hp.quniform('feature_fraction', 0.5, 1, 0.01),
'bagging_fraction': hp.quniform('bagging_fraction', 0.5, 1, 0.01),
'is_unbalance': hp.choice('is_unbalance', is_unbalance_list)
}
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
#fmin() will return the index of values chosen from the lists/arrays in 'space'
#to obtain actual values, index values are used to subset the original lists/arrays
#extract nested conditional parameters
try:
if best['boosting']['boosting'] == 'goss':
top_rate = best['boosting'].get('top_rate')
other_rate = best['boosting'].get('other_rate')
#0 <= top_rate + other_rate <= 1
top_rate = max(top_rate, 0)
top_rate = min(top_rate, 0.5)
other_rate = max(other_rate, 0)
other_rate = min(other_rate, 0.5)
best['top_rate'] = top_rate
best['other_rate'] = other_rate
except:
if boosting_list[best['boosting']]['boosting'] == 'goss':
top_rate = best['top_rate']
other_rate = best['other_rate']
#0 <= top_rate + other_rate <= 1
top_rate = max(top_rate, 0)
top_rate = min(top_rate, 0.5)
other_rate = max(other_rate, 0)
other_rate = min(other_rate, 0.5)
best['top_rate'] = top_rate
best['other_rate'] = other_rate
best['boosting'] = boosting_list[best['boosting']][
'boosting'] #nested dict, index twice
best['metric'] = metric_list[best['metric']]
best['objective'] = objective_list[best['objective']]
best['is_unbalance'] = is_unbalance_list[best['is_unbalance']]
#cast floats of integer params to int
for param in integer_params:
best[param] = int(best[param])
print('{' + '\n'.join('{}: {}'.format(k, v)
for k, v in best.items()) + '}')
if diagnostic:
return (best, trials)
else:
return (best)
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.svm import SVR
from hyperopt import hp, tpe
from hyperopt.fmin import fmin
X, y = datasets.load_diabetes(return_X_y=True)
X_train, X_test, y_train, y_test = train_test_split(X, y)
def objective(space):
clf = SVR(C=space['C'], epsilon=space['epsilon'])
#return -1.0 * cross_val_score(clf, X, y, scoring='neg_mean_squared_error', cv=5).mean()
return cross_val_score(clf, X, y, scoring='neg_mean_squared_error',
cv=5).mean()
space = {
'C': hp.quniform('C', 10.0, 5000.0, 10.0),
'epsilon': hp.quniform('epsilon', 0.1, 10.0, 0.5),
}
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=5,
verbose=3)
print best
def hyper_optimize(data_x,data_y,val_x=None,val_y=None,cat_labels=None,space=None,max_evals=20):
from hyperopt import tpe, hp, space_eval
from hyperopt.fmin import fmin
from hyperopt.pyll.base import scope
from sklearn.metrics import make_scorer, f1_score
from sklearn.model_selection import cross_val_score, StratifiedKFold
average="binary"
if space is not None:
if "average" in space:
average = "micro"
def f1_sklearn(truth,predictions):
if space is not None:
if "average" in space:
return -f1_score(truth,predictions)
return -f1_score(truth,predictions,average=average)
f1_scorer = make_scorer(f1_sklearn)
def objective(in_params):
clf = in_params['clf']
if clf == "rf":
clf = RandomForestClassifier(n_jobs=4,random_state=42)
elif clf == "gbm":
clf = GradientBoostingClassifier(random_state=42)
elif clf == "xgb":
from xgboost import XGBClassifier
clf = XGBClassifier(random_state=42,nthread=4)
elif clf == "cat":
from catboost import CatBoostClassifier
clf = CatBoostClassifier(random_state=42)
else:
clf = ExtraTreesClassifier(n_jobs=4,random_state=42)
if clf.__class__.__name__ == "XGBClassifier":
params = {
'n_estimators': int(in_params['n_estimators']),
'max_depth': int(in_params['max_depth']),
'eta': float(in_params['eta']),
'gamma': float(in_params['gamma']),
'colsample_bytree': float(in_params['colsample_bytree']),
'subsample': float(in_params['subsample'])
}
if "average" in in_params:
params["average"] = in_params["average"]
else:
params = {
'n_estimators': int(in_params['n_estimators']),
'max_depth': int(in_params['max_depth']),
'min_samples_split': int(in_params['min_samples_split']),
'min_samples_leaf': int(in_params['min_samples_leaf']),
'max_features': in_params['max_features']
}
clf.set_params(**params)
if val_x is None: # No fixed validation set given, perform cross-valiation on train
score = cross_val_score(clf, data_x, data_y, scoring=f1_scorer, cv=StratifiedKFold(n_splits=3), n_jobs=3).mean()
else: # validated on validation set
clf.fit(data_x,data_y)
pred_y = clf.predict(val_x)
score = -f1_score(val_y,pred_y)
if "Forest" in clf.__class__.__name__:
shortname = "RF"
elif "Cat" in clf.__class__.__name__:
shortname = "CAT"
elif "XG" in clf.__class__.__name__:
shortname = "XGB"
else:
shortname = "ET" if "Extra" in clf.__class__.__name__ else "GBM"
print("F1 {:.3f} params {} {}".format(-score, params, shortname))
return score
# For large corpora, consider raising max n_estimators up to 350
if space is None:
space = {
'n_estimators': scope.int(hp.quniform('n_estimators', 75, 250, 10)),
'max_depth': scope.int(hp.quniform('max_depth', 5, 40, 1)),
'min_samples_split': scope.int(hp.quniform('min_samples_split', 2, 10, 1)),
'min_samples_leaf': scope.int(hp.quniform('min_samples_leaf', 1, 10, 1)),
'max_features': hp.choice('max_features', ["sqrt", None, 0.5, 0.6, 0.7, 0.8]),
'clf': hp.choice('clf', ["rf","et","gbm"])
}
sys.stderr.write("o Using "+str(data_x.shape[0])+" tokens to choose hyperparameters\n")
if val_x is not None:
sys.stderr.write("o Using "+str(val_x.shape[0])+" held out tokens as fixed validation data\n")
else:
sys.stderr.write("o No validation data provided, using cross-validation on train set to score\n")
best_params = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=max_evals)
best_params = space_eval(space,best_params)
sys.stderr.write(str(best_params) + "\n")
best_clf = best_params['clf']
if best_clf == "rf":
best_clf = RandomForestClassifier(n_jobs=4,random_state=42)
elif best_clf == "gbm":
best_clf = GradientBoostingClassifier(random_state=42)
elif best_clf == "xgb":
from xgboost import XGBClassifier
best_clf = XGBClassifier(random_state=42,nthread=4)
else:
best_clf = ExtraTreesClassifier(n_jobs=4,random_state=42)
del best_params['clf']
best_clf.set_params(**best_params)
return best_clf, best_params
def quick_hyperopt(data, labels, package='lgbm', num_evals=NUM_EVALS, eval_metric='mae', diagnostic=False):
# LightGBM
if package=='lgbm':
print(f'Running {num_evals} rounds of LightGBM parameter optimisation:')
#clear space
gc.collect()
integer_params = ['max_depth', 'num_leaves', 'min_data_in_leaf', 'bagging_freq']
if eval_metric=='mae':
metric = 'mae'
elif eval_metric=='mse':
metric = 'mse'
else:
print(f'Metric {eval_metric} not found. Falling back to mae.')
metric = 'mae'
def objective(space_params):
#cast integer params from float to int
for param in integer_params:
space_params[param] = int(space_params[param])
cv_results = lgb.cv(space_params,
train,
nfold=N_FOLDS,
stratified=False,
early_stopping_rounds=200,
metrics=metric,
seed=42)
if metric=='mae':
best_loss = cv_results['l1-mean'][-1]
elif metric=='mse':
best_loss = cv_results['l2-mean'][-1]
return{'loss':best_loss, 'status': STATUS_OK }
train = lgb.Dataset(data, labels)
#integer and string parameters, used with hp.choice()
objective_list = ['huber', 'gamma', 'fair']
space ={
'boosting' : 'gbdt',
'num_leaves' : hp.quniform('num_leaves', 8, 92, 4),
'max_depth': hp.quniform('max_depth', -1, 16, 1),
'min_data_in_leaf': hp.quniform('min_data_in_leaf', 1, 100, 1),
'reg_alpha' : hp.uniform('reg_alpha', 0.1, 0.95),
'reg_lambda' : hp.uniform('reg_lambda', 0.1, 0.95),
'learning_rate' : hp.loguniform('learning_rate', np.log(0.005), np.log(0.2)),
'metric' : 'mae',
'objective' : hp.choice('objective', objective_list),
'bagging_fraction' : hp.uniform('bagging_fraction', 0.5, 0.95),
'bagging_freq': hp.quniform('bagging_freq', 3, 7, 1)
}
#optional: activate GPU for LightGBM
#follow compilation steps here:
#https://www.kaggle.com/vinhnguyen/gpu-acceleration-for-lightgbm/
#then uncomment lines below:
#space['device'] = 'gpu'
#space['gpu_platform_id'] = 0,
#space['gpu_device_id'] = 0
trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=num_evals,
trials=trials)
#fmin() will return the index of values chosen from the lists/arrays in 'space'
#to obtain actual values, index values are used to subset the original lists/arrays
best['objective'] = objective_list[best['objective']]
#cast floats of integer params to int
for param in integer_params:
best[param] = int(best[param])
print('{' + '\n'.join('{}: {}'.format(k, v) for k, v in best.items()) + '}')
if diagnostic:
return(best, trials)
else:
return(best)
else:
print('Package not recognised. Please use "lgbm" for LightGBM, "xgb" for XGBoost or "cb" for CatBoost.')
for k in params:
parameters[k] = int(params[k])
mdl = GradientBoostingRegressor(random_state=0, **parameters)
score = cross_val_score(mdl, X_train, y_train, scoring=gini_scorer, cv=5).mean()
print("Gini {:.3f} params {}".format(score, parameters))
mlflow.end_run()
return score
# need to match estimator
space = {
'n_estimators': hp.quniform('n_estimators', 10, 100, 5), # low # high # number of choices
'max_depth': hp.quniform('max_depth', 2, 4, 2)
}
best_params = fmin(fn=objective_gbr,
space=space,
algo=tpe.suggest,
max_evals=5)
for key in best_params.keys():
if int(best_params[key]) == best_params[key]:
best_params[key] = int(best_params[key])
print("Hyperopt estimated optimum {}".format(best_params))
else:
best_params = {
'n_estimators': 25,
'max_depth': 2
}
reg = GradientBoostingRegressor(random_state=0, **best_params)
learning_rate=0.05,
n_estimators=900,
random_state=9,
**params)
print(X_sub_check.shape)
sc = cv_lgboost(m, X_base=X_sub_check, y=y)
print("Score {:.3f} params {}".format(sc, params))
return sc
space = {
# 'max_depth': hp.quniform('max_depth', 2, 8, 1),
'num_leaves': hp.quniform('num_leaves', 4, 50, 1),
'feature_fraction': hp.uniform('feature_fraction', 0.005, 0.9),
'bagging_fraction': hp.uniform('bagging_fraction', 0.1, 1.0),
'lambda_l1': hp.uniform('lambda_l1', 0.1, 80)
# "min_data_in_leaf": hp.loguniform("min_data_in_leaf",1,8)
}
best_lgbm = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=80,
trials=trials)
losses = [
trials.trials[i]['result']['loss'] for i in range(len(trials.trials))
]
params = pd.DataFrame(trials.vals)
params['loss'] = losses
params.sort_values('loss', inplace=True)
params.to_csv(fpath_csv)
with open(fpath, 'wb') as f:
pickle.dump(trials, f)
def _build_lgb_model(X: pd.DataFrame, y: pd.Series, is_discrete: bool,
num_class: int, n_jobs: int,
opts: Dict[str, str]) -> Tuple[Any, float]:
import lightgbm as lgb # type: ignore[import]
def _get_option_value(*args) -> Any: # type: ignore
return get_option_value(opts, *args)
if is_discrete:
objective = "binary" if num_class <= 2 else "multiclass"
else:
objective = "regression"
fixed_params = {
"boosting_type": _get_option_value(*_opt_boosting_type),
"objective": objective,
"class_weight": _get_option_value(*_opt_class_weight),
"learning_rate": _get_option_value(*_opt_learning_rate),
"max_depth": _get_option_value(*_opt_max_depth),
"max_bin": _get_option_value(*_opt_max_bin),
"reg_alpha": _get_option_value(*_opt_reg_alpha),
"min_split_gain": _get_option_value(*_opt_min_split_gain),
"n_estimators": _get_option_value(*_opt_n_estimators),
"importance_type": _get_option_value(*_opt_importance_type),
"random_state": 42,
"n_jobs": n_jobs
}
# Set `num_class` only in the `multiclass` mode
if objective == "multiclass":
fixed_params["num_class"] = num_class
model_class = lgb.LGBMClassifier if is_discrete \
else lgb.LGBMRegressor
def _create_model(params: Dict[str, Any]) -> Any:
# Some params must be int
for k in ["num_leaves", "subsample_freq", "min_child_samples"]:
if k in params:
params[k] = int(params[k])
p = copy.deepcopy(fixed_params)
p.update(params)
return model_class(**p)
from hyperopt import hp, tpe, Trials # type: ignore[import]
from hyperopt.early_stop import no_progress_loss # type: ignore[import]
from hyperopt.fmin import fmin # type: ignore[import]
from sklearn.model_selection import ( # type: ignore[import]
cross_val_score, KFold, StratifiedKFold)
# TODO: Temporality supress `sklearn.model_selection` user's warning
import warnings
warnings.simplefilter("ignore", UserWarning)
# Forcibly disable INFO-level logging in the `hyperopt` module
from logging import getLogger, WARN
getLogger("hyperopt").setLevel(WARN)
param_space = {
"num_leaves": hp.quniform("num_leaves", 2, 100, 1),
"subsample": hp.uniform("subsample", 0.5, 1.0),
"subsample_freq": hp.quniform("subsample_freq", 1, 20, 1),
"colsample_bytree": hp.uniform("colsample_bytree", 0.01, 1.0),
"min_child_samples": hp.quniform("min_child_samples", 1, 50, 1),
"min_child_weight": hp.loguniform("min_child_weight", -3, 1),
"reg_lambda": hp.loguniform("reg_lambda", -2, 3)
}
scorer = "f1_macro" if is_discrete else "neg_mean_squared_error"
n_splits = int(_get_option_value(*_opt_n_splits))
cv = StratifiedKFold(n_splits=n_splits, shuffle=True) if is_discrete \
else KFold(n_splits=n_splits, shuffle=True)
def _objective(params: Dict[str, Any]) -> float:
model = _create_model(params)
fit_params: Dict[str, str] = {
# TODO: Raises an error if a single regressor is used
# "categorical_feature": "auto",
}
try:
# TODO: Replace with `lgb.cv` to remove the `sklearn` dependency
scores = cross_val_score(model,
X,
y,
scoring=scorer,
cv=cv,
fit_params=fit_params,
n_jobs=n_jobs)
return -scores.mean()
# it might throw an exception because `y` contains
# previously unseen labels.
except Exception as e:
_logger.warning(f"{e.__class__}: {e}")
return 0.0
def _early_stop_fn() -> Any:
no_progress_loss_fn = no_progress_loss(
int(_get_option_value(*_opt_no_progress_loss)))
timeout = int(_get_option_value(*_opt_timeout))
if timeout <= 0:
return no_progress_loss_fn
# Set base time for budget mechanism
start_time = time.time()
def timeout_fn(trials,
best_loss=None,
iteration_no_progress=0): # type: ignore
no_progress_loss, meta = no_progress_loss_fn(
trials, best_loss, iteration_no_progress)
to = time.time() - start_time > timeout
return no_progress_loss or to, meta
return timeout_fn
try:
trials = Trials()
max_evals = int(_get_option_value(*_opt_max_evals))
best_params = fmin(fn=_objective,
space=param_space,
algo=tpe.suggest,
trials=trials,
max_evals=max_evals,
early_stop_fn=_early_stop_fn(),
rstate=np.random.RandomState(42),
show_progressbar=False,
verbose=False)
_logger.info("hyperopt: #eval={}/{}".format(len(trials.trials),
max_evals))
# Builds a model with `best_params`
# TODO: Could we extract constraint rules (e.g., FD and CFD) from built statistical models?
model = _create_model(best_params)
model.fit(X, y)
def _feature_importances() -> List[Any]:
f = filter(lambda x: x[1] > 0.0,
zip(model.feature_name_, model.feature_importances_))
return list(sorted(f, key=lambda x: x[1], reverse=True))
_logger.debug(
f"lightgbm: feature_importances={_feature_importances()}")
sorted_lst = sorted(trials.trials, key=lambda x: x['result']['loss'])
min_loss = sorted_lst[0]['result']['loss']
return model, -min_loss
except Exception as e:
_logger.warning(f"Failed to build a stat model because: {e}")
return None, 0.0
results.append(loss)
error = np.mean(results)
print("INFO: iteration {} error {:.3f}".format(xl_objective.i, error))
return error
# LINEAR
partial_objective = lambda params: xl_objective(
params,
method="linear")
xl_objective.i = 0
start = time()
best_linear = fmin(
fn=partial_objective,
space=xl_parameter_space,
algo=tpe.suggest,
max_evals=10
)
end = time()-start
print("{} min".format(round(end/60,3)))
pickle.dump(best_linear, open(os.path.join(XLEARN_DIR,'best_linear.p'), "wb"))
# FM
partial_objective = lambda params: xl_objective(
params,
method="fm")
xl_objective.i = 0
start = time()
best_fm = fmin(
fn=partial_objective,
space=xl_parameter_space,
def tune(self, decoy_peaks, target_peaks, use_main_score=True):
def objective(params):
params = {
'eta': "{:.3f}".format(params['eta']),
'gamma': "{:.3f}".format(params['gamma']),
'max_depth': int(params['max_depth']),
'min_child_weight': int(params['min_child_weight']),
'subsample': "{:.3f}".format(params['subsample']),
'colsample_bytree': '{:.3f}'.format(params['colsample_bytree']),
'colsample_bylevel': '{:.3f}'.format(params['colsample_bylevel']),
'colsample_bynode': '{:.3f}'.format(params['colsample_bynode']),
'lambda': "{:.3f}".format(params['lambda']),
'alpha': "{:.3f}".format(params['alpha']),
'scale_pos_weight': "{:.3f}".format(params['scale_pos_weight']),
}
clf = xgb.XGBClassifier(random_state=42, silent=1, objective='binary:logitraw', eval_metric='auc', **params)
score = cross_val_score(clf, X, y, scoring='roc_auc', n_jobs=self.threads, cv=KFold(n_splits=3, shuffle=True, random_state=np.random.RandomState(42))).mean()
# click.echo("Info: AUC: {:.3f} hyperparameters: {}".format(score, params))
return score
click.echo("Info: Autotuning of XGB hyperparameters.")
assert isinstance(decoy_peaks, Experiment)
assert isinstance(target_peaks, Experiment)
X0 = decoy_peaks.get_feature_matrix(use_main_score)
X1 = target_peaks.get_feature_matrix(use_main_score)
X = np.vstack((X0, X1))
y = np.zeros((X.shape[0],))
y[X0.shape[0]:] = 1.0
# Tune complexity hyperparameters
xgb_params_complexity = self.xgb_params_tuned
xgb_params_complexity.update({k: self.xgb_params_space[k] for k in ('max_depth', 'min_child_weight')})
best_complexity = fmin(fn=objective, space=xgb_params_complexity, algo=tpe.suggest, max_evals=self.xgb_hyperparams['autotune_num_rounds'], rstate=np.random.RandomState(42))
best_complexity['max_depth'] = int(best_complexity['max_depth'])
best_complexity['min_child_weight'] = int(best_complexity['min_child_weight'])
self.xgb_params_tuned.update(best_complexity)
# Tune gamma hyperparameter
xgb_params_gamma = self.xgb_params_tuned
xgb_params_gamma['gamma'] = self.xgb_params_space['gamma']
best_gamma = fmin(fn=objective, space=xgb_params_gamma, algo=tpe.suggest, max_evals=self.xgb_hyperparams['autotune_num_rounds'], rstate=np.random.RandomState(42))
self.xgb_params_tuned.update(best_gamma)
# Tune subsampling hyperparameters
xgb_params_subsampling = self.xgb_params_tuned
xgb_params_subsampling.update({k: self.xgb_params_space[k] for k in ('subsample', 'colsample_bytree', 'colsample_bylevel', 'colsample_bynode')})
best_subsampling = fmin(fn=objective, space=xgb_params_subsampling, algo=tpe.suggest, max_evals=self.xgb_hyperparams['autotune_num_rounds'], rstate=np.random.RandomState(42))
self.xgb_params_tuned.update(best_subsampling)
# Tune regularization hyperparameters
xgb_params_regularization = self.xgb_params_tuned
xgb_params_regularization.update({k: self.xgb_params_space[k] for k in ('lambda', 'alpha')})
best_regularization = fmin(fn=objective, space=xgb_params_regularization, algo=tpe.suggest, max_evals=self.xgb_hyperparams['autotune_num_rounds'], rstate=np.random.RandomState(42))
self.xgb_params_tuned.update(best_regularization)
# Tune learning rate
xgb_params_learning = self.xgb_params_tuned
xgb_params_learning['eta'] = self.xgb_params_space['eta']
best_learning = fmin(fn=objective, space=xgb_params_learning, algo=tpe.suggest, max_evals=self.xgb_hyperparams['autotune_num_rounds'], rstate=np.random.RandomState(42))
self.xgb_params_tuned.update(best_learning)
click.echo("Info: Optimal hyperparameters: {}".format(self.xgb_params_tuned))
self.xgb_params = self.xgb_params_tuned
return self
'subsample_for_bin': hp.quniform('subsample_for_bin', 20000, 300000,
20000),
'min_child_samples': hp.quniform('min_child_samples', 20, 500, 5),
'reg_lambda': hp.uniform('reg_lambda', 0.0, 1.0),
'colsample_bytree': hp.uniform('colsample_by_tree', 0.6, 1.0)
}
global ITERATION
ITERATION = 0
df = pd.read_pickle('feature.pkl')
train_df = df[df['TARGET'].notnull()]
test_df = df[df['TARGET'].isnull()]
print("Starting LightGBM. Train shape: {}, test shape: {}".format(
train_df.shape, test_df.shape))
feats = [
f for f in train_df.columns if f not in
['TARGET', 'SK_ID_CURR', 'SK_ID_BUREAU', 'SK_ID_PREV', 'index']
]
bayes_trials = Trials()
best = fmin(fn=objective,
space=space,
algo=tpe.suggest,
max_evals=10000,
trials=bayes_trials,
rstate=np.random.RandomState(50))
bayes_trials_results = sorted(bayes_trials.results, key=lambda x: x['loss'])
bayes_trials_results[:2]
return 1-score
lgb_parameter_space = {
'learning_rate': hp.uniform('learning_rate', 0.01, 0.5),
'num_boost_round': hp.quniform('num_boost_round', 20, 100, 5),
'num_leaves': hp.quniform('num_leaves', 32,256,4),
'min_child_weight': hp.quniform('min_child_weight', 1, 50, 2),
'colsample_bytree': hp.uniform('colsample_bytree', 0.5, 1.),
'subsample': hp.uniform('subsample', 0.5, 1.),
'reg_alpha': hp.uniform('reg_alpha', 0.01, 1.),
'reg_lambda': hp.uniform('reg_lambda', 0.01, 1.),
}
lgb_objective.i = 0
best = fmin(fn=lgb_objective,
space=lgb_parameter_space,
algo=tpe.suggest,
max_evals=10)
best['num_boost_round'] = int(best['num_boost_round'])
best['num_leaves'] = int(best['num_leaves'])
best['verbose'] = -1
# Read the validation coupon features
df_coupons_valid_feat = pd.read_pickle(os.path.join(inp_dir, 'valid', 'df_coupons_valid_feat.p'))
df_coupons_valid_cat_feat = df_coupons_valid_feat.drop(drop_cols, axis=1)
# Read the interactions during validation
interactions_valid_dict = pickle.load(
open("../datasets/Ponpare/data_processed/valid/interactions_valid_dict.p", "rb"))
# Take the 358 validation coupons and the 6070 users seen in training and during
