def main(): usage = "%prog" parser = OptionParser(usage=usage) parser.add_option('-o', dest='output_dirname', default='bayes_opt_rnn_chars', help='Output directory name') parser.add_option('--reuse', dest='reuse', action="store_true", default=False, help='Use reusable holdout; default=%default') (options, args) = parser.parse_args() global output_dirname, output_filename, reuse, search_alpha, space reuse = options.reuse output_dirname = options.output_dirname if reuse: output_dirname += '_reuse' output_filename = fh.make_filename(defines.exp_dir, fh.get_basename_wo_ext(output_dirname), 'log') with codecs.open(output_filename, 'w') as output_file: output_file.write(output_dirname + '\n') #output_file.write('reuse = ' + str(reuse) + '\n') trials = Trials() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=100, trials=trials) print space_eval(space, best) print trials.losses()
def optimize_model_pytorch(device, args, train_GWAS, train_y, test_GWAS, test_y, out_folder ="", startupJobs = 40, maxevals = 200, noOut = False): global numTrials_pytorch numTrials_pytorch= 0 trials = Trials() trial_wrapper = partial(trial_pytorch,device = device, args = args , train_GWAS = train_GWAS, train_y = train_y , test_GWAS = test_GWAS , test_y = test_y) best_pars = fmin(trial_wrapper, parameter_space_pytorch(), algo=partial(tpe.suggest, n_startup_jobs=(startupJobs) ), max_evals=maxevals, trials=trials) # Print the selected 'best' hyperparameters. if noOut == False: print('\nBest hyperparameter settings: ',space_eval(parameter_space_pytorch(), best_pars),'\n') # loops through the 1st entry in the dict that holds all the lookup keys regression = True for p in trials.trials[0]['misc']['idxs']: plot_optimization_pytorch(trials, p, regression, out_folder = out_folder) best_pars = space_eval(parameter_space_pytorch(), best_pars) # this turns the indices into the actual params into the valid aprameter space # override the epochs with the early start lowestLossIndex = np.argmin(trials.losses()) trials.trial_attachments(trials.trials[lowestLossIndex])['highestAcc_epoch'] best_pars['earlyStopEpochs'] = trials.trial_attachments(trials.trials[lowestLossIndex])['highestAcc_epoch'] best_pars['earlyStopEpochs'] += 1 # as epochs are 0 based otherwise... best_pars['epochs'] = best_pars['earlyStopEpochs'] if best_pars['epochs'] <= 0 : best_pars['epochs'] = 1 # we dont want a network without any training, as that will cause a problem for deep dreaming return(best_pars)
def test_landing_screen(): # define an objective function def objective(args): case, val = args if case == 'case 1': return val else: return val ** 2 # define a search space from hyperopt import hp space = hp.choice('a', [ ('case 1', 1 + hp.lognormal('c1', 0, 1)), ('case 2', hp.uniform('c2', -10, 10)) ]) # minimize the objective over the space import hyperopt best = hyperopt.fmin(objective, space, algo=hyperopt.tpe.suggest, max_evals=100) print best # -> {'a': 1, 'c2': 0.01420615366247227} print hyperopt.space_eval(space, best)
def main(): usage = "%prog text.json labels.csv feature_dir output_dir" parser = OptionParser(usage=usage) parser.add_option('-m', dest='max_iter', default=4, help='Maximum iterations of Bayesian optimization; default=%default') (options, args) = parser.parse_args() max_iter = int(options.max_iter) global data_filename, label_filename, feature_dir, output_dir, log_filename data_filename = args[0] label_filename = args[1] feature_dir = args[2] output_dir = args[3] if not os.path.exists(output_dir): os.makedirs(output_dir) log_filename = os.path.join(output_dir, 'log.txt') with open(log_filename, 'w') as logfile: logfile.write(','.join([data_filename, label_filename, feature_dir, output_dir])) trials = Trials() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=max_iter, trials=trials) print space_eval(space, best) print trials.losses()
def main(args): print ("Made it to the start of main!") print ("the time at the start: " + str(time.time())) set_globals(args) trials = Trials() if args["run_bayesopt"]: space = space_manager.get_space(num_models, model_types, search_space) # DEBUGGING profile = cProfile.Profile() try: profile.enable() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=max_iter, trials=trials) profile.disable() finally: profile = pstats.Stats(profile).sort_stats("cumulative") profile.print_stats() print space_eval(space, best) printing_best(trials) # loading models from file else: with open(model_path) as f: for i in range(line_num - 1): f.readline() space = eval(f.readline()) best = call_experiment(space)
def test_space_eval(): space = hp.choice('a', [ ('case 1', 1 + hp.lognormal('c1', 0, 1)), ('case 2', hp.uniform('c2', -10, 10)) ]) assert space_eval(space, {'a': 0, 'c1': 1.0}) == ('case 1', 2.0) assert space_eval(space, {'a': 1, 'c2': 3.5}) == ('case 2', 3.5)
def main(): set_globals() trials = Trials() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=max_iter, trials=trials) print space_eval(space, best) print "losses:", [-l for l in trials.losses()] print('the best loss: ', max([-l for l in trials.losses()])) print("number of trials: " + str(len(trials.trials)))
def run(self): start = time.time() trials = Trials() best = fmin(self._obj, self.model_param_space._build_space(), tpe.suggest, self.max_evals, trials) best_params = space_eval(self.model_param_space._build_space(), best) best_params = self.model_param_space._convert_int_param(best_params) trial_rmses = np.asarray(trials.losses(), dtype=float) best_ind = np.argmin(trial_rmses) best_rmse_mean = trial_rmses[best_ind] best_rmse_std = trials.trial_attachments(trials.trials[best_ind])["std"] self.logger.info("-"*50) self.logger.info("Best RMSE") self.logger.info(" Mean: %.6f"%best_rmse_mean) self.logger.info(" std: %.6f"%best_rmse_std) self.logger.info("Best param") self.task._print_param_dict(best_params) end = time.time() _sec = end - start _min = int(_sec/60.) self.logger.info("Time") if _min > 0: self.logger.info(" %d mins"%_min) else: self.logger.info(" %d secs"%_sec) self.logger.info("-"*50)
def xgb3(train2, y, test2, v, z): cname = sys._getframe().f_code.co_name N_splits = 9 N_seeds = 4 from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = xgb.DMatrix(train2, y) def step_xgb(params): cv = xgb.cv(params=params, dtrain=dtrain, num_boost_round=10000, early_stopping_rounds=100, nfold=10, seed=params['seed']) score = cv.ix[len(cv)-1, 0] print(cname, score, len(cv), params) return dict(loss=score, status=STATUS_OK) space_xgb = dict( max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0, 10, 0.05), alpha = hp.quniform('alpha', 0.0, 1, 0.0001), objective = 'binary:logistic', eval_metric = 'logloss', seed = 1, silent = 1 ) trs = load_state(cname + '_trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin)) for n in range(25): best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) save_state(cname + '_trials', (tr, space_xgb)) xgb_params = space_eval(space_xgb, best) print(xgb_params) xgb_common(train2, y, test2, v, z, N_seeds, N_splits, cname, xgb_params)
def tune(estimator_class, X_train, y_train, space, eval_fun, cv=3, n_jobs=1, algo=tpe.suggest, max_evals=100): if (type(cv) == int): cv = KFold(len(X_train), n_folds=cv) def hp_eval(params): estimator = estimator_class(**params) return cross_val(estimator, X_train, y_train, cv, eval_fun, n_jobs) best = fmin(hp_eval, space, algo=algo, max_evals=max_evals) best_params = space_eval(space, best) return best_params
def eval_hyperopt_space(space, vals): """ Evaluate a set of parameter values within the hyperopt space. Optionally unpacks the values, if they are wrapped in lists. :param space: dict the hyperopt space dictionary :param vals: dict the values from a hyperopt trial :return: evaluated space """ unpacked_vals = unpack_hyperopt_vals(vals) return space_eval(space, unpacked_vals)
def main(args): print("Made it to the start of main!") set_globals(args) trials = Trials() if args['run_bayesopt']: space = space_manager.get_space(num_models, model_types, search_space) best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=max_iter, trials=trials) print space_eval(space, best) printing_best(trials) #loading models from file else: with open(model_path) as f: for i in range(line_num - 1): f.readline() space = eval(f.readline()) best = call_experiment(space)
def list_best(mongo, exp_key=None, space=None): mongo_trials = MongoTrials(mongo, exp_key=exp_key) jobs = mongo_trials.trials jobs_ok = [ (d['result']['loss'], d) for d in jobs if d['state'] == 2 and d['result']['status'] == 'ok'] jobs_ok.sort() for loss, job in reversed(jobs_ok): print(loss, job['owner'], job['result']) spec = spec_from_misc(job['misc']) print("spec: {}".format(spec)) if space is not None: print("space: {}".format(space_eval(space, spec))) print("total: {}/{}".format(len(jobs_ok), len(jobs))) return mongo_trials.argmin
def run(self,max_evals=100): trials = Trials() best = fmin(self.call_experiment, space=space, algo=tpe.suggest, max_evals=max_evals, trials=trials) #print best args = space_eval(space, best) #print "losses:", [-l for l in trials.losses()] #print max([-l for l in trials.losses()]) #TODO #return model and featurizer. Don't forget to set ascii option. featurizer = Featurizer(args) if args['text']['text'] == 'ascii': featurizer.set_ascii(True) return self.get_model(args),featurizer
'*****************************************************************************************' ) for k in range(n_folds): print('Fold ' + str(k + 1)) space['verbose'] = True root_mean_squared_error_k, mape_k, best_params_k = evaluate_on_test_set( space, results[k], test[k], objective, k, n_folds) root_mean_squared_error.append(root_mean_squared_error_k) mape.append(mape_k) best.append(best_params_k) ################################################################################################################ # Save gait specific metrics df_list = [] for k in range(n_folds): params = space_eval(space, results[k]) params['sample_ids'] = test[k] params['metric'] = 'get_res' params['verbose'] = False df = objective(params) df_list.append(df) df_for_task_j = pd.concat(df_list) df_gait_measures_by_task.append(df_for_task_j) ################################################################################################################ # Save results save_results = dict() save_results['best'] = best save_results['rmse'] = root_mean_squared_error save_results['mape'] = mape save_results['train'] = train
'max_depth': hp.choice("max_depth", range(1, 10, 1)), 'min_child_weight': hp.choice('min_child_weight', range(150, 171)), 'subsample': hp.uniform('subsample', 0.7, 1), 'colsample_bytree': hp.uniform('colsample_bytree', 0.7, 1), 'gamma': hp.uniform('gamma', 0.1, 0.5), 'reg_alpha': hp.uniform('reg_alpha', 5e-5, 5e-4) } trials = Trials() best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=500, trials=trials) best_params = space_eval(space, best) print(best_params) # Test on the test set NUM_TRIALS = int(np.ceil(200000 / train.shape[0])) print('Test {} times on the test set'.format(NUM_TRIALS)) accuracy_array = [] for i in range(NUM_TRIALS): xgb = XGBClassifier(**best_params, learning_rate=0.01, objective='binary:logistic', n_jobs=-1, scale_pos_weight=1, seed=i) model = xgb.fit(X_train, y_train) y_pred = model.predict(X_test) # evaluate predictions
def minimize(model, data, algo, max_evals, trials, functions=None, rseed=1337, notebook_name=None, verbose=True, eval_space=False, return_space=False): """ Minimize a keras model for given data and implicit hyperparameters. Parameters ---------- model: A function defining a keras model with hyperas templates, which returns a valid hyperopt results dictionary, e.g. return {'loss': -acc, 'status': STATUS_OK} data: A parameter-less function that defines and return all data needed in the above model definition. algo: A hyperopt algorithm, like tpe.suggest or rand.suggest max_evals: Maximum number of optimization runs trials: A hyperopt trials object, used to store intermediate results for all optimization runs rseed: Integer random seed for experiments notebook_name: If running from an ipython notebook, provide filename (not path) verbose: Print verbose output eval_space: Evaluate the best run in the search space such that 'choice's contain actually meaningful values instead of mere indices return_space: Return the hyperopt search space object (e.g. for further processing) as last return value Returns ------- If `return_space` is False: A pair consisting of the results dictionary of the best run and the corresponding keras model. If `return_space` is True: The pair of best result and correspondeing keras model, and the hyperopt search space """ best_run, space = base_minimizer(model=model, data=data, functions=functions, algo=algo, max_evals=max_evals, trials=trials, rseed=rseed, full_model_string=None, notebook_name=notebook_name, verbose=verbose) best_model = None for trial in trials: vals = trial.get('misc').get('vals') for key in vals.keys(): vals[key] = vals[key][0] if trial.get('misc').get('vals') == best_run and 'model' in trial.get('result').keys(): best_model = trial.get('result').get('model') if eval_space is True: # evaluate the search space best_run = space_eval(space, best_run) if return_space is True: # return the space as well return best_run, best_model, space else: # the default case for backwards compatibility with expanded return arguments return best_run, best_model
return -np.mean( cross_val_score(model, words, train_y, cv=3, scoring="f1_macro")) # Define a search space - Logistic regression so lets search for a value of C and which penalty to use. # For full list of configuration options see http://hyperopt.github.io/hyperopt/getting-started/search_spaces/ space = { "C": hp.uniform("C", 0, 10), "penalty": hp.choice("penalty", ["l2", "none"]), "word_rep": hp.choice("word_rep", ["tfidf", "tf", "bow"]), } # minimize the objective over the space best = fmin(objective, space, algo=tpe.suggest, max_evals=5000) print(hyperopt.space_eval(space, best)) # %% # Create model with best hyperparameters seen above. Need to manually select test representation args = hyperopt.space_eval(space, best) train_words = word_reps[args.pop("word_rep")] test_words = test_tfidf model = model_type(**args) # Fit to training data model.fit(train_words, train_y) # Print metrics print(
def fit(self, features, labels): X_train, X_test, y_train, y_test = train_test_split(features, labels, test_size=0.2, random_state=42) plt.figure(figsize=(18, 7 * len(self.classifiers))) i = 1 for classifier in self.classifiers: instance = clone(self.models[classifier]) self.results.append({ 'Name': classifier, 'Params': instance.get_params(), 'Score': cross_val_score(instance, features, labels, cv=4, scoring='roc_auc').mean() }) if self.search: instance = clone(self.models[classifier]) objective = partial(self.objective, model=instance, features=features, labels=labels) trials = Trials() best = fmin(fn=objective, space=self.space[classifier], trials=trials, algo=tpe.suggest, max_evals=self.max_evals) instance.set_params(**self.fixQuniform( space_eval(self.space[classifier], best))) self.results.append({ 'Name': classifier + '_Tuned', 'Params': instance.get_params(), 'Score': -trials.best_trial['result']['loss'], }) instance.fit(X_train, y_train) predict = [i[1] for i in instance.predict_proba(X_test)] fpr, tpr, _ = roc_curve(y_test, predict) precision, recall, _ = precision_recall_curve(y_test, predict) plt.subplot(len(self.classifiers), 2, i).plot(fpr, tpr) plt.plot([0, 1], [0, 1], ls='--') plt.title(classifier + ' Receiver Operating Characteristic Curve') plt.xlabel('False Positive Rate') plt.ylabel('True Positive Rate') plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) i += 1 plt.subplot(len(self.classifiers), 2, i).plot(recall, precision) plt.title(classifier + ' Precision Recall Curve') plt.xlabel('Recall') plt.ylabel('Precision') plt.xlim([-0.05, 1.05]) plt.ylim([-0.05, 1.05]) i += 1 try: features_temp = pd.DataFrame({ classifier: features.columns, 'importance': instance.coef_[0] if classifier == 'LR' else instance.feature_importances_ }) features_temp = features_temp.sort_values( 'importance', ascending=False).head(min(20, len(features_temp))) features_temp = features_temp.reset_index(drop=True) self.feature_importance.append(features_temp) except: print('no feature name exist') plt.savefig(self.imgbuffer, format='png')
def run(train, y, test, v, z): #cname = sys._getframe().f_code.co_name cname = 'p' train.drop('id', axis=1, inplace=True) test.drop('id', axis=1, inplace=True) from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = xgb.DMatrix(train, y) def step_xgb(params): cv = xgb.cv(params=params, dtrain=dtrain, num_boost_round=10000, early_stopping_rounds=50, nfold=10, seed=params['seed']) score = cv.ix[len(cv)-1, 0] print(cname, score, len(cv), params) return dict(loss=score, status=STATUS_OK) space_xgb = dict( max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0.5, 10, 0.05), objective = 'binary:logistic', eval_metric = 'logloss', seed = 1, silent = 1 ) trs = state.load('xgb_trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_xgb, tr.argmin)) best = tr.argmin while len(tr.trials) < 15: best = fmin(step_xgb, space_xgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) state.save('xgb_trials', (tr, space_xgb)) xgb_params = space_eval(space_xgb, best) print(xgb_params) N_splits = 9 N_seeds = 1 skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True) dtest = xgb.DMatrix(test) for s in range(N_seeds): scores = [] cname2 = cname + str(s) v[cname2], z[cname2] = 0, 0 xgb_params['seed'] = s + 4242 for n, (itrain, ival) in enumerate(skf.split(train, y)): dtrain = xgb.DMatrix(train.ix[itrain], y[itrain]) dvalid = xgb.DMatrix(train.ix[ival], y[ival]) watch = [(dtrain, 'train'), (dvalid, 'valid')] clf = xgb.train(xgb_params, dtrain, 10000, watch, early_stopping_rounds=100, verbose_eval=False) p = clf.predict(dvalid) v.loc[ival, cname2] += p score = metrics.log_loss(y[ival], p) z[cname2] += clf.predict(dtest) print(cname, 'seed %d step %d of %d: '%(xgb_params['seed'], n+1, skf.n_splits), score, state.now()) scores.append(score) z[cname2] /= N_splits cv = scores z['y'] = z[cname2] print('validation loss: ', cv, np.mean(cv), np.std(cv)) return cv, None
def main(): usage = "%prog <DRLD|MOLD|MIP|Primary|General|PK-...>" parser = OptionParser(usage=usage) parser.add_option('-m', dest='model', default='basic', help='Model: (basic|GRU|LSTM); default=%default') parser.add_option('-o', dest='output_dirname', default='bayes_opt_rnn_mod', help='Output directory name') parser.add_option('--reuse', dest='reuse', action="store_true", default=False, help='Use reusable holdout; default=%default') parser.add_option('--mod', dest='mod', action="store_true", default=False, help='Use modifications; default=%default') (options, args) = parser.parse_args() global output_dirname, output_filename, reuse, search_alpha, space, mod, dataset reuse = options.reuse output_dirname = options.output_dirname model = options.model mod = options.mod dataset = args[0] if model == 'basic': space['arch']['unit'] = 'basic' space['arch']['n_hidden'] = hp.quniform('n_hidden', 5, 200, 5) space['training']['learning_rate'] = hp.loguniform('learning_rate', -4, -1), elif model == 'GRU': space['arch']['unit'] = 'GRU' space['arch']['n_hidden'] = hp.quniform('n_hidden', 5, 150, 5) space['training']['learning_rate'] = hp.loguniform('learning_rate', -5, -1.5), elif model == 'LSTM': space['arch']['unit'] = 'LSTM' space['arch']['n_hidden'] = hp.quniform('n_hidden', 5, 100, 5) space['training']['learning_rate'] = hp.loguniform('learning_rate', -5, -1.5), else: sys.exit('Model not supported!') output_dirname += '_' + model if reuse: output_dirname += '_reuse' if mod: output_dirname += '_mod' output_dirname += '_' + dataset output_filename = fh.make_filename(defines.exp_dir, fh.get_basename(output_dirname), 'log') with codecs.open(output_filename, 'w') as output_file: output_file.write(output_dirname + '\n') #output_file.write('reuse = ' + str(reuse) + '\n') trials = Trials() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=60, trials=trials) print space_eval(space, best) print trials.losses()
} ]) score_key = 'score' <<<<<<< HEAD for i, df in enumerate(self.dfs): df.drop(self.colTypes['Identity'], axis=1, inplace=True) #Dropping Identity cols self.x_train, self.x_test, self.y_train, self.y_test = train_test_split(df.drop(self.y, axis=1), df[self.y], test_size=self.test_size) trials = Trials() ======= df=self.df.copy() >>>>>>> 7db86ccf9079027446285fcbf67bdc0a735f8a71 df.drop(self.colTypes['Identity'], axis=1, inplace=True) #Dropping Identity cols self.x_train, self.x_test, self.y_train, self.y_test = train_test_split(df.drop(self.y, axis=1), df[self.y], test_size=self.test_size) trials = Trials() hyperparam = space_eval(self.space, fmin(self.objective_func, self.space, trials=trials, algo=tpe.suggest, max_evals=100)) score = -min(trials.losses()) self.final_results['Hyperparameter']=hyperparam self.final_results[score_key]=score self.final_results['Data']=df def return_results(self): return self.final_results
if args.mode == "train": # Just train with given parameters res = optimize(args.lr, args.clip) else: assert (args.mode == "hyper") space = [ hp.loguniform('lr', -12, -2), hp.choice('clip_decision', [-1, hp.uniform('clip', 0.01, 1.0)]) ] best = fmin(fn=lambda args: optimize(args[0], args[1]), space=space, algo=tpe.suggest, max_evals=args.hyper_iter) print("Train model with best settings...") best = hyperopt.space_eval(space, best) res = optimize(best[0], best[1]) print("Reached validation loss of " + str(res["loss"]) + " with parameters:") print(best) # Load the best saved model. if res["status"] != "fail": with open(res["model_name"], 'rb') as f: model = torch.load(f) # Run on train data train_loss, train_acc = evaluate(train_x, train_y) print('-' * 89) print('| End of training | train loss {:5.2f} | train acc {:8.2f}'. format(train_loss, train_acc))
def run(train, y, test, v, z): np.random.seed(1) #cname = sys._getframe().f_code.co_name train = train.values test = test.values from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval space_stack = hp.choice('stacking by', [ dict(type='BayesianRidge'), dict(type='Lars'), dict(type='LinearRegression'), dict(type='Ridge'), dict(type='SGDRegressor', random_state=1), dict( type='XGBRegressor', max_depth=hp.choice('max_depth', range(2, 8)), subsample=hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree=hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate=hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight=hp.quniform('min_child_weight', 1, 6, 1), gamma=hp.quniform('gamma', 0, 10, 0.05), reg_alpha=hp.quniform('alpha', 0, 1, 0.0001), ), ]) def get_lr(params): t = params['type'] del params['type'] if t == 'BayesianRidge': lr = linear_model.BayesianRidge(**params) elif t == 'Lars': lr = linear_model.Lars(**params) elif t == 'LinearRegression': lr = linear_model.LinearRegression(**params) elif t == 'Ridge': lr = linear_model.Ridge(**params) elif t == 'SGDRegressor': lr = linear_model.SGDRegressor(**params) elif t == 'XGBRegressor': lr = xgb.XGBRegressor(**params) return lr def step(params): print(params, end=' ') cv = model_selection.cross_val_score(get_lr(params), train, y, cv=10, scoring=metrics.make_scorer( metrics.log_loss)) score = np.mean(cv) print(score) return dict(loss=score, status=STATUS_OK) trs = state.load('trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: best = tr.argmin print('reusing %d trials, best was:' % (len(tr.trials)), space_eval(space_stack, best)) mt = max(50, len(tr.trials) + 1) while len(tr.trials) < min(50, mt): best = fmin(step, space_stack, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials=tr) state.save('trials', (tr, space_stack)) params = space_eval(space_stack, best) print('best params:', params) lr = get_lr(params) cv = model_selection.cross_val_score(lr, train, y, cv=10, scoring=metrics.make_scorer( metrics.log_loss)) lr.fit(train, y) z['p'] = np.clip(lr.predict(test), 1e-5, 1 - 1e-5) z['y'] = z['p'] v['p'] = model_selection.cross_val_predict(lr, train, y, cv=10) print('cv:', np.mean(cv), np.std(cv)) return cv, None
The objective function gets curried and passed to the trials object with the data loaded :param multiplier: float uses to scale the objective function :return: objective funciton """ def objective_curried(args): case, val = args log.debug("Arguments parsed {}".format(args)) if case == 'case 1': return val else: return val**multiplier return objective_curried object_curried = objective_currier(multiplier) log.info("Starting Optimisation") # Run optimisation best = fmin(fn=object_curried, space=space, algo=tpe.suggest, max_evals=num_eval_steps, trials=trials, verbose=1) log.info("Finished Optimisation") # Get the values of the best space best_space = space_eval(space, best) log.info("Best value in optimisation is : " + str(best_space))
space_dnf = (hp.uniform('tau_dt', 0.0001, 1), hp.uniform('h', -1, 0), hp.uniform('gi', 0, 10), hp.uniform('excitation_amplitude', 0.0001, 5), hp.uniform('excitation_sigma', 0.0001, 1)) best = fmin(evaluate, space_dnf, algo=tpe.suggest, trials=tpe_trials, max_evals=700) full_results = pd.DataFrame({ 'loss': [x['loss'] for x in tpe_trials.results], 'iteration': tpe_trials.idxs_vals[0]['tau_dt'], 'tau_dt': tpe_trials.idxs_vals[1]['tau_dt'], 'h': tpe_trials.idxs_vals[1]['h'], 'gi': tpe_trials.idxs_vals[1]['gi'], 'excitation_amplitude': tpe_trials.idxs_vals[1]['excitation_amplitude'], 'excitation_sigma': tpe_trials.idxs_vals[1]['excitation_sigma'] }) full_results.to_csv(os.path.join(output_path, "dnf_cdnet.csv")) print(best) print(space_eval(space_dnf, best))
models_to_test = ['lstm', 'dense', 'embedding', 'bidi'] """datasets_helper.next_dataset() space = create_base_params('lstm',datasets_helper) smpl = sample(space) print(sample(space))""" for model in models_to_test: while datasets_helper.next_dataset(): space = create_base_params(model, datasets_helper, results_saver) best = fmin(optimize_model, space=space, algo=tpe.suggest, max_evals=30, max_queue_len=1, verbose=False) results_saver.add_log( 'Best params for network type {} and dataset {} are: {}\n{}'. format(model, datasets_helper.get_dataset_name(), best, space_eval(space, best))) results_saver.write_any('best_params', [ model, datasets_helper.get_dataset_name(), space_eval(space, best) ], 'a') #results_saver.write_2D_list([[model,datasets_helper.get_dataset_name(),best]],'best_params','a') datasets_helper.reset_dataset_counter() """best_run, best_model = optim.minimize(model=test, data=[], algo=tpe.suggest, max_evals=5, trials=Trials())"""
#name = model + '_' + regularizer + '_' + '_'.join(feature_list) + '_' + str(hyperparams[0]) name = model + '_' + regularizer + '_' + '_'.join(feature_list) datasets = ['Democrat-Likes', 'Democrat-Dislikes', 'Republican-Likes', 'Republican-Dislikes'] print name return experiment.run_group_experiment(name, datasets, 0, feature_list, model_type=model, regularizer=regularizer) trials = Trials() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=3, trials=trials) #rseed=np.random.randint(1, 4294967295) #print best print space_eval(space, best) print trials.losses() #for trial in trials.trials: # print trial #run_group_experiment('profile_test', ['Democrat-Likes', 'Democrat-Dislikes', 'Republican-Likes', 'Republican-Dislikes'], # 0, ['ngrams'], model_type='SVM') #experiment_new.main()
def test_space_eval(): space = hp.choice("a", [("case 1", 1 + hp.lognormal("c1", 0, 1)), ("case 2", hp.uniform("c2", -10, 10))]) assert space_eval(space, {"a": 0, "c1": 1.0}) == ("case 1", 2.0) assert space_eval(space, {"a": 1, "c2": 3.5}) == ("case 2", 3.5)
def optimize(self, input_data=None): X, Y = input_data # X = X.reshape(X.shape[0], X.shape[1], 1) in_shape = X.shape space = self.architecture.get_space() def get_calls(partition): from keras import callbacks as C calls = list() # calls.append( C.ModelCheckpoint(save_dir +'/'+'-partition_{}'.format(partition)+'-epoch_{epoch:02d}-weights.h5', save_best_only=True, save_weights_only=True, verbose=1) ) # calls.append( C.CSVLogger(args.save_dir + '/log.csv') ) # calls.append( C.TensorBoard(log_dir=args.save_dir + '/tensorboard-logs/{}'.format(actual_partition), batch_size=args.batch_size, histogram_freq=args.debug) ) calls.append( C.EarlyStopping(monitor='val_loss', patience=5, verbose=0) ) # calls.append( C.ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=2, min_lr=0.0001, verbose=0) ) calls.append( C.LearningRateScheduler(schedule=lambda epoch: 0.001 * (0.9 ** epoch)) ) # calls.append( C.LearningRateScheduler(schedule=lambda epoch: 0.001 * np.exp(-epoch / 10.)) ) # calls.append( CyclicLR(mode='exp_range', gamma=0.99994) ) # calls.append( CyclicLR(mode='triangular2') ) calls.append( ModelCheckpoint('best-weights.h5', monitor='val_loss', save_best_only=True, save_weights_only=True) ) return calls def f(hp_params): try: nsplits = 2 cnt1 = 0 scores, score = [], None seed = 123 if len(trials.trials)>1: for x in trials.trials[:-1]: space_point_index = dict([(key,value[0]) for key,value in x['misc']['vals'].items() if len(value)>0]) peval = space_eval(space,space_point_index) if hp_params == peval: # print('>>> Repeated Evaluation') loss = x['result']['loss'] return {'loss':loss, 'status':STATUS_FAIL} self.evaluations += 1 print('Testing parameters (eval {})'.format(self.evaluations)) print(hp_params) # Folds for train - test (Evaluate Model) folds1 = StratifiedShuffleSplit(n_splits=nsplits, random_state=seed) for train_index, test_index in folds1.split(X, Y): X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] cnt1 += 1 # Folds for train - validation (Guide training phase) folds2 = StratifiedShuffleSplit(n_splits=1, random_state=17, test_size=0.05) for t_index, v_index in folds2.split(X_train, Y_train): x_train, x_val = X_train[t_index], X_train[v_index] y_train, y_val = Y_train[t_index], Y_train[v_index] # val_data=(x_val, y_val) val_data=([x_val, y_val], [y_val, x_val]) lam_recon = 0.392 # lam_recon = 0.0001 # print('YYYYY', Y_train.shape) model = self.architecture.define_architecture(in_shape, hp_params) # model = Model([in_layer], [out_layer]) losses = [margin_loss, 'binary_crossentropy'] model.compile(optimizer=optimizers.Adam(lr=0.001), loss=losses, loss_weights=[1., lam_recon]) # if cnt1 == 1: # model.summary() # Update partition couter print('Partition', cnt1) # Get Updated Callbacks calls = get_calls(cnt1) _X = [X_train, Y_train] _Y = [Y_train, X_train] # Train model model.fit(_X, _Y, epochs=200, batch_size=128, verbose=1, callbacks=calls, validation_data=val_data) _X = [X_test, Y_test] _Y = [Y_test, X_test] model.load_weights('best-weights.h5') stats = self.eval(model, (_X, _Y)) score = -stats.Mcc scores.append(score) print('score', score) K.clear_session() del model # for t_index, v_index in folds.split(X, Y): # X_train, X_val = X[t_index], X[v_index] # Y_train, Y_val = Y[t_index], Y[v_index] # val_data=(X_val, Y_val) # # calls = get_calls() # # in_layer, out_layer = self.architecture.define_architecture(in_shape, hp_params) # model = Model([in_layer], [out_layer]) # model.compile(optimizer='adam', loss='binary_crossentropy') # print(model.summary) # # model.fit(X_train, Y_train, epochs=5, batch_size=32, verbose=1,callbacks=calls, validation_data=val_data) # # stats = self.eval(model, (X_train,Y_train)) # score = stats.Mcc # scores.append(score) # # K.clear_session() # del model final_score = np.mean(scores) print('scores', scores) print('final_score', final_score) print("") return {'loss':final_score, 'status':STATUS_OK} except Exception as e: print(e) return {'loss':None, 'status':STATUS_FAIL} trials = Trials() best = fmin(f, space, algo=tpe.suggest, trials=trials, max_evals=200) best_params = space_eval(space, best) return best, best_params
def model(self): #cname = sys._getframe().f_code.co_name cname = 'p' train, y, test = self.train_, self.y_, self.test_ train.drop('id', axis=1, inplace=True) test.drop('id', axis=1, inplace=True) from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval def step_et(params): clf = ensemble.ExtraTreesRegressor(**params) cv = model_selection.cross_val_score(clf, train, y, scoring=metrics.make_scorer(metrics.log_loss), cv = 5, n_jobs = -2) score = np.mean(cv) print(cname, score, params) return dict(loss=score, status=STATUS_OK) space_et = dict( n_estimators = hp.choice('n_estimators', range(50, 1500)), min_samples_split = hp.choice('min_samples_split', range(2, 10)), min_samples_leaf = hp.choice('min_samples_leaf', range(1, 10)), max_features = hp.choice('max_features', range(4, 16)), random_state = 1 ) trs = self.load('et_trials') if trs == None or self.debug_: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_et, tr.argmin)) best = tr.argmin while len(tr.trials) < 30: best = fmin(step_et, space_et, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) self.save('et_trials', (tr, space_et)) et_params = space_eval(space_et, best) print(et_params) N_splits = 9 N_seeds = 3 v, z = self.v_, self.z_ skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True) cv = [] for s in range(N_seeds): scores = [] cname2 = cname + str(s) v[cname2], z[cname2] = 0, 0 et_params['random_state'] = s + 4242 for n, (itrain, ival) in enumerate(skf.split(train, y)): clf = ensemble.ExtraTreesRegressor(**et_params) clf.fit(train.ix[itrain], y[itrain]) p = clf.predict(train.ix[ival]) v.loc[ival, cname2] += p score = metrics.log_loss(y[ival], p) z[cname2] += clf.predict(test) print(cname, 'seed %d step %d of %d: '%(et_params['random_state'], n+1, skf.n_splits), score, self.now()) scores.append(score) z[cname2] /= N_splits cv.append(np.mean(scores)) print('seed %d loss: '%(et_params['random_state']), scores, np.mean(scores), np.std(scores)) z['y'] = z[cname2] print('cv:', cv, np.mean(cv), np.std(cv)) return cv, None
def f(hp_params): try: nsplits = 2 cnt1 = 0 scores, score = [], None seed = 123 if len(trials.trials)>1: for x in trials.trials[:-1]: space_point_index = dict([(key,value[0]) for key,value in x['misc']['vals'].items() if len(value)>0]) peval = space_eval(space,space_point_index) if hp_params == peval: # print('>>> Repeated Evaluation') loss = x['result']['loss'] return {'loss':loss, 'status':STATUS_FAIL} self.evaluations += 1 print('Testing parameters (eval {})'.format(self.evaluations)) print(hp_params) # Folds for train - test (Evaluate Model) folds1 = StratifiedShuffleSplit(n_splits=nsplits, random_state=seed) for train_index, test_index in folds1.split(X, Y): X_train, X_test = X[train_index], X[test_index] Y_train, Y_test = Y[train_index], Y[test_index] cnt1 += 1 # Folds for train - validation (Guide training phase) folds2 = StratifiedShuffleSplit(n_splits=1, random_state=17, test_size=0.05) for t_index, v_index in folds2.split(X_train, Y_train): x_train, x_val = X_train[t_index], X_train[v_index] y_train, y_val = Y_train[t_index], Y_train[v_index] # val_data=(x_val, y_val) val_data=([x_val, y_val], [y_val, x_val]) lam_recon = 0.392 # lam_recon = 0.0001 # print('YYYYY', Y_train.shape) model = self.architecture.define_architecture(in_shape, hp_params) # model = Model([in_layer], [out_layer]) losses = [margin_loss, 'binary_crossentropy'] model.compile(optimizer=optimizers.Adam(lr=0.001), loss=losses, loss_weights=[1., lam_recon]) # if cnt1 == 1: # model.summary() # Update partition couter print('Partition', cnt1) # Get Updated Callbacks calls = get_calls(cnt1) _X = [X_train, Y_train] _Y = [Y_train, X_train] # Train model model.fit(_X, _Y, epochs=200, batch_size=128, verbose=1, callbacks=calls, validation_data=val_data) _X = [X_test, Y_test] _Y = [Y_test, X_test] model.load_weights('best-weights.h5') stats = self.eval(model, (_X, _Y)) score = -stats.Mcc scores.append(score) print('score', score) K.clear_session() del model # for t_index, v_index in folds.split(X, Y): # X_train, X_val = X[t_index], X[v_index] # Y_train, Y_val = Y[t_index], Y[v_index] # val_data=(X_val, Y_val) # # calls = get_calls() # # in_layer, out_layer = self.architecture.define_architecture(in_shape, hp_params) # model = Model([in_layer], [out_layer]) # model.compile(optimizer='adam', loss='binary_crossentropy') # print(model.summary) # # model.fit(X_train, Y_train, epochs=5, batch_size=32, verbose=1,callbacks=calls, validation_data=val_data) # # stats = self.eval(model, (X_train,Y_train)) # score = stats.Mcc # scores.append(score) # # K.clear_session() # del model final_score = np.mean(scores) print('scores', scores) print('final_score', final_score) print("") return {'loss':final_score, 'status':STATUS_OK} except Exception as e: print(e) return {'loss':None, 'status':STATUS_FAIL}
max_jobs = args.evals #= inf poll_interval = 300 #= 5 min reserve_timeout = 3600 #= 1 hour workdir = None sys.argv[0] = args.worker_helper sys.exit(main_worker_helper(Options(), None)) elif args.action == 'optimizer': # run distributed optimizer trials = MongoTrials(args.mongo, exp_key=args.exp_key) best = fmin(optimize_exec.objective, space, trials=trials, algo=tpe.suggest, max_evals=args.evals) # summary print print "evals: {}".format(args.evals) print "best: {}".format(best) print "space: {}".format(space_eval(space, best)) elif args.action == 'list_best': # list distributed evaluation results best = list_best(args.mongo, exp_key=args.exp_key, space=space) # summary print print "evals: {}".format(args.evals) print "best: {}".format(best) print "space: {}".format(space_eval(space, best)) else: raise Exception("Invalid action '{}'".format(args.action))
TP = CM[1][1] FP = CM[0][1] print("TP = {}".format(TP)) print("FP = {}".format(FP)) print("FN = {}".format(FN)) f1 = 2. * TP / (2. * TP + FP + FN) print("F1 : ", f1) return {'loss': 1 - f1, 'status': STATUS_OK} space = { 'n_estimators': hp.choice('n_estimators', np.arange(300, 1600, 100, dtype=int)), 'max_depth': hp.choice('max_depth', np.arange(10, 30, dtype=int)), 'max_features': hp.choice('max_features', np.arange(10, 50, dtype=int)), 'mss': hp.choice('mss', np.arange(2, 100, 2, dtype=int)), 'cw': hp.uniform('cw', 1, 30), 'msl': hp.choice('msl', np.arange(1, 30, dtype=int)) } trials = Trials() best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=100, trials=trials) pprint(hyperopt.space_eval(space, best)) best_pars = hyperopt.space_eval(space, best)
def optimize(self): assert self.opt.evaluation_on_learning, \ 'evaluation must be set to be true to do hyperparameter optimization.' if self.opt.optimize.loss.startswith("val"): assert self.opt.validation, \ 'validation option must be set to be true to do hyperparameter optimization with validation results.' opt = self.opt.optimize iters, max_trials = 0, opt.get('max_trials', -1) space = self._get_space(opt.space) with log.ProgressBar(log.INFO, desc='optimizing... ', total=None if max_trials == -1 else max_trials, mininterval=30) as pbar: tb_opt = None tb_opt, self.opt.tensorboard = self.opt.tensorboard, tb_opt # trick if opt.start_with_default_parameters: with log.supress_log_level(log.WARN): loss = self._optimize({}) self.logger.info(f'Starting with default parameter result: {loss}') self._optimization_info['best'] = loss if opt.deployment: self.logger.info('Saving model... to {}'.format(self.opt.model_path)) self.save(self.opt.model_path) # NOTE: need better way tb_opt, self.opt.tensorboard = self.opt.tensorboard, tb_opt # trick self.initialize_tensorboard(1000000 if max_trials == -1 else max_trials, name_postfix='.optimize') tb_opt, self.opt.tensorboard = self.opt.tensorboard, tb_opt # trick while(max_trials): with log.supress_log_level(log.WARN): raw_best_parameters = fmin(fn=self._optimize, space=space, algo=tpe.suggest, max_evals=len(self._optimization_info['trials'].trials) + 1, trials=self._optimization_info['trials'], show_progressbar=False) tb_opt, self.opt.tensorboard = self.opt.tensorboard, tb_opt # trick self.update_tensorboard_data(self._optimize_loss) tb_opt, self.opt.tensorboard = self.opt.tensorboard, tb_opt # trick iters += 1 max_trials -= 1 if self._optimization_info.get('best', {}).get('loss', 987654321) > self._optimize_loss['loss']: is_first_time = self._optimization_info['best'] == {} best_parameters = space_eval(space, raw_best_parameters) best_loss = self._optimize_loss # we cannot use return value of hyperopt due to randint behavior patch self.logger.info(f'Found new best parameters: {best_parameters} @ iter {iters}') self._optimization_info['best'] = best_loss self._optimization_info['best_parameters'] = best_parameters if opt.deployment and (is_first_time or not opt.min_trials or opt.min_trials >= iters): if not self.opt.model_path: raise RuntimeError('Failed to dump model: model path is not defined') self.logger.info('Saving model... to {}'.format(self.opt.model_path)) self.save(self.opt.model_path) if self.optimize_after_callback_fn: self.optimize_after_callback_fn(self) pbar.update(1) self.logger.debug('Params({}) Losses({})'.format(self._optimize_params, self._optimize_loss)) tb_opt, self.opt.tensorboard = self.opt.tensorboard, tb_opt # trick self.finalize_tensorboard() return self.get_optimization_data()
def best(self): trials = Trials() best = fmin(self.f, self.space, algo=tpe.suggest, max_evals = self.max_evals, rstate= np.random.RandomState(self.seed), trials=trials) self.clf.set_params(**best) return self.clf, self.name, self.name_short, space_eval(self.space, best), self.best_acc
def run(self): Atrails = Trials() trails = Trials() if self.adaptation == 'TCA': adptdefault_value = { 'kernel_type': 'linear', 'dim': 5, 'lamb': 1, 'gamma': 1 } adptparamSpace = { 'kernel_type': hp.choice('kernel_type', ['primal', 'linear', 'rbf', 'sam']), 'dim': hp.choice('dim', range(5, max(self.sx.shape[1], self.tx.shape[1]))), 'lamb': hp.uniform('lamb', 1e-6, 1e2), 'gamma': hp.uniform('gamma', 1e-5, 1e2) } if self.adaptation == 'DBSCANfilter': adptdefault_value = {'eps': 1, 'min_samples': 10} adptparamSpace = { 'eps': hp.uniform('eps', 0.1, 1e2), 'min_samples': hp.choice('min_samples', range(1, 100)) } if self.adaptation == 'Bruakfilter': adptdefault_value = {'n_neighbors': 10} adptparamSpace = { 'n_neighbors': hp.choice('n_neighbors', range(1, 100)) } if self.adaptation == 'Peterfilter': adptdefault_value = {'eachCluster': 10} adptparamSpace = { 'eachCluster': hp.choice('eachCluster', range(1, 100)) } if self.adaptation == 'Universal': adptdefault_value = {'pvalue': 0.05, 'QuantifyType': 'cliff'} adptparamSpace = { 'pvalue': hp.uniform('pvalue', 0.01, 0.1), 'QuantifyType': hp.choice('QuantifyType', ['cliff', 'cohen']) } if self.adaptation == 'DTB': adptdefault_value = {'DTBneighbors': 10, 'DTBT': 20} adptparamSpace = { 'DTBneighbors': hp.choice('DTBneighbors', range(1, 50)), 'DTBT': hp.choice('DTBT', range(5, 30)) } if self.adaptation == 'DS': adptdefault_value = {'DStopn': 5, 'DSfss': 0.2} adptparamSpace = { 'DStopn': hp.choice('DStopn', range(1, 15)), 'DSfss': hp.uniform('DSfss', 0.1, 0.5) } if self.adaptation == 'DSBF': adptdefault_value = {'DSBFtopk': 1, 'DSBFneighbors': 10} adptparamSpace = { 'DSBFtopk': hp.choice('DSBFtopk', range(1, 10)), 'DSBFneighbors': hp.choice('DSBFneighbors', range(1, 100)) } if self.clf == 'Boost': clfdefault_value = {'Boostnestimator': 50, 'BoostLearnrate': 1} clfparamSpace = { 'Boostnestimator': hp.choice('Boostnestimator', range(10, 1000)), 'BoostLearnrate': hp.uniform('BoostLearnrate', 0.01, 10) } if self.clf == 'RF': clfdefault_value = { 'n_estimators': 10, 'criterion': 'gini', 'max_features': 'auto', 'RFmin_samples_split': 2 } clfparamSpace = { 'n_estimators': hp.choice('n_estimators', range(10, 100)), 'criterion': hp.choice('criterion', ['gini', 'entropy']), 'max_features': hp.choice('max_features', ['auto', 'sqrt', 'log2']), 'RFmin_samples_split': hp.choice('RFmin_samples_split', range(2, int(len(self.sy) / 10))) } if self.clf == 'SVM': clfdefault_value = { 'SVCkernel': { 'kernel': 'poly', 'degree': 3, 'polycoef0': 0.0, 'polygamma': 1 }, 'C': 1.0 } clfparamSpace = { 'SVCkernel': hp.choice( 'SVCkernel', [{ 'kernel': 'linear' }, { 'kernel': 'poly', 'degree': hp.choice('degree', range(1, 5)), 'polycoef0': hp.uniform('polycoef0', 0, 10), 'polygamma': hp.choice('polygamma', ["auto", "scale"]) }, { 'kernel': 'sigmoid', 'sigcoef0': hp.uniform('sigcoef0', 0, 10), 'siggamma': hp.choice('siggamma', ["auto", "scale"]) }, { 'kernel': 'rbf', 'rbfgamma': hp.choice('rbfgamma', ["auto", "scale"]) }]), 'C': hp.uniform('C', 0.001, 1000), } if self.clf == 'NN': clfdefault_value = { 'NNactive': 'relu', 'NNalpha': 0.0001, 'NNmaxiter': 200 } clfparamSpace = { 'NNactive': hp.choice('NNactive', ['identity', 'logistic', 'tanh', 'relu']), 'NNalpha': hp.uniform('NNalpha', 1e-6, 1), 'NNmaxiter': hp.choice('NNmaxiter', range(100, 1000)) } if self.clf == 'KNN': clfdefault_value = {'KNNneighbors': 1} clfparamSpace = { 'KNNneighbors': hp.choice('KNNneighbors', range(1, 50)) } if self.clf == 'NB': clfdefault_value = {'NBType': 'gaussian'} clfparamSpace = { 'NBType': hp.choice('NBType', ['gaussian', 'multinomial', 'bernoulli']) } if self.clf == 'Ridge': clfdefault_value = {'Ridgealpha': 1, 'Ridgenormalize': False} clfparamSpace = { 'Ridgealpha': hp.uniform('Ridgealpha', 0.001, 1000), 'Ridgenormalize': hp.choice('Ridgenormlize', [True, False]) } if self.clf == 'CART': clfdefault_value = { 'criterion': 'gini', 'max_features': 'auto', 'CARTsplitter': 'best', 'RFmin_samples_split': 2 } clfparamSpace = { 'criterion': hp.choice('criterion', ['gini', 'entropy']), 'max_features': hp.choice('max_features', ['auto', 'sqrt', 'log2']), 'CARTsplitter': hp.choice('CARTsplitter', ['best', 'random']), 'RFmin_samples_split': hp.choice('RFmin_samples_split', range(2, int(len(self.sy) / 10))) } default_value = dict(adptdefault_value, **clfdefault_value) self.def_value = self.objFunc(default_value)['result'] self.Adptbest = fmin(self.objFunc, space=adptparamSpace, algo=tpe.suggest, max_evals=int(self.fe * 0.5), trials=self.Atrails) self.Adptbest = space_eval(adptparamSpace, self.Adptbest) his = dict() try: his['name'] = list( self.Atrails.trials[0]['misc']['vals'].keys()) + list( clfdefault_value.keys()) except: his['name'] = [None] i = 0 for item in self.Atrails.trials: if item['state'] == 2: results = list(deepflatten( item['misc']['vals'].values())) + list( clfdefault_value.values()) results.append(item['result']['result']) his[i] = results i += 1 self.SEQ = 1 Clfbest = fmin(self.objFunc, space=clfparamSpace, algo=tpe.suggest, max_evals=int(self.fe * 0.5), trials=self.trails) try: his['name1'] = list(self.Adptbest.keys()) + list( self.trails.trials[0]['misc']['vals'].keys()) except: his['name1'] = [None] for item in self.trails.trials: if item['state'] == 2: results = list(self.Adptbest.values()) + list( deepflatten(item['misc']['vals'].values())) results.append(item['result']['result']) his[i] = results i += 1 inc_value = self.trails.best_trial['result']['result'] return np.asarray([self.def_value, inc_value]), his, Clfbest
def param_optimize(self, X_train, y_train, X_test, y_test, embedding_dim=(3, 6), rnn_cells=[64, 128, 256, 512], hidden_units_layers=(2, 4), hidden_units_cells1=[64, 128, 256, 512], hidden_units_cells2=[16, 32, 64, 128], hidden_units_cells3=[4, 8, 16, 32], hidden_units_cells4=[2, 4, 8], activation=['swish', 'tanh', 'sigmoid', 'relu'], dropout=(0.1, 0.8), epochs=10, batch_size=256, max_evals=100): """ 通过贝叶斯调参寻找最优参数 Args: X: 训练集X数据集 y: 训练集y数据集 X_test: 测试集X数据集 y_test: 测试集y数据集 embedding_dim: 离散数据embedding层输出维度,tuple类型,表示最小最大整数范围 rnn_cells: 序列数据最终输出层神经元个数,list类型,指定枚举值 hidden_units_layers: MLP层的层数,最多支持4层 hidden_units_cells[i]: MLP每层神经元数 activation: 激活函数枚举值 dropout: dropout的调参范围,必须是0-1之间的实数 #######以下非调优参数####### epochs: 指定epochs,不对该参数进行调优 batch_size: 指定batch_size,不对该参数进行调优 max_evals: 优化器最大迭代次数 """ # space = { # 'x': hp.uniform('x', 0, 1), # 0-1的均匀分布 # 'y': hp.normal('y', 0, 1), # 0-1正态分布 # 'name': hp.choice('name', ['alice', 'bob']), } # 枚举值 # 根据入参构造待调优参数列表 space = { 'embedding_dim': hp.choice('embedding_dim', list(range(embedding_dim[0], embedding_dim[1] + 1))), 'rnn_cells': hp.choice('rnn_cells', rnn_cells), 'activation': hp.choice('activation', activation), 'dropout': hp.uniform('dropout', dropout[0], dropout[1]), # 0-1的均匀分布 'hidden_units_cells1': hp.choice('hidden_units_cells1', hidden_units_cells1), 'hidden_units_cells2': hp.choice('hidden_units_cells2', hidden_units_cells2), 'hidden_units_cells3': hp.choice('hidden_units_cells3', hidden_units_cells3), 'hidden_units_cells4': hp.choice('hidden_units_cells4', hidden_units_cells4), 'hidden_units_layers': hp.choice( 'hidden_units_layers', list(range(hidden_units_layers[0], hidden_units_layers[1] + 1))), } # define an objective function def objective(args): print("开始优化迭代,本次参数为:") print(args) hidden_units_layers = [] for i in range(1, args['hidden_units_layers'] + 1): hidden_units_layers.append(args['hidden_units_cells' + str(i)]) self.compile(embedding_dim=args['embedding_dim'], rnn_cells=args['rnn_cells'], activation=args['activation'], dropout=args['dropout'], hidden_units=hidden_units_layers, summary=False) self.fit(X_train, y_train, epochs=epochs, batch_size=batch_size, verbose=0, callback=False) result = self.predict(X_test) auc = lapras.AUC(result.reshape(-1, ), y_test.reshape(-1, )) print("本次验证集AUC值为:" + str(auc)) print( "===================================================================" ) return -auc # 优化目标AUC最大,也就是-AUC最小 # minimize the objective over the space best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=max_evals) best_params = hyperopt.space_eval(space, best) print("优化完成,最优参数为:" + str(best_params)) print("\n") print("开始按照最优参数重新训练模型……") hidden_units_layers = [] for i in range(1, best_params['hidden_units_layers'] + 1): hidden_units_layers.append(best_params['hidden_units_cells' + str(i)]) self.compile(embedding_dim=best_params['embedding_dim'], rnn_cells=best_params['rnn_cells'], activation=best_params['activation'], dropout=best_params['dropout'], hidden_units=hidden_units_layers, summary=False) self.fit(X_train, y_train, X_test, y_test, epochs=epochs, batch_size=batch_size, verbose=0) print("模型已按照最优参数重新训练,可直接使用") return best_params
def opt_params(X, y, session, list_session, classifier_name, ind_params, search_method, type_of_data, feature_select): # compute score for each param set def hyperopt_train_test(params): logo = LeaveOneGroupOut() # create fold indices logo_generator = logo.split(X, y, session) #lolo_generator = logo.split(X,y, list_session) skf = StratifiedKFold(n_splits=5) cv_generator = skf.get_n_splits(X, y) params_joint = ind_params.copy() params_joint.update(params) if classifier_name == 'RF': clf = RF(**params_joint) if classifier_name == 'XGB': clf = xgb.XGBClassifier(**params_joint) if classifier_name == 'SVM': clf = svm.SVC(**params_joint) if classifier_name == 'L2': clf = LogisticRegression(**params_joint) if classifier_name == 'noisy_L2': clf = noisy_LogisticRegression(**params_joint) if classifier_name == 'L1': clf = LogisticRegression(**params_joint) n_sessions = len(np.unique(session)) if n_sessions >= 2: cross_scores = cross_val_score(clf, X, y, cv=logo_generator, scoring='roc_auc', n_jobs=10) else: #cross_scores = cross_val_score(clf, X,y, cv = lolo_generator, n_jobs = 10, scoring = 'roc_auc') cross_scores = cross_val_score(clf, X, y, cv=cv_generator, n_jobs=10, scoring='roc_auc') return cross_scores.mean() # define search parameter space if classifier_name == 'RF': space4classifier = { 'n_estimators': hp.choice('n_estimators', [500]), #'n_estimators': hp.qloguniform('n_estimators', 100,1000,1), 'max_features': hp.choice('max_features', ['sqrt', 'log2', 0.2]), #'max_features': hp.choice('max_features', np.arange(100, X.shape[1],200)), 'max_depth': hp.choice('max_depth', np.arange(4, 10, step=2)), 'min_samples_leaf': hp.choice('min_samples_leaf', np.arange(5, 20, step=5)) } if classifier_name == 'XGB': space4classifier = { 'n_estimators': hp.choice('n_estimators', np.arange(50, 500, step=50)), #'n_estimators': hp.qloguniform('n_estimators', 5,10,1.0), 'max_depth': hp.choice('max_depth', np.arange(4, 10, 1)), 'learning_rate': hp.choice('learning_rate', [0.01, 0.1]), #'reg_lambda': hp.choice('reg_lambda', [0,0.01,0.05,0.1,1.0,10]), #'min_child_weight':hp.choice('min_child_weight',np.arange(3.0,7.0, step =2.0)), #'scale_pos_weight':hp.uniform('scale_pos_weight',1,10), 'colsample_bytree': hp.uniform('colsample_bytree', 0.5, 1.0), 'subsample': hp.uniform('subsample', 0.7, 1.0) } if classifier_name == 'SVM': space4classifier = { #'C': hp.choice('C', 10.**np.arange(-2,10, step = 1.0)), 'C': hp.loguniform('C', 0, 10), 'gamma': hp.loguniform('gamma', -5, 5) #'gamma': hp.choice('gamma', ['auto']), #'kernel':hp.choice('kernel', ['rbf', 'poly', 'sigmoid']), #'degree':hp.choice('degree', np.arange(3,6)) } if classifier_name == 'noisy_L2': space4classifier = { #'C': hp.choice('C', 10.**np.arange(-2,10, step = 1.0)), 'C': hp.loguniform('C', -10, -4), 'sigma_noise': hp.choice('sigma_noise', [0.01, 0.05, 0.1]), 'noise_penalty': hp.loguniform('noise_penalty', -10, -4) #'gamma': hp.choice('gamma', ['auto']), #'kernel':hp.choice('kernel', ['rbf', 'poly', 'sigmoid']), #'degree':hp.choice('degree', np.arange(3,6)) } if classifier_name == 'L2': #space4classifier ={'C': hp.choice('C',np.append(10.**np.arange(-5,-1,step =0.25), np.array(7.2e-4))) if type_of_data == 'long': space4classifier = {'C': hp.loguniform('C', -20, -10)} elif feature_select == 1: space4classifier = {'C': hp.loguniform('C', -20, -10)} else: space4classifier = {'C': hp.loguniform('C', -10, -5)} #'penalty': hp.choice('penalty',['l1','l2']) if classifier_name == 'L1': #space4classifier ={'C': hp.choice('C',np.append(10.**np.arange(-5,-1,step =0.25), np.array(7.2e-4))) if type_of_data == 'long': space4classifier = {'C': hp.loguniform('C', -15, -7)} elif feature_select == 1: space4classifier = {'C': hp.loguniform('C', -3, -2)} else: space4classifier = {'C': hp.loguniform('C', -3, -2)} global best best = 0.5 def f(params): global best acc = hyperopt_train_test(params) if acc > best: best = acc print('new best:', best, params) return {'loss': -acc, 'status': STATUS_OK} #trials = Trials() # saving trials if search_method == 'rand': best_params = fmin(f, space4classifier, algo=hyperopt.rand.suggest, max_evals=20) if search_method == 'tpe': best_params = fmin(f, space4classifier, algo=hyperopt.tpe.suggest, max_evals=20) return space_eval(space4classifier, best_params)
# randomly bag or subsample training data. 'bagging_fraction': hp.uniform('bagging_fraction', 0.7, 1) # bagging_fraction and bagging_freq: enables bagging (subsampling) # of the training data. Both values need to be set for bagging to be used. # The frequency controls how often (iteration) bagging is used. Smaller # fractions and frequencies reduce overfitting. } # Set algoritm parameters with Timer('XGBoost, Search') as t: best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=20) best_params = space_eval(space, best) # Print best parameters best_params['max_depth'] = int(best_params['max_depth']) print("BEST PARAMS: ", best_params) clf = xgb.XGBClassifier( n_estimators=300, **best_params, tree_method='hist', eval_metric="auc", n_jobs=-1, scale_pos_weight=136 )
def run(train, y, test, v, z): np.random.seed(1) #cname = sys._getframe().f_code.co_name train = train.values test = test.values from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval space_stack = hp.choice('stacking by', [ dict( type = 'BayesianRidge' ), dict( type = 'Lars' ), dict( type = 'LinearRegression' ), dict( type = 'Ridge' ), dict( type = 'SGDRegressor', random_state = 1 ), dict( type = 'XGBRegressor', max_depth = hp.choice('max_depth', range(2, 8)), subsample = hp.quniform('subsample', 0.6, 1, 0.05), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), learning_rate = hp.quniform('learning_rate', 0.005, 0.03, 0.005), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), gamma = hp.quniform('gamma', 0, 10, 0.05), reg_alpha = hp.quniform('alpha', 0, 1, 0.0001), ), ]) def get_lr(params): t = params['type'] del params['type'] if t == 'BayesianRidge': lr = linear_model.BayesianRidge(**params) elif t == 'Lars': lr = linear_model.Lars(**params) elif t == 'LinearRegression': lr = linear_model.LinearRegression(**params) elif t == 'Ridge': lr = linear_model.Ridge(**params) elif t == 'SGDRegressor': lr = linear_model.SGDRegressor(**params) elif t == 'XGBRegressor': lr = xgb.XGBRegressor(**params) return lr def step(params): print(params, end = ' ') cv = model_selection.cross_val_score(get_lr(params), train, y, cv=10, scoring=metrics.make_scorer(metrics.log_loss)) score = np.mean(cv) print(score) return dict(loss=score, status=STATUS_OK) trs = state.load('trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: best = tr.argmin print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_stack, best)) mt = max(50, len(tr.trials) + 1) while len(tr.trials) < min(50, mt): best = fmin(step, space_stack, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) state.save('trials', (tr, space_stack)) params = space_eval(space_stack, best) print('best params:', params) lr = get_lr(params) cv = model_selection.cross_val_score(lr, train, y, cv=10, scoring=metrics.make_scorer(metrics.log_loss)) lr.fit(train, y) z['p'] = np.clip(lr.predict(test), 1e-5, 1-1e-5) z['y'] = z['p'] v['p'] = model_selection.cross_val_predict(lr, train, y, cv=10) print('cv:', np.mean(cv), np.std(cv)) return cv, None
def bo_tpe_lightgbm(X, y): # 参考 # https://qiita.com/TomokIshii/items/3729c1b9c658cc48b5cb data = X target = y # 2次数据划分,这样可以分成3份数据 test train validation X_intermediate, X_test, y_intermediate, y_test = train_test_split( data, target, shuffle=True, test_size=0.2, random_state=1) # train/validation split (gives us train and validation sets) X_train, X_validation, y_train, y_validation = train_test_split( X_intermediate, y_intermediate, shuffle=False, test_size=0.25, random_state=1) # delete intermediate variables del X_intermediate, y_intermediate # 显示数据集的分配比例 print('train: {}% | validation: {}% | test {}%'.format( round((len(y_train) / len(target)) * 100, 2), round((len(y_validation) / len(target)) * 100, 2), round((len(y_test) / len(target)) * 100, 2))) starttime = datetime.datetime.now() space = { # 'learning_rate': hp.uniform('learning_rate', 0.001, 0.5), # 'minibatch_frac': hp.choice('minibatch_frac', [1.0, 0.5]), # 'Base': hp.choice('Base', [b1, b2, b3]) "lambda_l1": hp.uniform("lambda_l1", 1e-8, 1.0), "lambda_l2": hp.uniform("lambda_l2", 1e-8, 1.0), "min_child_samples": hp.uniformint("min_child_samples", 5, 100), 'learning_rate': hp.uniform("learning_rate", 0.001, 0.5), "n_estimators": hp.uniformint("n_estimators", 10, 100), "num_leaves": hp.uniformint("num_leaves", 5, 35) } # n_estimators表示一套参数下,有多少个评估器,简单说就是迭代多少次 default_params = { # "n_estimators": 80, "random_state": 1, "objective": "regression", "boosting_type": "gbdt", # "num_leaves": 30, # "learning_rate": 0.3, "feature_fraction": 0.9, "bagging_fraction": 0.8, "bagging_freq": 5, "verbose": -1, } def objective(params): # 下面这个是分类classification使用的模型,不能用在regressor # dtrain = lgb.Dataset(X_train, label=y_train) params.update(default_params) clf = lgb.LGBMRegressor(**params) score = -np.mean( cross_val_score(clf, X_train, y_train, cv=3, n_jobs=-1, scoring="neg_mean_squared_error")) return {'loss': score, 'status': STATUS_OK} trials_lgb = Trials() with warnings.catch_warnings(): warnings.simplefilter("ignore") best = fmin( fn=objective, space=space, algo=tpe.suggest, # max_evals是设定多少套参数组合,组合数越大准确度可能更高但是训练的时间越长 max_evals=50, trials=trials_lgb) best_params = space_eval(space, best) lgb_model = lgb.LGBMRegressor(**best_params).fit( X_train, y_train, eval_set=[(X_validation, y_validation)], verbose=-1, # 假定n_estimators迭代器有100个设定了早期停止后也许不到100次迭代就完成了训练停止了 early_stopping_rounds=2) y_pred = lgb_model.predict(X_test) test_MSE_lgb = mean_squared_error(y_pred, y_test) print("LightGBM MSE score:%.4f" % test_MSE_lgb) endtime = datetime.datetime.now() process_time_lgb = endtime - starttime print("程序执行时间(秒):{}".format(process_time_lgb)) print("最佳超参数值集合:", best_params) save_model_object(lgb_model, 'BO-TPE', 'NGBoost', 'NGBoost') return test_MSE_lgb, process_time_lgb, best_params
def main(): usage = "%prog project label_file" parser = OptionParser(usage=usage) parser.add_option('-m', dest='model', default='LR', help='Model: (LR|SVM|MNB|SVMNB); default=%default') parser.add_option('-f', dest='test_fold', default=0, help='Test fold; default=%default') parser.add_option('-o', dest='output_prefix', default='bayes_opt', help='Output prefix') parser.add_option('--reuse', dest='reuse', action="store_true", default=False, help='Use reusable holdout; default=%default') parser.add_option('--alpha', dest='alpha', action="store_true", default=False, help='Include alpha in search space (instead of grid search); default=%default') parser.add_option('--n_dev_folds', dest='n_dev_folds', default=None, help='Number of dev folds to use when tuning/evaluating; default=%default') parser.add_option('-t', dest='target_col', default=2, help='Index of column containing labels; default=%default') parser.add_option('-w', dest='weight_col', default=-1, help='Index of column containing weights (-1 for None); default=%default') parser.add_option('-v', dest='verbose', default=1, help='Level of verbosity; default=%default') parser.add_option('-n', dest='n_iter', default=60, help='Number of iterations; default=%default') parser.add_option('-a', dest='add_pseudo', action="store_true", default=False, help='Make use of pseudo-documents; default=%default') parser.add_option('--random', dest='random_search', action="store_true", default=False, help='Use random search instead of TPE; default=%default') parser.add_option('--only_unanimous', dest='only_unanimous', action="store_true", default=False, help='Use only the articles with unanimous agreement for evaluation; default=%default') #parser.add_option('--codes', dest='n_codes', default=33, # help='Number of codes (only matters with --alpha); default=%default') (options, args) = parser.parse_args() global output_dirname, output_filename, reuse, search_alpha, space, label_file, group, test_fold, n_dev_folds global weight_col, verbose, target, add_pseudo, only_unanimous project = args[0] dirs.make_base_dir(project) label_file = args[1] reuse = options.reuse search_alpha = options.alpha #n_codes = int(options.n_codes) output_prefix = options.output_prefix model = options.model test_fold = int(options.test_fold) if options.n_dev_folds is not None: n_dev_folds = int(options.n_dev_folds) else: n_dev_folds = None weight_col = int(options.weight_col) target = int(options.target_col) verbose = int(options.verbose) n_iter = int(options.n_iter) add_pseudo = options.add_pseudo random_search = options.random_search only_unanimous = options.only_unanimous # allow user to specfiy a particular choice of model if model == 'LR': space['model'] = { 'model': 'LR', 'regularization': hp.choice('regularization', ['l1', 'l2']) #'regularization': 'l1' } elif model == 'SVM': space['model'] = { 'model': 'SVM', 'kernel': hp.choice('ktype', [ {'ktype': 'linear'}, {'ktype': 'poly', 'degree': hp.choice('degree', [2, 3, 4])}, {'ktype': 'rbf'} ] ) } elif model == 'MNB': space['model'] = { 'model': 'MNB' } elif model == 'SVMNB': space['model'] = { 'model': 'SVMNB', 'beta': hp.uniform('beta', 0, 1) } else: sys.exit('Choice of model not supported!') if add_pseudo: add_pseudo_options() output_prefix += '_' + model if search_alpha: space['alpha'] = hp.loguniform('alpha', -3, 10) output_prefix += '_alpha' else: output_prefix += '_noalpha' if reuse: output_prefix += '_reuse' all_items, target_name, labels, weights, _ = lr.get_labels(label_file, target, weight_col=weight_col) output_dirname = experiment.make_exp_dir(test_fold, target_name, output_prefix) basedir = os.path.split(output_dirname)[0] output_filename = fh.make_filename(basedir, output_prefix, 'log') with codecs.open(output_filename, 'w') as output_file: output_file.write(output_dirname + '\n') output_file.write('reuse = ' + str(reuse) + '\n') output_file.write('search alphas = ' + str(search_alpha) + '\n') trials = Trials() if random_search: best = fmin(call_experiment, space=space, algo=rand.suggest, max_evals=n_iter, trials=trials) else: best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=n_iter, trials=trials) print space_eval(space, best) print trials.losses()
def model(self): #cname = sys._getframe().f_code.co_name cname = 'lgb' train, y, test = self.train_, self.y_, self.test_ train.drop('id', axis=1, inplace=True) test.drop('id', axis=1, inplace=True) from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval dtrain = lgb.Dataset(train, label=y) def fix_params(params): for p in ['min_data_in_leaf', 'num_leaves', 'max_bin' ]: params[p] = int(params[p]) params['num_leaves'] = max(params['num_leaves'], 2) def step_xgb(params): fix_params(params) cv = lgb.cv(params, dtrain, num_boost_round=10000, early_stopping_rounds=50, nfold=6, seed=params['seed']) rounds = np.argmin(cv['binary_logloss-mean']) score = np.min(cv['binary_logloss-mean']) print(cname, score, rounds, params, self.now()) return dict(loss=score, status=STATUS_OK) space_lgb = dict( bagging_fraction = hp.quniform('bagging_fraction', 0.5, 1, 0.001), colsample_bytree = hp.quniform('colsample_bytree', 0.6, 1, 0.05), feature_fraction = hp.quniform('feature_fraction', 0.5, 1, 0.001), lambda_l1 = hp.choice('lambda_l1', [0, hp.loguniform('lambda_l1_positive', -16, 2)]), lambda_l2 = hp.choice('lambda_l2', [0, hp.loguniform('lambda_l2_positive', -16, 2)]), learning_rate = hp.loguniform('learning_rate', -7, 0), max_bin = hp.qloguniform('max_bin', 0, 20, 1), max_depth = hp.choice('max_depth', range(2, 9)), min_child_weight = hp.quniform('min_child_weight', 1, 6, 1), min_data_in_leaf = hp.qloguniform('min_data_in_leaf', 0, 6, 1), min_sum_hessian_in_leaf = hp.loguniform('min_sum_hessian_in_leaf', -16, 5), num_leaves = hp.qloguniform('num_leaves', 2, 7, 1), reg_alpha = hp.quniform('reg_alpha', 0, 1, 0.001), subsample = hp.quniform('subsample', 0.6, 1, 0.05), bagging_freq = 1, objective = 'binary', metric = 'binary_logloss', seed = 1, #silent = 1, ) trs = self.load('lightgbm_trials') if trs == None or self.debug_: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_lgb, tr.argmin)) best = tr.argmin while len(tr.trials) < self.max_ho_trials_: print(len(tr.trials), end=' ') #best = fmin(step_xgb, space_lgb, algo=tpe.suggest, max_evals=len(tr.trials) + 1, trials = tr) best = fmin(step_xgb, space_lgb, algo=partial(tpe.suggest, n_startup_jobs=1), max_evals=len(tr.trials) + 1, trials = tr) self.save('lightgbm_trials', (tr, space_lgb)) lgb_params = space_eval(space_lgb, best) fix_params(lgb_params) print(lgb_params) N_splits = self.num_splits_ N_seeds = self.num_seeds_ v, z = self.v_, self.z_ skf = model_selection.StratifiedKFold(n_splits=N_splits, shuffle=True) cv = [] for s in range(N_seeds): scores = [] cname2 = cname + str(s) v[cname2], z[cname2] = 0, 0 lgb_params['seed'] = s + self.base_seed_ for n, (itrain, ival) in enumerate(skf.split(train, y)): dtrain = lgb.Dataset(train.ix[itrain], y[itrain]) dvalid = lgb.Dataset(train.ix[ival], y[ival]) clf = lgb.train(lgb_params, dtrain, num_boost_round=10000, valid_sets=[dtrain, dvalid], valid_names=['train', 'valid'], early_stopping_rounds=100, verbose_eval=False) p = clf.predict(train.ix[ival]) v.loc[ival, cname2] += p score = metrics.log_loss(y[ival], p) z[cname2] += clf.predict(test) print(cname, 'seed %d step %d of %d: '%(lgb_params['seed'], n+1, skf.n_splits), score, self.now()) scores.append(score) z[cname2] /= N_splits cv.append(np.mean(scores)) print('seed %d loss: '%(lgb_params['seed']), scores, np.mean(scores), np.std(scores)) z['y'] = z[cname2] print('cv:', cv, np.mean(cv), np.std(cv)) return cv, None
import hyperopt from hyperopt import hp, fmin, tpe, Trials # define an objective function def objective(args): case, val = args if case == 'case 1': return val else: return val ** 2 if __name__ == '__main__': trials = Trials() # define a search space space = hp.choice('a', [ ('case 1', 1 + hp.lognormal('c1', 0, 1)), ('case 2', hp.uniform('c2', -10, 10)) ]) # minimize the objective over the space best = fmin(objective, space, algo=tpe.suggest, max_evals=250, trials=trials) print best # -> {'a': 1, 'c2': 0.01420615366247227} print hyperopt.space_eval(space, best) # -> ('case 2', 0.01420615366247227}
def optimize(self): logging.warning('Start optimization for:' + self.__class__.__name__) evals = Configuration.HYPEROPT_EVALS_PER_SEARCH result = fmin(fn=self._objective, space=self._hyper_space, algo=tpe.suggest, max_evals=evals) return space_eval(self._hyper_space, result)
from sklearn import datasets from hyperopt import fmin, tpe, hp, Trials import numpy as np import hyperopt import space import clf_objective_function get_space = space.get_space classifier_objective = clf_objective_function.classifier_objective iris = datasets.load_iris() X, y = iris.data, iris.target X, y = X[y != 0, :2], y[y != 0] X_og, y_og = X, y trials = Trials() best = fmin(fn=lambda x: classifier_objective(x, X, y), space=get_space(), algo=tpe.suggest, max_evals=100, trials=trials) print '\n\n' print best config = hyperopt.space_eval(get_space(), best) print classifier_objective(config, X, y) # print trials.results
def bo_tpe_ngb(X, y): # 参考例子 # https://github.com/stanfordmlgroup/ngboost/blob/master/examples/tuning/hyperopt.ipynb data = X target = y # 2次数据划分,这样可以分成3份数据 test train validation X_intermediate, X_test, y_intermediate, y_test = train_test_split( data, target, shuffle=True, test_size=0.2, random_state=1) # train/validation split (gives us train and validation sets) X_train, X_validation, y_train, y_validation = train_test_split( X_intermediate, y_intermediate, shuffle=False, test_size=0.25, random_state=1) # delete intermediate variables del X_intermediate, y_intermediate # 显示数据集的分配比例 print('train: {}% | validation: {}% | test {}%'.format( round((len(y_train) / len(target)) * 100, 2), round((len(y_validation) / len(target)) * 100, 2), round((len(y_test) / len(target)) * 100, 2))) starttime = datetime.datetime.now() # 搜索空间设定 b1 = DecisionTreeRegressor(criterion='friedman_mse', max_depth=2) b2 = DecisionTreeRegressor(criterion='friedman_mse', max_depth=3) b3 = DecisionTreeRegressor(criterion='friedman_mse', max_depth=4) space = { 'learning_rate': hp.uniform('learning_rate', 0.001, 0.5), 'minibatch_frac': hp.choice('minibatch_frac', [1.0, 0.5]), 'Base': hp.choice('Base', [b1, b2, b3]) } # n_estimators表示一套参数下,有多少个评估器,简单说就是迭代多少次 default_params = {"n_estimators": 20, "verbose_eval": 1, "random_state": 1} def objective(params): params.update(default_params) ngb = NGBRegressor(**params, verbose=False).fit( X_train, y_train, X_val=X_validation, Y_val=y_validation, # 假定n_estimators迭代器有100个设定了早期停止后也许不到100次迭代就完成了训练停止了 early_stopping_rounds=2) loss = ngb.evals_result['val']['LOGSCORE'][ngb.best_val_loss_itr] results = {'loss': loss, 'status': STATUS_OK} return results trials_ngb = Trials() with warnings.catch_warnings(): warnings.simplefilter("ignore") best = fmin( fn=objective, space=space, algo=tpe.suggest, # max_evals是设定多少套参数组合,组合数越大准确度可能更高但是训练的时间越长 max_evals=50, trials=trials_ngb) best_params = space_eval(space, best) ngb_new = NGBRegressor(**best_params, verbose=False).fit( X_train, y_train, X_val=X_validation, Y_val=y_validation, # 假定n_estimators迭代器有100个设定了早期停止后也许不到100次迭代就完成了训练停止了 early_stopping_rounds=2) y_pred = ngb_new.predict(X_test) test_MSE_ngb = mean_squared_error(y_pred, y_test) print("NGBoost MSE score:%.4f" % test_MSE_ngb) endtime = datetime.datetime.now() process_time_ngb = endtime - starttime print("程序执行时间(秒):{}".format(process_time_ngb)) print("最佳超参数值集合:", best_params) save_model_object(ngb_new, 'BO-TPE', 'NGBoost', 'NGBoost') return test_MSE_ngb, process_time_ngb, best_params
def main(main_args): usage = "%prog project label_file splits_file" parser = OptionParser(usage=usage) parser.add_option('-m', dest='model', default='LR', help='Model: (LR|SVM|MNB|SVMNB); default=%default') parser.add_option('-o', dest='output_prefix', default='bayes_opt', help='Output prefix') parser.add_option('--alpha', dest='alpha', action="store_true", default=False, help='Include alpha in search space (instead of grid search); default=%default') parser.add_option('--n_dev_folds', dest='n_dev_folds', default=None, help='Number of dev folds to use when tuning/evaluating; default=%default') parser.add_option('-t', dest='target_col', default=2, help='Index of column containing labels; default=%default') parser.add_option('-w', dest='weight_col', default=-1, help='Index of column containing weights (-1 for None); default=%default') parser.add_option('-v', dest='verbose', default=1, help='Level of verbosity; default=%default') parser.add_option('-n', dest='n_iter', default=2, help='Number of iterations; default=%default') parser.add_option('-a', dest='add_pseudo', action="store_true", default=False, help='Make use of pseudo-documents; default=%default') parser.add_option('--random', dest='random_search', action="store_true", default=False, help='Use random search instead of TPE; default=%default') parser.add_option('--personas_old', dest='personas_old', action="store_true", default=False, help='Use personas from the DPM; default=%default') parser.add_option('--personas_new', dest='personas_new', action="store_true", default=False, help='Use personas from my model; default=%default') parser.add_option('--story_types', dest='story_types', action="store_true", default=False, help='Use story types from my model; default=%default') #parser.add_option('--codes', dest='n_codes', default=33, # help='Number of codes (only matters with --alpha); default=%default') (options, args) = parser.parse_args() if len(args) == 0: args = main_args global output_dirname, output_filename, search_alpha, space, label_file, group, n_dev_folds global weight_col, verbose, target, add_pseudo, output_prefix global personas_old, personas_new, story_types project = args[0] label_file = args[1] splits_file = args[2] dirs.set_project(project, splits_file) search_alpha = options.alpha output_prefix = options.output_prefix model = options.model if options.n_dev_folds is not None: n_dev_folds = int(options.n_dev_folds) else: n_dev_folds = None weight_col = int(options.weight_col) target = int(options.target_col) verbose = int(options.verbose) n_iter = int(options.n_iter) add_pseudo = options.add_pseudo random_search = options.random_search personas_old = options.personas_old personas_new = options.personas_new story_types = options.story_types # allow user to specfiy a particular choice of model if model == 'LR': space['model'] = { 'model': 'LR', #'regularization': hp.choice('regularization', ['l1', 'l2']) 'regularization': 'l1' } elif model == 'SVM': space['model'] = { 'model': 'SVM', 'kernel': hp.choice('ktype', [ {'ktype': 'linear'}, {'ktype': 'poly', 'degree': hp.choice('degree', [2, 3, 4])}, {'ktype': 'rbf'} ] ) } elif model == 'MNB': space['model'] = { 'model': 'MNB' } elif model == 'SVMNB': space['model'] = { 'model': 'SVMNB', 'beta': hp.uniform('beta', 0, 1) } else: sys.exit('Choice of model not supported!') if personas_old: space['features']['personas'] = hp.choice('personas', [ { 'use': False }, { 'use': True, 'transform': hp.choice('personas_transform', ['binarize', 'normalizel1']), 'subdir': 'personas', 'source': 'personasdpm', } ]) if personas_new: space['features']['personas'] = hp.choice('personas', [ { 'use': False }, { 'use': True, 'transform': hp.choice('personas_transform', ['binarize', 'normalizel1']), 'subdir': 'personas', 'source': 'personas', } ]) if story_types: space['features']['storytypes'] = hp.choice('storytypes', [ { 'use': False }, { 'use': True, 'transform': 'normalizel1', 'subdir': 'personas', 'source': 'storytypesold', } ]) if add_pseudo: add_pseudo_options() output_prefix += '_' + model if search_alpha: space['alpha'] = hp.loguniform('alpha', -3, 10) output_prefix += '_alpha' else: output_prefix += '_noalpha' all_items, target_name, labels, weights, _ = lr.get_labels(label_file, target, weight_col=weight_col) #for t in range(10): # output_dirname = experiment2.make_exp_dir(t, target_name, output_prefix) output_dirname = fh.makedirs(dirs.exp_dir, target_name) output_filename = os.path.join(output_dirname, output_prefix + '.log') with codecs.open(output_filename, 'w') as output_file: output_file.write(output_dirname + '\n') output_file.write('search alphas = ' + str(search_alpha) + '\n') trials = Trials() if random_search: best = fmin(call_experiment, space=space, algo=rand.suggest, max_evals=n_iter, trials=trials) else: best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=n_iter, trials=trials) print space_eval(space, best) print trials.losses()
} df = data() df = df[df["ad"] == outlier] agg = df.groupby("ad").sum().iloc[outlier]["av"] # define a search space a1_min = min(list(df["a1"].unique())) a1_max = max(list(df["a1"].unique())) a2_min = min(list(df["a2"].unique())) a2_max = max(list(df["a2"].unique())) q = 1 space = hp.choice( "parameters", [ { "a1_lower": hp.quniform("a1_lower", a1_min, a1_max, q), "a1_upper": hp.quniform("a1_upper", a1_min, a1_max, q), "a2_lower": hp.quniform("a2_lower", a2_min, a2_max, q), "a2_upper": hp.quniform("a2_upper", a2_min, a2_max, q), }, ], ) trials = Trials() # minimize the objective over the space best = fmin(objective, space, algo=tpe.suggest, max_evals=1000, trials=trials) print(space_eval(space, best))
x = list( x ) x.insert( 0, error ) writer.writerow( x ) return error ### space = ( hp.qloguniform( 'l1_dim', log( 10 ), log( 1000 ), 1 ), hp.qloguniform( 'l2_dim', log( 10 ), log( 1000 ), 1 ), hp.loguniform( 'learning_rate', log( 1e-5 ), log( 1e-2 )), hp.uniform( 'momentum', 0.5, 0.99 ), hp.uniform( 'l1_dropout', 0.1, 0.9 ), hp.uniform( 'decay_factor', 1 + 1e-3, 1 + 1e-1 ) ) run_counter = 0 start_clock = clock() output_file = 'results.csv' writer = csv.writer( open( output_file, 'wb' )) best = fmin( run_test, space, algo = tpe.suggest, max_evals = 50 ) print best print run_test( hyperopt.space_eval( space, best )) print "Seconds", clock() - start_clock
'C': hp.uniform('C', -15, 15), 'gamma': hp.uniform('gamma', -15, 15) } # optimize parameters trial_nr = 0 trials = Trials() all_results = [] best = fmin(create_and_evaluate_model, space, algo=tpe.suggest, max_evals=n_iter, trials=trials) # extract the best parameters best_params = hyperopt.space_eval(space, best) # write to file outfile = os.path.join( params_dir, "optimal_params_%s_%s_%s.pickle" % (cls_method, dataset_name, method_name)) with open(outfile, "wb") as fout: pickle.dump(best_params, fout) dt_results = pd.DataFrame(all_results, columns=[ "iter", "param", "value", "nr_events", "auc", "rmspd", "score" ]) dt_results["dataset"] = dataset_name dt_results["cls"] = cls_method
'gamma':hp.choice('gamma',[0.001, 0.01, 0.1, 1]) } # Trials trials = Trials() # Set optimization algorithm algo = partial(tpe.suggest, n_startup_jobs=10, gamma=0.25) best = fmin(fn=objective_func, space=space, algo=algo, max_evals=100, trials=trials) best_parameters = space_eval(space, best) print("Best parameters:", str(best_parameters)) # Print time tdiff = trials.trials[-1]['book_time'] - trials.trials[0]['book_time'] print("Time:" + str(tdiff.total_seconds() / 60)) # Set params clf = SVC(C=best_parameters['C'], kernel=best_parameters['kernel'], gamma=best_parameters['gamma'], degree=best_parameters['degree']) # Fit clf.fit(X_train, y_train) y_pred = clf.predict(X_test) # Predict score = classification_report(y_pred, y_test)
def hyperopt_lightgbm(X_train: pd.DataFrame, y_train: pd.Series, X_val: pd.DataFrame, y_val: pd.Series, params: Dict, config: Config): train_data = lgb.Dataset(X_train, label=y_train) valid_data = lgb.Dataset(X_val, label=y_val) space = { "learning_rate": hp.loguniform("learning_rate", np.log(0.01), np.log(0.2)), #"max_depth": hp.choice("max_depth", [-1, 2, 3, 4, 5, 6]), "num_leaves": hp.choice("num_leaves", np.linspace(16, 64, 4, dtype=int)), #"feature_fraction": hp.quniform("feature_fraction", 0.5, 1.0, 0.1), #"bagging_fraction": hp.quniform("bagging_fraction", 0.5, 1.0, 0.1), #"bagging_freq": hp.choice("bagging_freq", np.linspace(0, 10, 1, dtype=int)), # "reg_alpha": hp.uniform("reg_alpha", 0, 2), # "reg_lambda": hp.uniform("reg_lambda", 0, 2), # "min_child_weight": hp.uniform('min_child_weight', 0.5, 10), # "scale_pos_weight": hp.uniform('x', 0, 5), } def objective(hyperparams): model = lgb.train({ **params, **hyperparams }, train_data, 500, valid_data, early_stopping_rounds=100, verbose_eval=0) score = model.best_score["valid_0"][params["metric"]] feature_importance_df = pd.DataFrame() feature_importance_df["features"] = X_train.columns feature_importance_df["importance_gain"] = model.feature_importance( importance_type='gain') record_zero_importance = feature_importance_df[ feature_importance_df["importance_gain"] == 0.0] to_drop = list(record_zero_importance['features']) # in classification, less is better return { 'loss': -score, 'status': STATUS_OK, "drop_feature": to_drop, "best_iter": model.best_iteration } trials = Trials() best = hyperopt.fmin(fn=objective, space=space, trials=trials, algo=tpe.suggest, max_evals=10, verbose=1, rstate=np.random.RandomState(1)) hyperparams = space_eval(space, best) log(f"hyperopt auc = {-trials.best_trial['result']['loss']:0.4f} {hyperparams}" ) drop_feature = set(X_train.columns.tolist()) for result in trials.results: drop_feature = drop_feature & set(result['drop_feature']) return hyperparams, drop_feature, trials.best_trial['result']['best_iter']
h_score += tmp.sum()*1.0/(cnt*(cnt-1)) h_score = h_score/numtopics ''' spacetosearch = { 'numsentilabel': hp.choice('numsentilabel', [10, 20, 50]), 'numtopics': hp.choice('numtopics', [10, 20, 50]), } trials = Trials() best = fmin(f1, spacetosearch, algo=tpe.suggest, max_evals=9, trials=trials) best_params1 = space_eval(spacetosearch, best) print('best {} with params {}'.format(best, best_params1)) numsentilabel, numtopics = best_params1['numsentilabel'], best_params1[ 'numtopics'] run_experiment(numsentilabel, numtopics, alpha, beta, gamma, maxiter, numwordspertopic) spacetosearch = { 'numwordspertopic': hp.choice('numwordspertopic', [5, 10, 20, 25, 50]) } trials = Trials() best = fmin(fn=f2, space=spacetosearch, algo=tpe.suggest, trials=trials,
validation_step(post_text_val, truth_class_val, post_text_len_val, truth_mean_val, target_description_val, target_description_len_val, image_feature_val) print("\n") min_mse_val = np.inf for i in range(FLAGS.epochs): batches = get_batch(train_data, FLAGS.batch_size) for batch in batches: post_text_batch, truth_class_batch, post_text_len_batch, truth_mean_batch, target_description_batch, target_description_len_batch, image_feature_batch = zip(*batch) train_step(post_text_batch, truth_class_batch, post_text_len_batch, truth_mean_batch, target_description_batch, target_description_len_batch, image_feature_batch) print("\nValidation: ") mse_val = validation_step(post_text_val, truth_class_val, post_text_len_val, truth_mean_val, target_description_val, target_description_len_val, image_feature_val) print("\n") if mse_val < min_mse_val: min_mse_val = mse_val # saver.save(sess, checkpoint_prefix) round += 1 val_scores.append(min_mse_val) return np.mean(val_scores) if __name__ == "__main__": space = { "batch_size": hyperopt.hp.choice("batch_size", [16, 32, 64, 128]), "dropout_rate_hidden": hyperopt.hp.choice("dropout_rate_hidden", [0.3, 0.5, 0.7]), "learning_rate": hyperopt.hp.choice("learning_rate", [0.001, 0.005, 0.01, 0.05]), "gradient_clipping_value": hyperopt.hp.choice("gradient_clipping_value", [0.5, 1, 2, 5, 10]) } best_model = hyperopt.fmin(main, space, algo=hyperopt.tpe.suggest, max_evals=100) print(best_model) print(hyperopt.space_eval(space, best_model))
def main(): usage = "%prog <DRLD|MIP|MOLD|Primary|General|Terrorist|PK-Brown|PK-Roberts|PK-Pelosi|PK-Cheney>" parser = OptionParser(usage=usage) parser.add_option("-m", dest="model", default="LR", help="Model: (LR|SVM|MNB|SVMNB); default=%default") parser.add_option("-t", dest="test_fold", default=0, help="Test fold; default=%default") parser.add_option("-o", dest="output_dirname", default="bayes_opt", help="Output directory name") parser.add_option( "--reuse", dest="reuse", action="store_true", default=False, help="Use reusable holdout; default=%default" ) parser.add_option( "--alpha", dest="alpha", action="store_true", default=False, help="Include alpha in search space (instead of grid search); default=%default", ) parser.add_option( "--n_dev_folds", dest="n_dev_folds", default=5, help="Number of dev folds to use when tuning/evaluating; default=%default", ) # parser.add_option('--codes', dest='n_codes', default=33, # help='Number of codes (only matters with --alpha); default=%default') (options, args) = parser.parse_args() global output_dirname, output_filename, reuse, search_alpha, space, run, group, test_fold, n_dev_folds run = args[0] reuse = options.reuse search_alpha = options.alpha # n_codes = int(options.n_codes) output_dirname = options.output_dirname model = options.model test_fold = int(options.test_fold) n_dev_folds = int(options.n_dev_folds) # allow user to specfiy a particular choice of model if model == "LR": space["model"] = { "model": "LR", #'regularization': hp.choice('regularization', ['l1', 'l2']) "regularization": "l1", } elif model == "SVM": space["model"] = { "model": "SVM", "kernel": hp.choice( "ktype", [{"ktype": "linear"}, {"ktype": "poly", "degree": hp.choice("degree", [2, 3, 4])}, {"ktype": "rbf"}], ), } elif model == "MNB": space["model"] = {"model": "MNB"} elif model == "SVMNB": space["model"] = {"model": "SVMNB", "beta": hp.uniform("beta", 0, 1)} else: sys.exit("Choice of model not supported!") if run == "DRLD": add_drld() group = ["Democrat-Likes", "Democrat-Dislikes", "Republican-Likes", "Republican-Dislikes"] n_codes = 33 elif run == "MIP": add_MIP() group = ["MIP-Personal-1", "MIP-Personal-2", "MIP-Political-1", "MIP-Political-2"] n_codes = 74 elif run == "MOLD": add_MOLD() group = ["McCain-Likes", "McCain-Dislikes", "Obama-Likes", "Obama-Dislikes"] n_codes = 34 elif run == "Primary": add_obama() add_clinton() group = ["Obama-Primary", "Clinton-Primary"] n_codes = 42 elif run == "General": add_obama() add_mccain() group = ["Obama-General", "McCain-General"] n_codes = 41 elif run == "Terrorists": group = [run] n_codes = 28 elif run == "PK-Brown": group = [run] n_codes = 14 elif run == "PK-Cheney": group = [run] n_codes = 12 elif run == "PK-Pelosi": group = [run] n_codes = 15 elif run == "PK-Roberts": group = [run] n_codes = 14 else: sys.exit("Dataset not recognized") output_dirname += "_" + model if search_alpha: space["alphas"] = [] for i in range(n_codes): space["alphas"].append(hp.loguniform("alpha" + str(i), -1.15, 9.2)) output_dirname += "_alphas" if reuse: output_dirname += "_reuse" else: output_dirname += "_noreuse" output_dirname += "_" + run if n_dev_folds != 5: output_dirname += "_" + str(n_dev_folds) output_filename = fh.make_filename(defines.exp_dir, fh.get_basename(output_dirname), "log") with codecs.open(output_filename, "w") as output_file: output_file.write(output_dirname + "\n") output_file.write("reuse = " + str(reuse) + "\n") output_file.write("search alphas = " + str(search_alpha) + "\n") trials = Trials() best = fmin(call_experiment, space=space, algo=tpe.suggest, max_evals=40, trials=trials) print space_eval(space, best) print trials.losses()
def _train_on_dataset( self, dataset, train_size=0.67, batch_size=64, num_epochs=100, num_cpus=1, num_gpus=0, max_evals=None, progress_cb=None, abnormal=None, ): if max_evals is None: max_evals = self.settings.get('max_evals', 21) # latent_dim*intermediate_dim self.current_eval = 0 self.stat_dataset(dataset) dataset = self.scale_dataset(dataset) def cross_val_model(params): keras_model = None # Destroys the current TF graph and creates a new one. # Useful to avoid clutter from old models / layers. K.clear_session() self._set_xpu_config(num_cpus, num_gpus) self.span = W = params.span (X_miss, X_train), (X_miss_val, X_test) = self.train_test_split( dataset, train_size=train_size, abnormal=abnormal, ) if len(X_train) == 0: raise errors.NoData("insufficient training data") if len(X_test) == 0: raise errors.NoData("insufficient validation data") # expected input data shape: (batch_size, timesteps,) # network parameters input_shape = (W, ) intermediate_dim = params.intermediate_dim latent_dim = params.latent_dim # VAE model = encoder + decoder # build encoder model main_input = Input(shape=input_shape) aux_input = Input( shape=input_shape) # bool vector to flag missing data points aux_output = Lambda(lambda x: x)(aux_input) x = Dense(intermediate_dim, kernel_regularizer=regularizers.l2(0.01), activation='relu')(main_input) z_mean = Dense(latent_dim, name='z_mean')(x) z_log_var = Dense(latent_dim, name='z_log_var')(x) # use reparameterization trick to push the sampling out as input # note that "output_shape" isn't necessary with the TensorFlow backend z = Lambda(sampling, output_shape=(latent_dim, ), name='z')([z_mean, z_log_var]) # build decoder model x = Dense(intermediate_dim, kernel_regularizer=regularizers.l2(0.01), activation='relu', name='dense_1')(z) main_output = Dense(W, activation='linear', name='dense_2')(x) # instantiate Donut model keras_model = _Model([main_input, aux_input], [main_output, aux_output], name='donut') add_loss(keras_model, W) optimizer_cls = None if params.optimizer == 'adam': optimizer_cls = tf.keras.optimizers.Adam() keras_model.compile(optimizer=optimizer_cls, ) _stop = EarlyStopping( monitor='val_loss', patience=5, verbose=_verbose, mode='auto', ) keras_model.fit_generator( generator(X_train, X_miss, batch_size, keras_model), epochs=num_epochs, steps_per_epoch=len(X_train) / batch_size, verbose=_verbose, validation_data=([X_test, X_miss_val], None), callbacks=[_stop], workers=0, # https://github.com/keras-team/keras/issues/5511 ) # How well did it do? score = keras_model.evaluate( [X_test, X_miss_val], batch_size=batch_size, verbose=_verbose, ) self.current_eval += 1 if progress_cb is not None: progress_cb(self.current_eval, max_evals) return score, keras_model hyperparameters = HyperParameters() # Parameter search space def objective(args): hyperparameters.assign(args) try: score, _ = cross_val_model(hyperparameters) return {'loss': score, 'status': STATUS_OK} except Exception as exn: logging.warning("iteration failed: %s", exn) return {'loss': None, 'status': STATUS_FAIL} space = hp.choice('case', [{ 'span': self.get_hp_span('span'), 'latent_dim': hp.choice('latent_dim', [3, 5, 8]), 'intermediate_dim': hp.choice('i1', [21, 34, 55, 89, 144, 233, 377]), 'optimizer': hp.choice('optimizer', ['adam']), }]) # The Trials object will store details of each iteration trials = Trials() # Run the hyperparameter search using the tpe algorithm try: best = fmin( objective, space, algo=tpe.suggest, max_evals=max_evals, trials=trials, ) except ValueError: raise errors.NoData( "training failed, try to increase the time range") # Get the values of the optimal parameters best_params = space_eval(space, best) score, self._keras_model = cross_val_model( HyperParameters(best_params)) self.span = best_params['span'] return (best_params, score)
def model(self): #cname = sys._getframe().f_code.co_name if self.preselect_features_: self.greedy_select_features() train = self.train_.values test = self.test_.values y = self.y_ self.baseline_score_, _ = self.ccv(linear_model.BayesianRidge(), train, y, metrics.make_scorer(metrics.log_loss)) self.baseline_stacker_ = linear_model.BayesianRidge() for fit_intercept in [False, True]: for normalize in [False, True]: lr = linear_model.LinearRegression(fit_intercept=fit_intercept, normalize=normalize) score, _ = self.ccv(lr, train, y, metrics.make_scorer(metrics.log_loss)) if score < self.baseline_score_: self.baseline_score_ = score self.baseline_stacker_ = lr print('baseline:', self.baseline_score_, self.baseline_stacker_) np.random.seed(1) from hyperopt import fmin, tpe, hp, STATUS_OK, Trials, space_eval space_stack = hp.choice('stacking by', [ dict( type = 'Ridge', random_state = 1, #hp.choice('random_state', range(1, 100000)), alpha = hp.loguniform('alpha', -7, 5), fit_intercept = hp.choice('fit_intercept1', [True, False]), normalize = hp.choice('normalize1', [True, False]) ), ]) def get_lr(params): t = params['type'] del params['type'] if t == 'LinearRegression': lr = linear_model.LinearRegression(**params) elif t == 'Ridge': lr = linear_model.Ridge(**params) else: raise Exception() return lr def step(params): print(params, end = ' ') score, _ = self.ccv(get_lr(params), train, y, metrics.make_scorer(metrics.log_loss)) print(score, self.now()) return dict(loss=score, status=STATUS_OK) trs = self.load('trials') if trs == None: tr = Trials() else: tr, _ = trs if len(tr.trials) > 0: print('reusing %d trials, best was:'%(len(tr.trials)), space_eval(space_stack, tr.argmin)) mt = max(self.hyperopt_rounds_, len(tr.trials) + 1) while len(tr.trials) < mt: print(len(tr.trials), end=' ') best = fmin(step, space_stack, algo=partial(tpe.suggest, n_startup_jobs=1), max_evals=len(tr.trials) + 1, trials = tr) self.save('trials', (tr, space_stack)) params = space_eval(space_stack, best) print('best params:', params) lr = get_lr(params) cv = model_selection.cross_val_score(lr, train, y, cv=self.n_fold_, scoring=metrics.make_scorer(metrics.log_loss)) if np.mean(cv) > self.baseline_score_: lr = self.baseline_stacker_ cv = model_selection.cross_val_score(lr, train, y, cv=self.n_fold_, scoring=metrics.make_scorer(metrics.log_loss)) lr.fit(train, y) v, z = self.v_, self.z_ z['p'] = np.clip(lr.predict(test), 1e-5, 1-1e-5) z['y'] = z['p'] v['p'] = model_selection.cross_val_predict(lr, train, y, cv=10) print('cv:', np.mean(cv), np.std(cv)) return cv, None
case, val = args if case == "case 1": return val else: return val**2 from hyperopt import hp space = hp.choice( "a", [ ("case 1", 1 + hp.lognormal("c1", 0, 1)), ("case 2", hp.uniform("c2", -10, 10)), ], ) from hyperopt import fmin, tpe, space_eval best = fmin(objective, space, algo=tpe.suggest, max_evals=100) print(best) print(space_eval(space, best)) print(objective(space_eval(space, best))) if __name__ == "__main__": # info = [] # test_method(sean_solution, info) # test_opt() test_file_writer()