def fit_xcorr_bo(detector_image, detector_pitch, as_to_m, src_peak_photons): # Assume npixels = 512 model = Model() # Use cross correlation to estimate major source location model.set_model_params([0,0,1.,0,0,0]) psf_template=model.model_image()[256-50:256+50,256-50:256+50] psf_template -= psf_template.mean() corr = signal.correlate2d(detector_image, psf_template, boundary='symm', mode='same') max_loc = peak_local_max(corr,num_peaks=1)[0] # Estimate major peak est_flux = detector_image.max(axis=None)*1. / src_peak_photons model_bsf = [(max_loc[0] - 256)*detector_pitch/as_to_m, (max_loc[1] - 256)*detector_pitch/as_to_m, est_flux,0.,0.,0.] model.set_model_params(model_bsf) # Fine tune major position with BO params_init = [(-.5+model_bsf[0],.5+model_bsf[0]),(-.5+model_bsf[1],.5+model_bsf[1])] res = gbrt_minimize(model.model_err_major_pos, params_init) model_bsf[0] = res.x[0] model_bsf[1] = res.x[1] model.set_pos_a(model_bsf[0],model_bsf[1]) # Tune major source flux with BO res = gp_minimize(model.model_err_major_flux, [(0.2,2.)]) model.set_major_flux(res.x[0]) model_bsf[2]=res.x[0] # Step 3: minor source position residual = detector_image - model.model_image() est_flux = residual.max(axis=None)*1. / src_peak_photons model.set_minor_flux(est_flux) model_bsf[5]=est_flux # Use cross correlation to estimate minor source location corr = signal.correlate2d(residual, psf_template, boundary='symm', mode='same') max_loc = peak_local_max(corr,num_peaks=1)[0] model_bsf[3] = (max_loc[0] - 256)*detector_pitch/as_to_m model_bsf[4] = (max_loc[1] - 256)*detector_pitch/as_to_m # Fine tune minor position with BO res = gbrt_minimize(model.model_err_minor_pos, [(-.5+model_bsf[3],.5+model_bsf[3]),(-.5+model_bsf[4],.5+model_bsf[4])]) model_bsf[3] = res.x[0] model_bsf[4] = res.x[1] model.set_pos_b(res.x[0],res.x[1]) # Final tweak params_init = [ (-.1+model_bsf[0],.1+model_bsf[0]), (-.1+model_bsf[1],.1+model_bsf[1]), (-.2+model_bsf[2],.2+model_bsf[2]), (-.1+model_bsf[3],.1+model_bsf[3]), (-.1+model_bsf[4],.1+model_bsf[4]), (-.2+model_bsf[5],.2+model_bsf[5])] res = gbrt_minimize(model.model_err, params_init, x0=[model_bsf]) return res
def opt_content_based(): space = [ Integer(0, 100), # artist Integer(0, 100), # album Integer(0, 100), # inferred album Integer(0, 100), # duration_tracks Integer(0, 100), # inferred duration Integer(0, 100), # playcount Integer(0, 100), # inferred playcount Integer(0, 100), # tags Integer(0, 100), # n_rating Integer(0, 100), # urm weight ] x0 = [100, 90, 80, 20, 10, 20, 10, 10, 10, 50] res = gbrt_minimize(objective, space, x0=x0, verbose=True, n_random_starts=20, n_calls=10000, n_jobs=-1, callback=result) print('Maximimum p@k found: {:6.5f}'.format(-res.fun)) print('Optimal parameters:') params = [ 'Artist', 'Album', 'inferred_album', 'duration_tracks', 'inferred_duration', 'playcount', 'inferred_playcount', 'tags', 'n_rating', 'urm_weight' ] for (p, x_) in zip(params, res.x): print('{}: {}'.format(p, x_))
def minimize(self, initial_shape, bounds, c3, n_random_starts, n_calls): return gbrt_minimize(self.change_shape_and_calculate, dimensions=bounds, x0=initial_shape, y0=c3, n_random_starts=n_random_starts, n_calls=n_calls)
def tune(): tune_results = gbrt_minimize( func=layout_cost, dimensions=PARAMS_SPACE, # callback=skopt.callbacks.DeltaYStopper(delta=5e-4, n_best=4), n_calls=20, ) best_geometry = params_to_geometry(tune_results.x) return best_geometry, tune_results
def _suggest_x(dims, x0, y0, random_start, random_state, opts): res = skopt.gbrt_minimize(lambda *args: 0, dims, n_calls=1, n_random_starts=1 if random_start else 0, x0=x0, y0=y0, random_state=random_state, kappa=opts["kappa"], xi=opts["xi"]) return res.x_iters[-1], res.random_state
def fit(self, X, Y, wavelengths): def cost_func(guess): assert(np.all(np.isfinite(guess))), guess guess = dict(zip(self.opt_vars, guess)) return np.nanmedian(np.abs((self(X, wavelengths, **guess) - Y) / Y)) return np.abs(np.nanmean(self(X, wavelengths, **guess) - Y)) return ((self(X, wavelengths, **guess) - Y) ** 2).sum() ** 0.5 from skopt import gbrt_minimize init = [(1e-2,100)]*len(self.opt_vars) # res = minimize(cost_func, init, tol=1e-6, options={'maxiter':1e3}, method='BFGS') res = gbrt_minimize(cost_func, init, n_random_starts=10000, n_calls=10000)#, method='SLSQP')#, tol=1e-10, options={'maxiter':1e5}, method='SLSQP') print(self.__name__, res.x, res.fun) self.trained_function = partial(self.function, wavelengths=wavelengths, **dict(zip(self.opt_vars, res.x)))
def _init_trial(trial, state): import skopt random_starts, x0, y0, dims = state.minimize_inputs(trial.run_id) res = skopt.gbrt_minimize( lambda *args: 0, dims, n_calls=1, n_random_starts=random_starts, x0=x0, y0=y0, random_state=state.random_state, kappa=state.batch_flags["kappa"], xi=state.batch_flags["xi"]) state.random_state = res.random_state return skopt_util.trial_flags(state.flag_names, res.x_iters[-1])
def run_session(self): super(SKOptSession, self)._write_headers(path=self.RESULTS_PATH) for planner in self.planners: params_set = dict(self.planner_config[planner].items()) search_space = self._load_search_space(params_set) self.n_trial = 0 # Reset to n_trials to zero for each planner self.planner = planner # Keeping track of current planner self.start_time = timer() # Keeping track of start_time if (self.MAX_RUNTIME != 'None'): self.end_time = self.start_time + self.MAX_RUNTIME # Keeping track of end_time rospy.loginfo('Executing %s on %s for %d secs', self.MODE, planner, self.MAX_RUNTIME) else: rospy.loginfo('Executing %s on %s for %d trials', self.MODE, planner, self.MAX_TRIALS) if self.MODE == 'gp': result = gp_minimize(self._skopt_obj, search_space, n_calls=self.MAX_TRIALS, random_state=0, acq_func='gp_hedge') # gp_hedge means probabilistically choose betwn LCB, EI and PI acquisition functions at every iteration elif self.MODE == 'rf': result = forest_minimize(self._skopt_obj, search_space, n_calls=self.MAX_TRIALS, random_state=0, base_estimator='RF', acq_func='EI') elif self.MODE == 'et': result = forest_minimize(self._skopt_obj, search_space, n_calls=self.MAX_TRIALS, random_state=0, base_estimator='ET', acq_func='EI') elif self.MODE == 'gbrt': result = gbrt_minimize(self._skopt_obj, search_space, n_calls=self.MAX_TRIALS, random_state=0, acq_func='EI') rospy.loginfo('Saved results to %s\n', self.RESULTS_PATH)
def opt_mf(): # Parameter to optimize: # components, user_reg, item_reg, l_rate space = [Integer(1, 10), # components Real(1e-9, 0.8), # user_reg Real(1e-9, 0.8), # pos_item_reg Real(1e-9, 0.8), # neg_item_reg Real(1e-5, 1), # l_rate ] x0 = [2, 1e-2, 1e-3, 1e-3, 5e-3] x1 = [3, 1e-2, 1e-3, 1e-2, 5e-3] x2 = [4, 1e-2, 1e-3, 1e-2, 5e-1] x3 = [4, 1e-2, 1e-2, 5e-3, 1e-1] x4 = [5, 1e-2, 1e-3, 1e-3, 5e-2] x0s = [x0, x1, x2, x3, x4] # get the current fold res = gbrt_minimize(objective, space, x0=x0s, verbose=True, n_random_starts=20, n_calls=1000, xi=0.01,n_jobs=-1, callback=result) print('Maximimum p@k found: {:6.5f}'.format(-res.fun)) print('Optimal parameters:') params = ['components', 'user_reg', 'CSLIM'] for (p, x_) in zip(params, res.x): print('{}: {}'.format(p, x_))
def globalVQE(ham_dict, TN, initial_guess=None, n_calls=100, pre_applied_circ=None): """Optimizes the TN by simultaneously changing all the parameters """ q = QuantumRegister(TN.n_qubits) c = ClassicalRegister(TN.n_qubits) def objective(x): TN.params = x return measure_ham(q, c, ham_dict, TN, pre_applied_circ=pre_applied_circ) res = gbrt_minimize(objective, [(0, 2 * pi)] * TN.n_params, n_calls=n_calls, verbose=False, n_random_starts=30) return res.fun
def optimize(X, y, range_file, learner_choice='brtR', optimizer='gp', cv_folds=10, n_calls=50): # read in ranges from range_file ranges = read_csv(range_file) # get number of features of input data n_features = X.shape[1] # make dataframe of limits parse-able by objective space = space_maker(ranges) # make weights weights = bernoulli(list(y)) # make compatible search spaces and objective functions for each learner_choice if (learner_choice == 'brtR' or learner_choice == 'brtC'): space += [(1, n_features)] # define objective function to minimize <- skopt def objective(params): (learning_rate, max_depth, n_estimators, max_features) = params if (learner_choice == 'brtR'): learner = GradientBoostingRegressor() elif (learner_choice == 'brtC'): learner = GradientBoostingClassifier() learner.set_params(learning_rate=learning_rate, loss='ls', max_depth=max_depth, max_features=max_features, n_estimators=n_estimators, subsample=0.75) return -mean( cross_val_score(learner, X, y, cv=cv_folds, n_jobs=-1, scoring="neg_mean_absolute_error")) elif (learner_choice == 'xgbR' or learner_choice == 'xgbC'): # define objective function to minimize <- xgboost def objective(params): (learning_rate, max_depth, n_estimators) = params if (learner_choice == 'xgbR'): learner = XGBRegressor() elif (learner_choice == 'xgbC'): learner = XGBClassifier() learner.set_params(booster='gbtree', learning_rate=learning_rate, max_depth=max_depth, n_estimators=n_estimators, subsample=0.75) return -mean( cross_val_score(learner, X, y, cv=cv_folds, n_jobs=-1, scoring="neg_mean_absolute_error")) else: error_lrn = 'not an available choice, use: brtR, brtC, xgbR, or xgbC' # run fits: gaussian process, random forest, gradient-boosted regression trees (resp.) if (optimizer == 'gp'): best_pars = skopt.gp_minimize(objective, space, n_calls=n_calls) elif (optimizer == 'rf'): best_pars = skopt.forest_minimize(objective, space, n_calls=n_calls) elif (optimizer == 'brt'): best_pars = skopt.gbrt_minimize(objective, space, n_calls=n_calls) else: error_opt = 'not an available choice, use: gp, rf, or brt' # return fit params conditional on no user type-o's if 'error_opt' in locals(): print(error_opt) return None elif 'error_lrn' in locals(): print(error_lrn) return None else: return best_pars.x
from matplotlib.colors import LogNorm from skopt import gbrt_minimize from skopt.benchmarks import branin from skopt.acquisition import gaussian_ei plt.figure(figsize=(10, 10)) plt.set_cmap("viridis") dimensions = [(-5.0, 10.0), (0.0, 15.0)] x1_values = np.linspace(-5, 10, 100) x2_values = np.linspace(0, 15, 100) x_ax, y_ax = np.meshgrid(x1_values, x2_values) vals = np.c_[x_ax.ravel(), y_ax.ravel()] res = gbrt_minimize( branin, dimensions, maxiter=200, random_state=1) model = res.models[-1] opt_points = res.x_iters y_opt = res.fun x_opt = res.x acquis_values = gaussian_ei(vals, model, y_opt) acquis_values = acquis_values.reshape(100, 100) branin_vals = np.reshape([branin(val) for val in vals], (100, 100)) plt.subplot(211) plt.pcolormesh(x_ax, y_ax, acquis_values) plt.plot(opt_points[:, 0], opt_points[:, 1], 'ro', markersize=4, lw=0, label='samples')
callback=None, n_points=num, xi=0.01, kappa=1.96, n_jobs=1) elif method == 'GradientBoostTree': gbrt_minimize(simulator.objfunction, dimensions=dimensions, base_estimator=None, n_calls=num, n_random_starts=10, acq_func='EI', acq_optimizer='auto', x0=None, y0=None, random_state=None, verbose=False, callback=None, n_points=num, xi=0.01, kappa=1.96, n_jobs=1) elif method == 'GaussianProcess': gp_minimize(simulator.objfunction, dimensions=dimensions, base_estimator=None, n_calls=num, n_random_starts=10,
import matplotlib.pyplot as plt from skopt import gbrt_minimize from skopt.benchmarks import bench3 from skopt.gbrt_opt import _expected_improvement bounds = [[-1, 1]] x = np.linspace(-1, 1, 200) func_values = [bench3(xi) for xi in x] plt.figure(figsize=(10, 5)) vals = np.reshape(x, (-1, 1)) col_no = 1 row_no = 6 res = gbrt_minimize( bench3, bounds, maxiter=6, n_start=1, random_state=1) best_xs = res.x_iters.ravel() best_ys = res.func_vals.ravel() models = res.models for n_iter in range(5): model = models[n_iter] best_x = best_xs[:n_iter+1] best_y = best_ys[:n_iter+1] low, mu, high = model.predict(vals).T std = (high - low) / 2 acquis_values = _expected_improvement(vals, model, best_y[-1]) acquis_values = acquis_values.ravel() posterior_mean = mu.ravel() posterior_std = std.ravel()
def findbestparams(modelclass, encoder, corpus, modelsfolder, n_calls=100, verbose=1, valmask=None, patience=20, maxepochs=1000, checkpointfile=None): """Find the best parameters for a given model architecture and param grid Returns - list with the best parameters found for the model - validation loss for such model - best model found - OptimizeResult object with info on the optimization procedure """ if verbose >= 1: print("Will save all tested models under %s" % modelsfolder) # Load checkpoint (if any) x0 = None y0 = None previoustrials = 0 if checkpointfile: prevtrials = checkpointload(checkpointfile) if len(prevtrials) > 0: params, losses = zip(*prevtrials) x0 = list(params) y0 = list(losses) previoustrials = len(x0) if verbose >= 1: print("Checkpoint x0", x0) print("Checkpoint y0", y0) # Prepare and run optimizer fobj = createobjective(modelclass, encoder, corpus, verbose=verbose, valmask=valmask, patience=patience, maxepochs=maxepochs, modelsfolder=modelsfolder, checkpointfile=checkpointfile) grid = addoptimizerparams(modelclass.paramgrid) trials = max(n_calls - previoustrials, 1) # Run remaining trials, minimum 10 randomtrials = max(RANDOMTRIALS - previoustrials, 0) optres = gbrt_minimize(fobj, grid, n_calls=trials, n_random_starts=randomtrials, random_state=0, x0=x0, y0=y0) # Recover best parameters, best loss, best model bestparams = optres.x bestloss = optres.fun bestmodel = load_model(modelsfolder + "/" + loss2modelname(bestloss)) return bestparams, bestloss, bestmodel, optres
def main(config): """Main Function""" seed = np.random.randint(0, high=10000) if 'seed' in config: seed = config['seed'] torch.manual_seed(seed) np.random.seed(seed) protected_index = descriptions.index(config['protected_attr']) prediction_index = descriptions.index(config['prediction_attr']) valid_results, test_results = {}, {} _, _, _, trainloader, valloader, testloader = load_celeba( trainsize=config['trainsize'], testsize=config['testsize'], num_workers=config['num_workers'], batch_size=config['batch_size']) if config['print_priors']: logger.info('train priors') compute_priors(trainloader, protected_index, prediction_index) logger.info('val priors') compute_priors(valloader, protected_index, prediction_index) logger.info('test priors') compute_priors(testloader, protected_index, prediction_index) net = get_resnet_model() criterion = nn.BCEWithLogitsLoss() if config['optimizer'] == 'sgd': optimizer = optim.SGD(net.parameters(), lr=config['lr']) else: optimizer = optim.Adam(net.parameters(), lr=config['lr']) checkpoint_file = Path('seed' + str(seed) + config['optimizer'] + str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' + config['prediction_attr'] + '_' + config['checkpoint']) start_epoch = 0 if checkpoint_file.is_file() and (not config['retrain']): checkpoint = torch.load(checkpoint_file, map_location=device) logger.info('loaded from %s', checkpoint_file) net.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) start_epoch = checkpoint['epoch'] else: train_model(net, trainloader, valloader, criterion, optimizer, str(checkpoint_file), protected_index, prediction_index, epochs=config['epochs'], start_epoch=start_epoch) _, best_thresh = val_model(net, valloader, get_best_balanced_accuracy, protected_index, prediction_index) logger.info(f'val_results, thresh, {best_thresh.item()}') valid_results['base_model'] = print_objective_results( valloader, net, best_thresh, protected_index, prediction_index) logger.info('test_results') result_dict = print_objective_results(testloader, net, best_thresh, protected_index, prediction_index) test_results['base_model'] = result_dict def to_dataframe(y_true, y_pred, y_prot): y_true, y_pred, y_prot = y_true.float().cpu().numpy(), y_pred.float( ).cpu().numpy(), y_prot.float().cpu().numpy() df = pd.DataFrame({ 'y_true': y_true, 'y_pred': y_pred, 'y_prot': y_prot }) dataset = StandardDataset(df, 'y_true', [1.], ['y_prot'], [[1.]]) dataset.scores = y_pred.reshape(-1, 1) return dataset val_dataset = val_model(net, valloader, to_dataframe, protected_index, prediction_index) test_dataset = val_model(net, testloader, to_dataframe, protected_index, prediction_index) def eval_aif360_algorithm(y_pred, dataset, verbose=True): global yaml_config acc = float(np.mean(y_pred == dataset.labels.reshape(-1))) bias = compute_bias( torch.tensor(y_pred), torch.tensor(dataset.labels.reshape(-1)), torch.tensor(dataset.protected_attributes.reshape(-1)), yaml_config['metric']).item() obj = compute_objective(acc, bias, margin=0.0) if verbose: logger.info(f'accuracy {acc}') logger.info(f'{yaml_config["metric"]} {bias}') logger.info(f'objective {obj}') return { 'roc_auc': None, 'accuracy': float(acc), 'bias': float(bias), 'objective': float(obj) } # Evaluate ROC if "ROC" in config['models']: ROC = RejectOptionClassification(unprivileged_groups=[{ 'y_prot': 1. }], privileged_groups=[{ 'y_prot': 0. }], low_class_thresh=0.01, high_class_thresh=0.99, num_class_thresh=100, num_ROC_margin=50, metric_name="Average odds difference", metric_ub=0.05, metric_lb=-0.05) logger.info("Training ROC model with validation dataset.") ROC = ROC.fit(val_dataset, val_dataset) logger.info("Evaluating ROC model.") logger.info('ROC val results') val_y_pred = ROC.predict(val_dataset).labels.reshape(-1) valid_results['ROC'] = eval_aif360_algorithm(val_y_pred, val_dataset) logger.info('ROC test results') test_y_pred = ROC.predict(test_dataset).labels.reshape(-1) test_results['ROC'] = eval_aif360_algorithm(test_y_pred, test_dataset) ROC = None if 'EqOdds' in config['models']: eo = EqOddsPostprocessing(privileged_groups=[{ 'y_prot': 0. }], unprivileged_groups=[{ 'y_prot': 1. }]) logger.info("Training Equality of Odds model with validation dataset.") eo = eo.fit(val_dataset, val_dataset) logger.info("Evaluating Equality of Odds model.") logger.info('Equality of Odds val results') val_y_pred = eo.predict(val_dataset).labels.reshape(-1) valid_results['EqOdds'] = eval_aif360_algorithm( val_y_pred, val_dataset) logger.info('Equality of Odds test results') test_y_pred = eo.predict(test_dataset).labels.reshape(-1) test_results['EqOdds'] = eval_aif360_algorithm(test_y_pred, test_dataset) eo = None if 'CalibEqOdds' in config['models']: cost_constraint = config['CalibEqOdds']['cost_constraint'] cpp = CalibratedEqOddsPostprocessing(privileged_groups=[{ 'y_prot': 0. }], unprivileged_groups=[{ 'y_prot': 1. }], cost_constraint=cost_constraint) logger.info( "Training Calibrated Equality of Odds model with validation dataset." ) cpp = cpp.fit(val_dataset, val_dataset) logger.info("Evaluating Calibrated Equality of Odds model.") logger.info('Calibrated Equality of Odds val results') valid_y_pred = cpp.predict(val_dataset).labels.reshape(-1) valid_results['CalibEqOdds'] = eval_aif360_algorithm( valid_y_pred, val_dataset) logger.info('Equality of Odds test results') test_y_pred = cpp.predict(test_dataset).labels.reshape(-1) test_results['CalibEqOdds'] = eval_aif360_algorithm( valid_y_pred, val_dataset) cpp = None if 'random' in config['models']: rand_result = [-np.inf, None, -1] for iteration in range(101): rand_model = copy.deepcopy(net) rand_model.to(device) for param in rand_model.parameters(): param.data = param.data * (torch.randn_like(param) * 0.1 + 1) rand_model.eval() best_obj, best_thresh = val_model(rand_model, valloader, get_best_objective, protected_index, prediction_index) logger.info(f'iteration {iteration} obj {float(best_obj)}') if best_obj > rand_result[0]: logger.info('found new best') del rand_result[1] rand_result = [ best_obj, copy.deepcopy(rand_model.state_dict()), best_thresh ] if iteration % 10 == 0: logger.info(f"{iteration} / 101 trials have been sampled.") logger.info(f'current best obj {float(rand_result[0])}') # evaluate best random model best_model = copy.deepcopy(net) best_model.load_state_dict(rand_result[1]) best_model.to(device) best_thresh = rand_result[2] logger.info('val_results') valid_results['random'] = print_objective_results( valloader, best_model, best_thresh, protected_index, prediction_index) logger.info('test_results') test_results['random'] = print_objective_results( testloader, best_model, best_thresh, protected_index, prediction_index) torch.save( best_model.state_dict(), 'seed' + str(seed) + config['metric'] + config['optimizer'] + str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' + config['prediction_attr'] + config['random']['checkpoint']) if 'layerwiseOpt' in config['models']: base_model = copy.deepcopy(net) best_state_dict, best_thresh, best_obj = None, None, np.inf total_params = len(list(base_model.parameters())) for index, param in enumerate(base_model.parameters()): if index < total_params - config['layerwiseOpt']['num_layers']: continue logger.info(f'Evaluating param number {index} of {total_params}') param_copy = copy.deepcopy(param) def objective(new_param): param.data[indices] = torch.tensor(new_param).to(device) base_model.eval() best_obj, thresh = val_model(base_model, valloader, get_best_objective, protected_index, prediction_index) logger.info( f'Evaluating param number {index} of {total_params}') return -float(best_obj) std = param.flatten().cpu().detach().numpy().std() num_elems = param.size().numel() ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems) indices = torch.rand(param.size()) < ratio space = [ Real( float(x.cpu().detach()) - 2.2 * std, float(x.cpu().detach()) + 2.2 * std) for x in param[indices] ] logger.info(f'Number of sparse indices: {indices.sum().item()}') res_gbrt = gbrt_minimize(objective, space, n_calls=20, verbose=True) if res_gbrt.fun < best_obj: param.data[indices] = torch.tensor(res_gbrt.x).to(device) best_state_dict = copy.deepcopy(base_model.state_dict()) best_obj, best_thresh = val_model(base_model, valloader, get_best_objective, protected_index, prediction_index) param.data = param_copy.data best_model = copy.deepcopy(net) best_model.load_state_dict(best_state_dict) best_model.to(device) logger.info('val_results') valid_results['layerwiseOpt'] = print_objective_results( valloader, best_model, best_thresh, protected_index, prediction_index) logger.info('test_results') test_results['layerwiseOpt'] = print_objective_results( testloader, best_model, best_thresh, protected_index, prediction_index) if 'adversarial' in config['models']: unrefined_net = get_resnet_model() base_model = copy.deepcopy(unrefined_net) base_model.fc = nn.Linear(base_model.fc.in_features, base_model.fc.in_features) actor = nn.Sequential(base_model, nn.Linear(base_model.fc.in_features, 2)) actor.to(device) actor_optimizer = optim.Adam(actor.parameters(), lr=1e-4) actor_loss_fn = nn.BCEWithLogitsLoss() actor_loss = 0. actor_steps = config['adversarial']['actor_steps'] critic = Critic(config['batch_size'] * unrefined_net.fc.in_features) critic.to(device) critic_optimizer = optim.Adam(critic.parameters(), lr=1e-4) critic_loss_fn = nn.MSELoss() critic_loss = 0. critic_steps = config['adversarial']['critic_steps'] for epoch in range(config['adversarial']['epochs']): for param in critic.parameters(): param.requires_grad = True for param in actor.parameters(): param.requires_grad = False actor.eval() critic.train() for step, (inputs, labels) in enumerate(valloader): if step > critic_steps: break inputs, labels = inputs.to(device), labels.to(device) if inputs.size(0) != config['batch_size']: continue critic_optimizer.zero_grad() with torch.no_grad(): y_pred = actor(inputs) y_true = labels[:, prediction_index].float().to(device) y_prot = labels[:, protected_index].float().to(device) bias = compute_bias(y_pred, y_true, y_prot, config['metric']) res = critic(base_model(inputs)) loss = critic_loss_fn(bias.unsqueeze(0), res[0]) loss.backward() critic_loss += loss.item() critic_optimizer.step() if step % 100 == 0: print_loss = critic_loss if ( epoch * critic_steps + step ) == 0 else critic_loss / (epoch * critic_steps + step) logger.info( f'=======> Epoch: {(epoch, step)} Critic loss: {print_loss:.3f}' ) for param in critic.parameters(): param.requires_grad = False for param in actor.parameters(): param.requires_grad = True actor.train() critic.eval() for step, (inputs, labels) in enumerate(valloader): if step > actor_steps: break inputs, labels = inputs.to(device), labels.to(device) if inputs.size(0) != config['batch_size']: continue actor_optimizer.zero_grad() y_true = labels[:, prediction_index].float().to(device) y_prot = labels[:, protected_index].float().to(device) est_bias = critic(base_model(inputs)) loss = actor_loss_fn(actor(inputs)[:, 0], y_true) loss = max( 1, config['adversarial']['lambda'] * (abs(est_bias) - config['objective']['epsilon'] + config['adversarial']['margin']) + 1) * loss loss.backward() actor_loss += loss.item() actor_optimizer.step() if step % 100 == 0: print_loss = critic_loss if ( epoch * actor_steps + step ) == 0 else critic_loss / (epoch * actor_steps + step) logger.info( f'=======> Epoch: {(epoch, step)} Actor loss: {print_loss:.3f}' ) print_objective_results(valloader, actor, best_thresh, protected_index, prediction_index) _, best_thresh = val_model(actor, valloader, get_best_objective, protected_index, prediction_index) logger.info('val_results') valid_results['adversarial'] = print_objective_results( valloader, actor, best_thresh, protected_index, prediction_index) logger.info('test_results') test_results['adversarial'] = print_objective_results( testloader, actor, best_thresh, protected_index, prediction_index) torch.save(actor.state_dict(), config['adversarial']['checkpoint']) with open( 'valid_seed' + str(seed) + config['metric'] + config['optimizer'] + str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' + config['prediction_attr'] + config['output'], 'w') as filehandler: json.dump(valid_results, filehandler) with open( 'test_seed' + str(seed) + config['metric'] + config['optimizer'] + str(config['lr']) + '_pro_' + config['protected_attr'] + '_pre_' + config['prediction_attr'] + config['output'], 'w') as filehandler: json.dump(test_results, filehandler)
def train(): (xtrain, ytrain), (xvalid, yvalid), (_, _), y_std, y_mean = get_mc_data(FLAGS.dataset_name) min_tau = setup['tau_range'][FLAGS.dataset_name][0] max_tau = setup['tau_range'][FLAGS.dataset_name][1] # FIND BEST N_EPOCHS; TAU with tf.Graph().as_default() as g: with tf.Session() as sess: saver, model_dir, pyx, x, n_epochs = train_model(sess, xtrain, ytrain, FLAGS.seed, FLAGS.seed, xvalid, yvalid) # CRPS OPTIMIZE TO FIND STD DEV print "Finding optimal Tau" saver.restore(sess, model_dir + 'model') # Make optimization run take extra arguments optimize_fun = partial(run_tau_opt, sess, pyx, x, y_std, y_mean, xvalid, yvalid, False) tau_opt = gbrt_minimize(optimize_fun, [(min_tau, max_tau)], n_random_starts=100, n_calls=200 ) opt_tau = tau_opt.x[0] # CRPS OPTIMIZE TO FIND STD DEV print "Finding optimal CU Tau" # Make optimization run take extra arguments optimize_fun = partial(run_tau_opt, sess, pyx, x, y_std, y_mean, xvalid, yvalid, True) cutau_opt = gbrt_minimize(optimize_fun, [(min_tau, max_tau)], n_random_starts=100, n_calls=200 ) cu_opt_tau = cutau_opt.x[0] print "OPT TAU: {}. CRPS: {}".format(opt_tau, tau_opt.fun) print "CU OPT TAU: {}. CRPS: {}".format(cu_opt_tau, cutau_opt.fun) tf.reset_default_graph() # TRAIN AND EVALUATE FINAL MODEL 5 TIMES WITH DIFFERENT SEED: for final_seed in range(FLAGS.seed + 1, FLAGS.seed + 6): (xtrain, ytrain), (xtest, ytest), y_std, y_mean = get_mc_data(FLAGS.dataset_name, False) with tf.Graph().as_default() as g: with tf.Session() as sess: # Write csv file column headers if not yet written. plot_file_path = os.path.join(PLOT_RESULTS_PATH, FLAGS.dataset_name + '.txt') fid = open(plot_file_path, 'a') if sum(1 for line in open(plot_file_path)) == 0: fid.write(',MNF const std dev,MNF std dev,y,yHat,run_count,dataset_split_seed\n') pll_file_path = os.path.join(RESULTS_PATH, FLAGS.dataset_name + '-pll.txt') fid_pll = open(pll_file_path, 'a') if sum(1 for line in open(pll_file_path)) == 0: fid_pll.write('dataset_split,run_count,pll result,pll baseline,pll best,pll normalized\n') crps_file_path = os.path.join(RESULTS_PATH, FLAGS.dataset_name + '-crps.txt') fid_crps = open(crps_file_path, 'a') if sum(1 for line in open(crps_file_path)) == 0: fid_crps.write('dataset_split,run_count,crps result,crps baseline,crps best,crps normalized\n') rmse_file_path = os.path.join(RESULTS_PATH, FLAGS.dataset_name + '-rmse.txt') fid_rmse = open(rmse_file_path, 'a') if sum(1 for line in open(rmse_file_path)) == 0: fid_rmse.write('dataset_split,run_count,rmse\n') saver, model_dir, pyx, x, _ = train_model(sess, xtrain, ytrain, final_seed, final_seed, xvalid=False, yvalid=False, n_epochs=n_epochs) #EVALUATE TEST SET preds = np.zeros_like(ytest) all_preds = np.zeros([len(ytest), FLAGS.L]) widgets = ["Sampling |", Percentage(), Bar(), ETA()] pbar = ProgressBar(FLAGS.L, widgets=widgets) pbar.start() for i in range(FLAGS.L): pbar.update(i) for j in range(int(xtest.shape[0] / 100)): pyxi = sess.run(pyx, feed_dict={x: xtest[j * 100:(j + 1) * 100]}) preds[j * 100:(j + 1) * 100] += pyxi / FLAGS.L all_preds[j * 100:(j + 1) * 100, i] = np.squeeze(pyxi * y_std + y_mean) pyxi = sess.run(pyx, feed_dict={x: xtest[int(xtest.shape[0] / 100) * 100:]}) preds[int(xtest.shape[0] / 100) * 100:] += pyxi / FLAGS.L all_preds[int(xtest.shape[0] / 100) * 100:, i] = np.squeeze(pyxi * y_std + y_mean) # FIND PLL AND CRPS samples = all_preds[:, :, newaxis].T.reshape(FLAGS.L, len(all_preds), 1) mean, var = np.mean(samples, axis=0), np.var(samples, axis=0) + opt_tau ** (-1) pll_res = pll(samples, ytest * y_std + y_mean, FLAGS.L, opt_tau) crps_res = crps(ytest * y_std + y_mean, mean, var) # FIND BASELINE PLL AND CRPS pll_baseline = pll(np.array([mean]), ytest * y_std + y_mean, 1, cu_opt_tau) crps_baseline = crps(ytest * y_std + y_mean, mean, cu_opt_tau**(-1)) # FIND OPTIMAL PLL AND CRPS pll_best = pll_maximum(mean, ytest * y_std + y_mean) crps_best = crps_minimum(mean, ytest * y_std + y_mean) # GET NORMALIZED SCORES pll_norm = (pll_res - pll_baseline) / (pll_best - pll_baseline) crps_norm = (crps_res - crps_baseline) / (crps_best - crps_baseline) sample_accuracy = np.sqrt(np.mean((preds-ytest)*(preds-ytest))) print('Sample test accuracy: {}'.format(sample_accuracy)) ytest_u = (ytest * y_std + y_mean) preds_u = (preds * y_std + y_mean) unnormalized_rmse = np.sqrt(np.mean((preds_u - ytest_u) * (preds_u - ytest_u))) print('Sample test accuracy (unnormalized): {}'.format(unnormalized_rmse)) print('Test uncertainty quality metrics:') print "PLL: {}, PLL LOWER: {}, PLL UPPER: {}, NORM: {}".format(pll_res, pll_baseline, pll_best, pll_norm) print "CRPS: {}, CRPS LOWER: {}, CRPS UPPER: {}, NORM: {}".format(crps_res, crps_baseline, crps_best, crps_norm) all_preds_mean = all_preds.mean(axis=1) all_preds_std = all_preds.std(axis=1) # Write results to files for i in range(len(ytest)): fid.write('%d,%f,%f,%f,%f,%d,%d\n' % (i, np.sqrt(cu_opt_tau**(-1)), all_preds_std[i], ytest_u[i], all_preds_mean[i], final_seed, FLAGS.dataset_split_seed)) fid_rmse.write('%d,%d,%f\n' % (FLAGS.dataset_split_seed, final_seed, unnormalized_rmse)) fid_pll.write('%d,%d %f,%f,%f,%f\n' % (FLAGS.dataset_split_seed, final_seed, pll_res, pll_baseline, pll_best, pll_norm)) fid_crps.write('%d,%d,%f,%f,%f,%f\n' % (FLAGS.dataset_split_seed, final_seed, crps_res, crps_baseline, crps_best, crps_norm)) fid.close() fid_rmse.close() fid_pll.close() fid_crps.close() tf.reset_default_graph()
print('Last epoch: ', epoch) print('Credit Cost: ', CreditCost) if os.path.exists('checkpoint.pt'): os.remove('checkpoint.pt') print() torch.cuda.empty_cache() print() return CreditCost # not working #res_gp = gp_minimize(objective, dimensions=dimensions, n_calls=TRIALS, random_state=1, verbose=True, acq_func='gp_hedge', acq_optimizer='auto', n_jobs=1) # res_gp = forest_minimize(objective, dimensions=dimensions, base_estimator='RF', n_calls=TRIALS, n_random_starts=RANDOM_STARTS, acq_func='EI', x0=None, y0=None, random_state=None, verbose=True, callback=None, n_points=10000, xi=0.01, kappa=1.96, n_jobs=128) res_gp = gbrt_minimize(objective, dimensions=dimensions, base_estimator='ET', n_calls=TRIALS + RANDOM_STARTS, n_random_starts=RANDOM_STARTS, acq_func='LCB', x0=None, y0=None, random_state=None, verbose=True, callback=None, n_points=100, xi=0.01, kappa=1.96, n_jobs=1) # res_gp = dummy_minimize(objective, dimensions=dimensions, n_calls=TRIALS, x0=None, y0=None, random_state=None, verbose=True, callback=None) "Best score=%.4f" % res_gp.fun print("""Best parameters: - optimization=%d""" % (res_gp.x[0])) print(res_gp) plot_convergence(res_gp) # plot_evaluations(res_gp) # plot_objective(res_gp) # In[4]: if OPTIMIZATION_PLUGIN == 'Bayesian' : from bayes_opt import BayesianOptimization
from skopt import gbrt_minimize from skopt.benchmarks import branin from skopt.gbrt_opt import _expected_improvement plt.figure(figsize=(10, 10)) plt.set_cmap("viridis") bounds = np.asarray([[-5, 10], [0, 15]]) x1_values = np.linspace(-5, 10, 100) x2_values = np.linspace(0, 15, 100) x_ax, y_ax = np.meshgrid(x1_values, x2_values) vals = np.c_[x_ax.ravel(), y_ax.ravel()] res = gbrt_minimize( branin, bounds, maxiter=200, random_state=1) model = res.models[-1] opt_points = res.x_iters y_opt = res.fun x_opt = res.x acquis_values = _expected_improvement(vals, model, y_opt) acquis_values = acquis_values.reshape(100, 100) branin_vals = np.reshape([branin(val) for val in vals], (100, 100)) plt.subplot(211) plt.pcolormesh(x_ax, y_ax, acquis_values) plt.plot(opt_points[:, 0], opt_points[:, 1], 'ro', markersize=4, lw=0, label='samples')
def layerwiseOpt_debiasing(model_state_dict, data, config, device): logger.info('Training layerwiseOpt model.') base_model = load_model(data.num_features, config.get('hyperparameters', {})) base_model.load_state_dict(model_state_dict) base_model.to(device) best_state_dict, best_obj, best_thresh = None, math.inf, -1 total_params = len(list(base_model.parameters())) for index, param in enumerate(base_model.parameters()): if index < total_params - config['layerwiseOpt']['num_layers']: continue logger.info(f'Evaluating param number {index} of {total_params}') param_copy = copy.deepcopy(param) def objective(new_param, return_thresh=False): param.data[indices] = torch.tensor(new_param) base_model.eval() with torch.no_grad(): scores = base_model(data.X_valid_gpu)[:, 0].reshape(-1).numpy() best_thresh, best_obj = get_best_thresh( scores, np.linspace(0, 1, 501), data, config, valid=False, margin=config['layerwiseOpt']['margin']) print(f'Evaluating param number {index} of {total_params}') if return_thresh: return -float(best_obj), float(best_thresh) return -float(best_obj) mean = param.flatten().cpu().detach().numpy().mean() std = param.flatten().cpu().detach().numpy().std() num_elems = param.size().numel() ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems) indices = torch.rand(param.size()) < ratio space = [ Real( float(x.cpu().detach()) - 2.2 * std, float(x.cpu().detach()) + 2.2 * std) for x in param[indices] ] # std = param.flatten().cpu().detach().numpy().std() # num_elems = param.size().numel() # ratio = min(1., config['layerwiseOpt']['max_sparsity'] / num_elems) # indices = torch.rand(param.size()) < ratio logger.info(f'Number of sparse indices: {indices.sum().item()}') res_gbrt = gbrt_minimize(objective, space, n_calls=config['layerwiseOpt']['n_calls'], verbose=True) if res_gbrt.fun < best_obj: param.data[indices] = torch.tensor(res_gbrt.x) best_state_dict = base_model.state_dict() best_obj, best_thresh = objective(res_gbrt.x, return_thresh=True) best_obj = -best_obj param.data = param_copy.data best_model = load_model(data.num_features, config.get('hyperparameters', {})) best_model.to(device) best_model.load_state_dict(best_state_dict) best_model.eval() with torch.no_grad(): y_pred = (best_model(data.X_valid_gpu)[:, 0] > best_thresh).reshape(-1).numpy() results_valid = get_valid_objective(y_pred, data, config) best_model.eval() with torch.no_grad(): y_pred = (best_model(data.X_test_gpu)[:, 0] > best_thresh).reshape(-1).numpy() results_test = get_test_objective(y_pred, data, config) return results_valid, results_test
def get_params_SKopt(model, X, Y, space, cv_search, alg = 'catboost', cat_features = None, eval_dataset = None, UBM = False, opt_method = 'gbrt_minimize', verbose = True, multi = False, scoring = 'neg_mean_squared_error', n_best = 50, total_time = 7200): """The method performs parameters tuning of an algorithm using scikit-optimize library. Parameters: 1. 2. 3. multi - boolean, is used when a multioutput algorithm is tuned UPDATES: 1. In this current version, the support of the catboost algorithms is added """ if alg == 'catboost': fitparam = { 'eval_set' : eval_dataset, 'use_best_model' : UBM, 'cat_features' : cat_features, 'early_stopping_rounds': 20 } else: fitparam = {} @use_named_args(space) def objective(**params): model.set_params(**params) return -np.mean(cross_val_score(model, X, Y, cv=cv_search, scoring= scoring, fit_params=fitparam)) if opt_method == 'gbrt_minimize': HPO_PARAMS = {'n_calls':1000, 'n_random_starts':20, 'acq_func':'EI',} reg_gp = gbrt_minimize(objective, space, n_jobs = -1, verbose = verbose, callback = [DeltaYStopper(delta = 0.01, n_best = 5), RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)], **HPO_PARAMS, random_state = RANDOM_STATE) elif opt_method == 'forest_minimize': HPO_PARAMS = {'n_calls':1000, 'n_random_starts':20, 'acq_func':'EI',} reg_gp = forest_minimize(objective, space, n_jobs = -1, verbose = verbose, callback = [RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)], **HPO_PARAMS, random_state = RANDOM_STATE) elif opt_method == 'gp_minimize': HPO_PARAMS = {'n_calls':1000, 'n_random_starts':20, 'acq_func':'gp_hedge',} reg_gp = gp_minimize(objective, space, n_jobs = -1, verbose = verbose, callback = [RepeatedMinStopper(n_best = n_best), DeadlineStopper(total_time = total_time)], **HPO_PARAMS, random_state = RANDOM_STATE) TUNED_PARAMS = {} for i, item in enumerate(space): if multi: TUNED_PARAMS[item.name.split('__')[1]] = reg_gp.x[i] else: TUNED_PARAMS[item.name] = reg_gp.x[i] return [TUNED_PARAMS,reg_gp]
def compare_optimizers(num_instances=4, graph_size=15, n_calls=8, n_random_starts=2): global pbar pbar = None # For progress bar. pbar = tqdm(total=num_instances*n_calls*4) instances = [Graph(graph_size) for _ in range(num_instances)] # Percent of optimal score acheived by each algorithm. dummy = [] decision_trees = [] gradient_boosted_trees = [] baynesian = [] # For each instance, run each algorithm. for inst in instances: # Scikit functions only take in parameters and want to minimize values. # Create a wrapper function to format get_expectation. sk_get_exp = lambda x: -1*get_expectation(x, inst) opt = inst.optimal_score()[0] # Dummy. inst.clear_runs() dummy_minimize(func=sk_get_exp, dimensions=[(0,2*np.pi),(0,np.pi)], n_calls=n_calls) dummy.append(float(inst.currentScore) / opt) # Decision Trees. inst.clear_runs() forest_minimize(func=sk_get_exp, dimensions=[(0,2*np.pi),(0,np.pi)], n_calls=n_calls, n_random_starts=n_random_starts) decision_trees.append(float(inst.currentScore) / opt) # Gradient Boosted Decision Trees. inst.clear_runs() gbrt_minimize(func=sk_get_exp, dimensions=[(0,2*np.pi),(0,np.pi)], n_calls=n_calls, n_random_starts=n_random_starts) gradient_boosted_trees.append(float(inst.currentScore) / opt) # Baynesian. inst.clear_runs() gp_minimize(func=sk_get_exp, dimensions=[(0,2*np.pi),(0,np.pi)], n_calls=n_calls, n_random_starts=n_random_starts) baynesian.append(float(inst.currentScore) / opt) # Compare mean/stdev of % opt. achieved for each algorithm. print("-- % of Optimal Achieved, Mean and Std. Dev --") print("Random Sampling:\nMean: %s\nStd. Dev: %s" % (mean(dummy), stdev(dummy))) print("Decision Trees:\nMean: %s\nStd. Dev: %s" % (mean(decision_trees), stdev(decision_trees))) print("Gradient Boosted Decision Trees:\nMean: %s\nStd. Dev: %s" % (mean(gradient_boosted_trees), stdev(gradient_boosted_trees))) print("Baynesian Optimization:\nMean: %s\nStd. Dev: %s" % (mean(baynesian), stdev(baynesian)))
alpha = params['alpha'] elif method is "skopt": alpha = params[0] score = mapk_similarity(alpha) if method is "hyperopt": print("INFO: iteration {} error {:.3f}".format(sim_objective.i, score)) return 1-score partial_objective = lambda params: sim_objective(params, method=method) hp_params = {'alpha': hp.uniform('alpha', 0.01, 1.)} method = "hyperopt" sim_objective.i=0 hp_best = fmin(fn=partial_objective, space=hp_params, algo=tpe.suggest, max_evals=100) sk_params = [(0.01, 1, 'uniform')] method = "skopt" sk_best = gbrt_minimize(partial_objective, sk_params, n_calls=100, random_state=0, verbose=False, n_jobs=-1)
def any_order_VQE(ham_dict, TN, tensor_order=None, n_calls=20, method="GP"): """ Performs VQE by optimizing the tensor network in the order supplied in tensor_order. The n_sweeps parameter is redundant as one can simply send a duplicate of the tuple or call this function again :param ham_dict: Hamiltonian to minimize :param TN: TensorNetwork used as ansatz :param tensor_order: the order of tensors. Can be a tuple with integers or tuples of integers. Integer means 'optimize this tensor', tuple means 'optimize these tensors jointly' :param method: method of optimization. Can be "GP", "local" :returns: value of the function. The `res` object is of limited meaning, the parameters are stored in TN """ H = hamiltonians.explicit_hamiltonian(ham_dict) if not isinstance(TN, TensorNetwork.TensorNetwork): raise TypeError('Not a TensorNetwork') q = QuantumRegister(TN.n_qubits) c = ClassicalRegister(TN.n_qubits) circ = QuantumCircuit(q, c) block_size = TN.entangler.n_params n_entanglers = TN.n_tensors if tensor_order is None: order = tuple(i for i in range(n_entanglers)) else: order = tensor_order for v in order: if isinstance(v, int): param_numbers = [v * block_size + i for i in range(block_size)] elif isinstance(v, tuple): param_numbers = [] for j in v: param_numbers += [ j * block_size + i for i in range(block_size) ] else: raise TypeError('tensor_order should be a tuple or nested tuple') print('Optimizing params', param_numbers, end=' -> ') def f(x): return arb_index_objective([float64(xi) for xi in x], param_numbers, ham_dict, TN, q, c, circuit=circ, explH=H) x0 = [TN.params[i] for i in param_numbers] res = gbrt_minimize(f, [(0, 2 * pi)] * len(x0), x0=x0, n_calls=n_calls, verbose=False) #res = shgo(f, [(0, 4*pi)] * len(x0)) for i, num in enumerate(param_numbers): TN.params[num] = res.x[i] print('E = {0:0.4f}'.format(res.fun)) return res.fun
def main(): args = setup() print("Preparing directories") filename = f"{args.prefix}{args.model}_{args.data}_{args.estimator}{args.suffix}" factors_path = os.path.join(args.root_dir, "factors", filename) weights_path = os.path.join(args.root_dir, "weights", f"{args.model}_{args.data}.pth") if args.exp_id == -1: if not args.no_results: os.makedirs(os.path.join(args.results_dir, args.model, "data", args.estimator, args.optimizer), exist_ok=True) if args.plot: os.makedirs(os.path.join(args.results_dir, args.model, "figures", args.estimator, args.optimizer), exist_ok=True) results_path = os.path.join(args.results_dir, args.model, "data", args.estimator, args.optimizer, filename) else: if not args.no_results: os.makedirs(os.path.join(args.results_dir, args.model, "data", args.estimator, args.optimizer, args.exp_id), exist_ok=True) if args.plot: os.makedirs(os.path.join(args.results_dir, args.model, "figures", args.estimator, args.optimizer, args.exp_id), exist_ok=True) results_path = os.path.join(args.results_dir, args.model, "data", args.estimator, args.optimizer, args.exp_id, filename) print("Loading model") if args.model == 'lenet5': model = lenet5(pretrained=args.data, device=args.device) elif args.model == 'resnet18' and args.data != 'imagenet': model = resnet18(pretrained=weights_path, num_classes=43 if args.data == 'gtsrb' else 10, device=args.device) else: model_class = getattr(torchvision.models, args.model) if args.model in ['googlenet', 'inception_v3']: model = model_class(pretrained=True, aux_logits=False) else: model = model_class(pretrained=True) model.to(args.device).eval() if args.parallel: model = torch.nn.parallel.DataParallel(model) print("Loading data") if args.data == 'mnist': val_loader = datasets.mnist(args.torch_data, splits='val') elif args.data == 'cifar10': val_loader = datasets.cifar10(args.torch_data, splits='val') elif args.data == 'gtsrb': val_loader = datasets.gtsrb(args.data_dir, batch_size=args.batch_size, splits='val') elif args.data == 'imagenet': img_size = 224 if args.model in ['googlenet', 'inception_v3']: img_size = 299 val_loader = datasets.imagenet(args.data_dir, img_size, args.batch_size, args.workers, splits='val', use_cache=True, pre_cache=True) print("Loading factors") if args.estimator in ["diag", "kfac"]: factors = torch.load(factors_path + '.pth') elif args.estimator == 'efb': kfac_factors = torch.load(factors_path.replace("efb", "kfac") + '.pth') lambdas = torch.load(factors_path + '.pth') factors = list() eigvecs = get_eigenvectors(kfac_factors) for eigvec, lambda_ in zip(eigvecs, lambdas): factors.append((eigvec[0], eigvec[1], lambda_)) elif args.estimator == 'inf': factors = torch.load(f"{factors_path}{args.rank}.pth") torch.backends.cudnn.benchmark = True norm_min = -10 norm_max = 10 scale_min = -10 scale_max = 10 if args.boundaries: x0 = list() boundaries = [[norm_min, scale_min], [norm_max, scale_max], [norm_min, scale_max], [norm_max, scale_min], [norm_min / 2., scale_min], [norm_max / 2., scale_max], [norm_min, scale_max / 2.], [norm_max, scale_min / 2.], [norm_min / 2., scale_min / 2.], [norm_max / 2., scale_max / 2.], [norm_min / 2., scale_max / 2.], [norm_max / 2., scale_min / 2.]] for b in boundaries: tmp = list() for _ in range(3 if args.layer else 1): tmp.extend(b) x0.append(tmp) else: x0 = None f_norms = np.array([factor.norm().cpu().numpy() for factor in factors]) space = list() for i in range(3 if args.layer else 1): space.append( skopt.space.Real(norm_min, norm_max, name=f"norm{i}", prior='uniform')) space.append( skopt.space.Real(scale_min, scale_max, name=f"scale{i}", prior='uniform')) stats = { "norms": [], "scales": [], "acc": [], "ece": [], "nll": [], "ent": [], "cost": [] } @skopt.utils.use_named_args(dimensions=space) def objective(**params): norms = list() scales = list() for f in f_norms: if args.layer: # Closest to max if abs(f_norms.max() - f) < abs(f_norms.min() - f) and abs( f_norms.max() - f) < abs(f_norms.mean() - f): norms.append(10**params['norm0']) scales.append(10**params['scale0']) # Closest to min elif abs(f_norms.min() - f) < abs(f_norms.max() - f) and abs( f_norms.min() - f) < abs(f_norms.mean() - f): norms.append(10**params['norm1']) scales.append(10**params['scale1']) # Closest to mean else: norms.append(10**params['norm2']) scales.append(10**params['scale2']) else: norms.append(10**params['norm0']) scales.append(10**params['scale0']) if args.layer: print( tabulate.tabulate( { 'Layer': np.arange(len(factors)), 'F-Norm:': f_norms, 'Norms': norms, 'Scales': scales }, headers='keys', numalign='right')) else: print("Norm:", norms[0], "Scale:", scales[0]) try: inv_factors = invert_factors(factors, norms, args.pre_scale * scales, args.estimator) except (RuntimeError, np.linalg.LinAlgError): print(f"Error: Singular matrix") return 200 predictions, labels, _ = eval_bnn(model, val_loader, inv_factors, args.estimator, args.samples, stats=False, device=args.device, verbose=False) err = 100 - accuracy(predictions, labels) ece = 100 * expected_calibration_error(predictions, labels)[0] nll = negative_log_likelihood(predictions, labels) ent = predictive_entropy(predictions, mean=True) stats["norms"].append(norms) stats["scales"].append(scales) stats["acc"].append(100 - err) stats["ece"].append(ece) stats["nll"].append(nll) stats["ent"].append(ent) stats["cost"].append(err + ece) print( f"Err.: {err:.2f}% | ECE: {ece:.2f}% | NLL: {nll:.3f} | Ent.: {ent:.3f}" ) return err + ece with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FutureWarning) if args.optimizer == "gbrt": res = skopt.gbrt_minimize(func=objective, dimensions=space, n_calls=args.calls, x0=x0, verbose=True, n_jobs=args.workers, n_random_starts=0 if x0 else 10, acq_func='EI') # EI (neg. expected improvement) # LCB (lower confidence bound) # PI (neg. prob. of improvement): Usually favours exploitation over exploration # gp_hedge (choose probabilistically between all) if args.optimizer == "gp": res = skopt.gp_minimize(func=objective, dimensions=space, n_calls=args.calls, x0=x0, verbose=True, n_jobs=args.workers, n_random_starts=0 if x0 else 1, acq_func='gp_hedge') # acq_func: EI (neg. expected improvement), LCB (lower confidence bound), PI (neg. prob. of improvement) # xi: how much improvement one wants over the previous best values. # kappa: Importance of variance of predicted values. High: exploration > exploitation # base_estimator: RF (random forest), ET (extra trees) elif args.optimizer == "forest": res = skopt.forest_minimize(func=objective, dimensions=space, n_calls=args.calls, x0=x0, verbose=True, n_jobs=args.workers, n_random_starts=0 if x0 else 1, acq_func='EI') elif args.optimizer == "random": res = skopt.dummy_minimize(func=objective, dimensions=space, n_calls=args.calls, x0=x0, verbose=True) elif args.optimizer == "grid": space = [ np.arange(norm_min, norm_max + 1, 10), np.arange(scale_min, scale_max + 1, 10) ] res = grid(func=objective, dimensions=space) print(f"Minimal cost of {min(stats['cost'])} found at:") if args.layer: print( tabulate.tabulate( { 'Layer': np.arange(len(factors)), 'F-Norm:': f_norms, 'Norms': stats['norms'][np.argmin(stats['cost'])], 'Scales': stats['scales'][np.argmin(stats['cost'])] }, headers='keys', numalign='right')) else: print("Norm:", stats['norms'][np.argmin(stats['cost'])][0], "Scale:", stats['scales'][np.argmin(stats['cost'])][0]) if not args.no_results: print("Saving results") del res.specs['args']['func'] np.save( results_path + f"_best_params{'_layer.npy' if args.layer else '.npy'}", [ stats['norms'][np.argmin(stats['cost'])], stats['scales'][np.argmin(stats['cost'])] ]) np.save( results_path + f"_hyperopt_stats{'_layer.npy' if args.layer else '.npy'}", stats) skopt.dump( res, results_path + f"_hyperopt_dump{'_layer.pkl' if args.layer else '.pkl'}") if args.plot: print("Plotting results") hyperparameters(args)
def run(self, exp_function: list or function, params: dict = None, artifacts: dict = None, param_space=None, max_evals=10) -> list: """ Runs the given experiment function or list of functions with various parameter configuration # Arguments exp_function (function or list): Method that implements the experiment. params (dict): Dictionary that contains the configuration (e.g. hyperparameters) for the experiment (optional). artifacts (dict): Dictionary that contains artifacts (any kind of python object) required for the experiment (optional). param_space (iterator): Parameter Space suited for Skopt Library. max_evals (int): Maximum number of evaluations. Default: 10. # Returns List of finished experiments """ from skopt.utils import use_named_args @use_named_args(param_space) def skopt_objective(**param_selection): exp = self.create_exp( name_suffix=text_utils.simplify_dict_to_str(param_selection)) params.update(param_selection) exp.run_exp(exp_function, params, artifacts) score = exp.exp_metadata.result if not score or not isinstance(score, numbers.Number): exp.log.info( "Provided exp_function did not return a numeric score/result." ) return None if self.maximize_score: # hyperopt only supports min score = -score return score if self.skopt_algo == "gp": from skopt import gp_minimize self.skopt_result = gp_minimize(skopt_objective, param_space, n_calls=max_evals, **self.skopt_args) elif self.skopt_algo == "gbrt": from skopt import gbrt_minimize self.skopt_result = gbrt_minimize(skopt_objective, param_space, n_calls=max_evals, **self.skopt_args) elif self.skopt_algo == "forest": from skopt import forest_minimize self.skopt_result = forest_minimize(skopt_objective, param_space, n_calls=max_evals, **self.skopt_args) elif self.skopt_algo == "dummy": from skopt import dummy_minimize self.skopt_result = dummy_minimize(skopt_objective, param_space, n_calls=max_evals, **self.skopt_args) else: self.log.warning("The selected algorithm is unknown: " + self.skopt_algo) return self.experiments()
def local_lipschitz_estimate(self, x, optim='gp', eps=None, bound_type='box', clip=True, n_calls=100, njobs=-1, verbose=False): """ Compute one-sided lipschitz estimate for explainer. Adequate for local Lipschitz, for global must have the two sided version. This computes: max_z || f(x) - f(z)|| / || x - z|| Instead of: max_z1,z2 || f(z1) - f(z2)|| / || z1 - z2|| If eps provided, does local lipzshitz in: - box of width 2*eps along each dimension if bound_type = 'box' - box of width 2*eps*va, along each dimension if bound_type = 'box_norm' (i.e. normalize so that deviation is eps % in each dim ) - box of width 2*eps*std along each dimension if bound_type = 'box_std' max_z || f(x) - f(z)|| / || x - z|| , with f = theta clip: clip bounds to within (min, max) of dataset """ # Compute bounds for optimization if eps is None: # If want to find global lipzhitz ratio maximizer - search over "all space" - use max min bounds of dataset fold of interest # gp can't have lower bound equal upper bound - so move them slightly appart lwr = self.train_stats['min'].flatten() - 1e-6 upr = self.train_stats['max'].flatten() + 1e-6 elif bound_type == 'box': lwr = (x - eps).flatten() upr = (x + eps).flatten() elif bound_type == 'box_std': # gp can't have lower bound equal upper bound - so set min std to 0.001 lwr = (x - eps * np.maximum(self.train_stats['std'], 0.001)).flatten() upr = (x + eps * np.maximum(self.train_stats['std'], 0.001)).flatten() if clip: lwr = lwr.clip(min=self.train_stats['min'].min()) upr = upr.clip(max=self.train_stats['max'].max()) bounds = list(zip(*[lwr, upr])) if x.ndim > 2: # This is an image, will need to reshape orig_shape = x.shape x = x.flatten() else: orig_shape = x.shape # Run optimization if optim == 'gp': print('Running BlackBox Minimization with Bayesian Optimization') # Need minus because gp only has minimize method f = partial(self.lipschitz_ratio, x, reshape=orig_shape, minus=True) res = gp_minimize(f, bounds, n_calls=n_calls, verbose=verbose, n_jobs=njobs) elif optim == 'gbrt': print('Running BlackBox Minimization with Gradient Boosted Trees') f = partial(self.lipschitz_ratio, x, reshape=orig_shape, minus=True) res = gbrt_minimize(f, bounds, n_calls=n_calls, verbose=verbose, n_jobs=njobs) lip, x_opt = -res['fun'], np.array(res['x']) if verbose: print(lip, np.linalg.norm(x - x_opt)) return lip, x_opt.reshape(orig_shape)
def main(): # Evaluate command line arguments cd = os.path.dirname(os.path.abspath(__file__)) parser = argparse.ArgumentParser( description="Performs point cloud registration.") parser.add_argument("-c", "--config", default=os.path.join(cd, "hyperopt.ini"), type=str, help="/path/to/hyperopt.ini.") parser.add_argument("-v", "--verbose", action="store_true", help="Get verbose output during execution.") parser.add_argument("-d", "--draw", action="store_true", help="Visualize results results.") parser.add_argument("-o", "--output", default=os.path.join(cd, "output"), type=str, help="/path/to/output/dir") args = parser.parse_args() # Make output directory os.makedirs(args.output, exist_ok=True) # Read hyperparameter optimization config (hyper config) from argument hyper_config = configparser.ConfigParser(inline_comment_prefixes='#') hyper_config.read(args.config) # Set frequent config options optimization = hyper_config["optimization"] optimizer = optimization.get("optimizer").lower() options = hyper_config["options"] # Read run registration config from file run_config = configparser.ConfigParser(inline_comment_prefixes='#') run_config.read(os.path.join(cd, "registration.ini")) # Enable verbose output verbose = args.verbose or options.getboolean("verbose") if verbose: logger.setLevel(logging.DEBUG) print_config(hyper_config) # Set run config with values from hyper config set_config_params_with_config_or_dict(config_or_dict_from=hyper_config, config_to=run_config) run_config["options"]["verbose"] = str(False) run_config["options"]["progress"] = str(False) run_config["options"]["print_results"] = str(False) run_config["options"][ "return"] = "results+errors" if "error" in optimization.get( "minimize").lower() else "results" run_config["options"]["use_degrees"] = str(False) configs = list() # Create a Scikit-Optimize search space from the hyper config space = get_skopt_space_from_config(config=hyper_config) # Make a progress bar progress = tqdm.tqdm(range(optimization.getint("n_calls")), desc=f"Optimization ({optimizer.upper()})", file=sys.stdout, disable=not options.getboolean("progress") or verbose) def progress_callback(_result: skopt.utils.OptimizeResult) -> None: progress.set_postfix_str( f"Current minimum ({optimization.get('minimize')}): {np.min(_result.func_vals)}" ) progress.update() # Make a checkpointer def checkpoint_callback(_result: skopt.utils.OptimizeResult) -> None: try: if len(_result.x_iters) % 10 == 1: _res = copy.deepcopy(_result) del _res.specs['args']['callback'] del _res.specs['args']['func'] skopt.dump(res=_res, filename=os.path.join(args.output, "checkpoint.pkl"), compress=True) except Exception as ex: message = f"Couldn't save checkpoint due to exception: {ex}. Skipping." logger.exception(message) if progress.disable else progress.write( message) # The optimization objective function @skopt.utils.use_named_args(dimensions=space) def objective(**params: Dict[str, Any]) -> Union[float, Tuple[float, float]]: start = time.time() if verbose: print( tabulate.tabulate( { "Option": list(params.keys()), "Values": list(params.values()) }, headers="keys", missingval="None")) set_config_params_with_config_or_dict( config_or_dict_from=params, config_to=run_config, sections_to_skip=["DEFAULT", "optimization", "options", "data"]) configs.append(copy.deepcopy(run_config)) try: results = run(config=run_config) if any("error" in key for key in results.keys()): if optimization.get("minimize").lower() in [ "errors", "errors_rot+errors_trans" ]: cost = sum(results["errors_rot"] + results["errors_trans"]) elif optimization.get("minimize").lower() == "errors_rot": cost = sum(results["errors_rot"]) elif optimization.get("minimize").lower() == "errors_trans": cost = sum(results["errors_trans"]) else: raise ValueError( f"Invalid config option '{optimization.get('minimize')}' for 'minimize'." ) else: if optimization.get("minimize").lower() in [ "errors", "errors_rot+errors_trans" ]: message = f"Optimization objective is '{optimization.get('minimize')}' which was not found in" \ f"returned results. Did you provide ground truth data? Using inlier RMSE instead." raise ValueError(message) elif optimization.get( "minimize").lower() == "inlier_rmse-fitness": cost = sum([ r.inlier_rmse - r.fitness for r in results["results"] ]) elif optimization.get("minimize").lower() in [ "fitness", "-fitness" ]: cost = -sum([r.fitness for r in results["results"]]) elif optimization.get("minimize").lower() == "inlier_rmse": cost = sum([r.inlier_rmse for r in results["results"]]) else: raise ValueError( f"Invalid config option '{optimization.get('minimize')}' for 'minimize'." ) if optimization.getboolean("scale_objective"): num_correspondences = sum([ len(r.correspondence_set) for r in results["results"] ]) cost = cost / num_correspondences if num_correspondences > 0 else 1e30 except Exception as ex: message = f"Caught exception during execution of 'run_registration.run': {ex}" logger.exception(message) if progress.disable else progress.write( message) cost = 1e30 if not np.isfinite(cost): cost = 1e30 if optimizer not in [ "rnd", "random" ] and optimization.get("acq_func").lower() in ["eips", "pips"]: return cost, time.time() - start else: return cost # Load initial points init = optimization.get("init_from_file_or_list") x0, y0 = None, None try: init = eval(init) if init is not None and isinstance(init, (list, tuple)): x0 = init y0 = None except (NameError, SyntaxError): try: init = skopt.load(init) x0 = init.x_iters y0 = init.func_vals except FileNotFoundError: logger.exception( f"Couldn't load initial points from {init}. Skipping.") x0 = None y0 = None # Run optimization if optimizer in ["rnd", "rand", "random"]: result = skopt.dummy_minimize( func=objective, dimensions=space, n_calls=optimization.getint("n_calls"), initial_point_generator=optimization.get( "initial_point_generator"), x0=x0, y0=y0, random_state=eval(optimization.get("random_state")), verbose=verbose, callback=[progress_callback, checkpoint_callback]) elif all(c in optimizer for c in list("rf")): result = skopt.forest_minimize( func=objective, dimensions=space, base_estimator=optimization.get("base_estimator").upper(), n_calls=optimization.getint("n_calls"), n_initial_points=optimization.getint("n_initial_points"), acq_func=optimization.get("acq_func"), initial_point_generator=optimization.get( "initial_point_generator"), x0=x0, y0=y0, random_state=eval(optimization.get("random_state")), verbose=verbose, callback=[progress_callback, checkpoint_callback], n_points=optimization.getint("n_points"), xi=optimization.getfloat("xi"), kappa=optimization.getfloat("kappa"), n_jobs=optimization.getint("n_jobs")) elif all(c in optimizer for c in list("gp")): noise = optimization.get("noise").lower() noise = noise if "gauss" in noise else optimization.getfloat("noise") result = skopt.gp_minimize( func=objective, dimensions=space, n_calls=optimization.getint("n_calls"), n_initial_points=optimization.getint("n_initial_points"), acq_func=optimization.get("acq_func"), acq_optimizer=optimization.get("acq_optimizer"), initial_point_generator=optimization.get( "initial_point_generator"), x0=x0, y0=y0, random_state=eval(optimization.get("random_state")), verbose=verbose, callback=[progress_callback, checkpoint_callback], n_points=optimization.getint("n_points"), n_restarts_optimizer=optimization.getint("n_restarts_optimizer"), xi=optimization.getfloat("xi"), kappa=optimization.getfloat("kappa"), noise=noise, n_jobs=optimization.getint("n_jobs")) elif all(c in optimizer for c in list("gbrt")): result = skopt.gbrt_minimize( func=objective, dimensions=space, n_calls=optimization.getint("n_calls"), n_initial_points=optimization.getint("n_initial_points"), acq_func=optimization.get("acq_func"), initial_point_generator=optimization.get( "initial_point_generator"), x0=x0, y0=y0, random_state=eval(optimization.get("random_state")), verbose=verbose, callback=[progress_callback, checkpoint_callback], n_points=optimization.getint("n_points"), xi=optimization.getfloat("xi"), kappa=optimization.getfloat("kappa"), n_jobs=optimization.getint("n_jobs")) else: raise ValueError(f"Invalid optimizer {optimizer}.") progress.close() # Print results names = None dims = None try: res = np.concatenate( [np.expand_dims(result.func_vals, axis=1), result.x_iters], axis=1) iters = np.argsort(res[:, 0]) table_values = res[iters] names = [dim.name for dim in space] dims = [f"X_{i}" for i in range(len(names))] if len(names) > 10 else names table_headers = ["Iter", "Cost"] + dims print() print("OPTIMIZATION RESULTS:\n====================") print( tabulate.tabulate(table_values, headers=table_headers, missingval="None", showindex=list(iters))) if len(names) > 10: print() print("DIMENSIONS <-> NAMES:\n====================") print( tabulate.tabulate({ "Dimension": dims, "Name": names }, headers="keys")) except Exception as e: logger.exception( f"Printing of results failed due to exception: {e}. Trying to save results." ) # Visualize results if (args.draw or options.getboolean("draw")) and names is not None: try: if len(names) > 5: logger.info( f"Drawing visualizations for upt to 10 dimensions. This can take some time." ) from matplotlib import pyplot as plt plot_convergence(result) try: plot_dims = eval(options.get("plot_dims")) if isinstance(plot_dims, float): plot_dims = int(plot_dims) elif isinstance(plot_dims, int) or plot_dims is None: pass else: raise TypeError( f"'plot_dims' must by None or int but is {type(plot_dims)}. Skipping selection." ) except (NameError, SyntaxError, TypeError): plot_dims = None if (plot_dims is None and len(names) > 10) or (isinstance(plot_dims, int) and plot_dims > 10): logger.info( f"Too many dimensions ({len(names)}) for 'evaluations' and 'objective' plots. " f"Only drawing the first 10.") plot_dims = 10 if isinstance(plot_dims, int): plot_dims = names[:plot_dims] else: plot_dims = None plot_evaluations( result, dimensions=dims if plot_dims is None else plot_dims, plot_dims=plot_dims) if optimizer not in ["rnd", "rand", "random"]: plot_objective( result, dimensions=dims if plot_dims is None else plot_dims, plot_dims=plot_dims) else: logger.info( f"Can't plot 'objective' for optimizer '{optimizer}'.") plt.show() except Exception as e: logger.exception( f"Drawing failed due to exception: {e}. Skipping.") # Save best result config filename = None config_hash = None if options.getboolean("save") or names is None: try: filename = options.get("filename") try: filename = eval(filename) except (NameError, SyntaxError): if filename.lower() == "none": filename = None best_config = configs[int(np.argmin(result.func_vals))] best_config["options"]["progress"] = str(True) best_config["options"]["print_results"] = str(True) if filename is None: if options.getboolean("overwrite"): config_hash = str().join([ value for option, value in hyper_config.items("data") ]) else: config_hash = str() for section in hyper_config.sections(): for option, value in hyper_config.items(section): config_hash += value config_hash = hashlib.md5( config_hash.encode('utf-8')).hexdigest() output_path = os.path.join(args.output, f"{config_hash}.ini") else: output_path = os.path.join(args.output, f"{filename.split('.')[0]}.ini") with open(output_path, 'w') as configfile: best_config.write(configfile) except Exception as e: config_hash = hex(random.getrandbits(128)) logger.exception( f"Saving of best result failed due to exception: {e}. Trying to dump all results." ) # Save results if filename is None: output_path = os.path.join(args.output, f"{config_hash}.pkl") else: output_path = os.path.join(args.output, f"{filename.split('.')[0]}.pkl") try: del result.specs['args']['callback'] skopt.dump(res=result, filename=output_path, compress=True) except Exception as e: logger.debug(f"Caught exception during 'skopt.dump': {e}") logger.debug("Trying to store the result without the objective.") skopt.dump(res=result, filename=output_path, store_objective=False, compress=True) finally: logger.debug("Deleting the objective.") del result.specs['args']['func'] skopt.dump(res=result, filename=output_path, compress=True)
def hyperdrive(objective, hyperparameters, results_path, model="GP", n_iterations=50, verbose=False, checkpoints_path=None, deadline=None, sampler=None, n_samples=None, random_state=0): """ Distributed optimization - one optimization per node. Parameters ---------- * `objective` [function]: User defined function which calls a learner and returns a metric of interest. * `hyperparameters` [list, shape=(n_hyperparameters,)]: * `results_path` [string] Path to save optimization results * `checkpoint_path` [string] Path to previously saved results. Used to resume optimization. * `model` [string, default="GP"] Probilistic learner used to model our objective function. Options: - "GP": Gaussian process - "RF": Random forest - "GBRT": Gradient boosted regression trees - "RAND": Random search * `n_iterations` [int, default=50] Number of optimization iterations * `verbose` [bool, default=False] Verbosity of optimization. * `checkpoints` [bool, default=False] Whether to checkpoint at each step of the optimization. * `deadline` [int, optional] Deadline (seconds) for the optimization to finish within. * `sampler` [str, default=None] Random sampling scheme for optimizer's initial runs. Options: - "lhs": latin hypercube sampling * `n_samples` [int, default=None] Number of random samples to be drawn from the `sampler`. - Required if you would like to use `sampler`. - Must be <= the number of elements in the smallest hyperparameter bound's set. * `random_state` [int, default=0] Random state for reproducibility. """ comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if checkpoints_path and sampler: raise ValueError('Cannot use both a restart from a previous run and ' \ 'use latin hypercube sampling for initial search points!') # Setup savefile if rank < 10: # Ensure results are sorted by rank filename = 'hyperspace' + str(0) + str(rank) else: filename = 'hyperspace' + str(rank) savefile = os.path.join(results_path, filename) # Create hyperspaces, and either sampling bounds or checkpoints hyperspace = create_hyperspace(hyperparameters) space = hyperspace[rank] # Latin hypercube sampling if sampler and not n_samples: raise ValueError(f'Sampler requires n_samples > 0. Got {n_samples}') elif sampler and n_samples: hyperbounds = create_hyperbounds(hyperparameters) bounds = hyperbounds[rank] # Get initial points in domain via latin hypercube sampling init_points = lhs_start(bounds, n_samples) init_response = None n_rand = 10 - len(init_points) else: init_points = None init_response = None n_rand = 10 # Resuming from checkpoint if checkpoints_path: checkpoint = _load_checkpoint(checkpoints_path, rank) try: init_points = checkpoint.x_iters init_response = checkpoint.func_vals n_rand = 10 - len(init_points) except AttributeError: # Missing saves won't have initial values. init_points = None init_response = None n_rand = 10 callbacks = [] if deadline: deadline = DeadlineStopper(deadline) callbacks.append(deadline) if checkpoints_path: checkpoint_callback = CheckpointSaver(checkpoints_path, filename) callbacks.append(checkpoint_callback) # Thanks Guido for refusing to believe in switch statements. # Case 0 if model == "GP": # Verbose mode should only run on node 0. if verbose and rank == 0: result = gp_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=callbacks, x0=init_points, y0=init_response, n_random_starts=n_rand, random_state=random_state) else: result = gp_minimize(objective, space, n_calls=n_iterations, callback=callbacks, x0=init_points, y0=init_response, n_random_starts=n_rand, random_state=random_state) # Case 1 elif model == "RF": if verbose and rank == 0: result = forest_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=callbacks, x0=init_points, y0=init_response, n_random_starts=n_rand, random_state=random_state) else: result = forest_minimize(objective, space, n_calls=n_iterations, callback=callbacks, x0=init_points, y0=init_response, n_random_starts=n_rand, random_state=random_state) # Case 2 elif model == "GBRT": if verbose and rank == 0: result = gbrt_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=callbacks, x0=init_points, y0=init_response, n_random_starts=n_rand, random_state=random_state) else: result = gbrt_minimize(objective, space, n_calls=n_iterations, callback=callbacks, x0=init_points, y0=init_response, n_random_starts=n_rand, random_state=random_state) # Case 3 elif model == "RAND": if verbose and rank == 0: result = dummy_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=callbacks, x0=init_points, y0=init_response, random_state=random_state) else: result = dummy_minimize(objective, space, n_calls=n_iterations, callback=callbacks, x0=init_points, y0=init_response, random_state=random_state) else: raise ValueError("Invalid model {}. Read the documentation for " "supported models.".format(model)) # Each worker will independently write their results to disk dump(result, savefile)
print('\n') lgb_mod = lgb.LGBMClassifier(**params) best_mod = lgb_mod.fit(X_train,y_train,**fit_params) y_pred = best_mod.predict(X_test) print('y_pred.value_counts() -> \n{}'.format(np.unique(y_pred,return_counts=True))) loss = lgbm_custom_loss(y_test,y_pred) if loss[2] == True:#higher is better loss = 1 - loss[1] else: loss = loss[1] print('loss -> {}'.format(loss)) return loss HPO_PARAMS = { 'n_calls':100 } results = skopt.gbrt_minimize(train_func,CONFIG['lgbm_training']['skopt_params'],**HPO_PARAMS) print('\nBEST RESULT - {} -> {:.4f}'.format(CONFIG['lgbm_training']['custom_metric'],results.fun)) best_params = {k.name:v for k, v in zip(CONFIG['lgbm_training']['skopt_params'],results.x)} print('\nBEST PARAMS -> {}'.format(best_params)) run_time.end() ### FIT THE FINAL MODEL USIN BEST PARAMS ### final_models = lgb.LGBMClassifier(**best_params) final_models.fit(X_train,y_train) #Show the best and worst features lgb.plot_importance(final_models,figsize=(12,36)) #Run on the testing set y_pred = final_models.predict(X_test) print(calc_tpr(y_pred,y_test,['FINAL RF MODEL - TEST SET'],y_train.unique()))
import matplotlib.pyplot as plt from skopt import gbrt_minimize from skopt.benchmarks import bench3 from skopt.acquisition import gaussian_ei dimensions = [(-1.0, 1.0)] x = np.linspace(-1, 1, 200) func_values = [bench3(xi) for xi in x] plt.figure(figsize=(10, 5)) vals = np.reshape(x, (-1, 1)) col_no = 1 row_no = 6 res = gbrt_minimize( bench3, dimensions, maxiter=6, n_start=1, random_state=1) best_xs = res.x_iters.ravel() best_ys = res.func_vals.ravel() models = res.models for n_iter in range(5): model = models[n_iter] best_x = best_xs[:n_iter+1] best_y = best_ys[:n_iter+1] low, mu, high = model.predict(vals).T std = (high - low) / 2 acquis_values = -gaussian_ei(vals, model, best_y[-1]) acquis_values = acquis_values.ravel() posterior_mean = mu.ravel() posterior_std = std.ravel()
def hyperdrive(objective, hyperparameters, results_path, model="GP", n_iterations=50, verbose=False, deadline=None, sampler=None, n_samples=None, random_state=0): """ Distributed optimization - one optimization per node. Parameters ---------- * `objective` [function]: User defined function which calls a learner and returns a metric of interest. * `hyperparameters` [list, shape=(n_hyperparameters,)]: * `results_path` [string] Path to save optimization results * `model` [string, default="GP"] Probilistic learner used to model our objective function. Options: - "GP": Gaussian process - "RF": Random forest - "GBRT": Gradient boosted regression trees - "RAND": Random search * `n_iterations` [int, default=50] Number of optimization iterations * `verbose` [bool, default=False] Verbosity of optimization. * `deadline` [int, optional] Deadline (seconds) for the optimization to finish within. * `sampler` [str, default=None] Random sampling scheme for optimizer's initial runs. Options: - "lhs": latin hypercube sampling * `n_samples` [int, default=None] Number of random samples to be drawn from the `sampler`. - Required if you would like to use `sampler`. - Must be <= the number of elements in the smallest hyperparameter bound's set. * `random_state` [int, default=0] Random state for reproducibility. """ comm = MPI.COMM_WORLD rank = comm.Get_rank() size = comm.Get_size() if rank == 0: hyperspace = create_hyperspace(hyperparameters) if sampler and not n_samples: raise ValueError( 'Sampler requires n_samples > 0. Got {}'.format(n_samples)) elif sampler and n_samples: hyperbounds = create_hyperbounds(hyperparameters) else: hyperspace = None if sampler is not None: hyperbounds = None space = comm.scatter(hyperspace, root=0) if sampler: bounds = comm.scatter(hyperbounds, root=0) # Get initial points in the obj. function domain via latin hypercube sampling init_points = lhs_start(bounds, n_samples) n_rand = 10 - len(init_points) else: init_points = None n_rand = 10 if deadline: deadline = DeadlineStopper(deadline) # Thanks Guido for refusing to believe in switch statements. # Case 0 if model == "GP": # Verbose mode should only run on node 0. if verbose and rank == 0: result = gp_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) else: result = gp_minimize(objective, space, n_calls=n_iterations, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) # Case 1 elif model == "RF": if verbose and rank == 0: result = forest_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) else: result = forest_minimize(objective, space, n_calls=n_iterations, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) # Case 2 elif model == "GRBRT": if verbose and rank == 0: result = gbrt_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) else: result = gbrt_minimize(objective, space, n_calls=n_iterations, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) # Case 3 elif model == "RAND": if verbose and rank == 0: result = dummy_minimize(objective, space, n_calls=n_iterations, verbose=verbose, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) else: result = dummy_minimize(objective, space, n_calls=n_iterations, callback=deadline, x0=init_points, n_random_starts=n_rand, random_state=random_state) else: raise ValueError("Invalid model {}. Read the documentation for " "supported models.".format(model)) # Each worker will independently write their results to disk dump(result, results_path + '/hyperspace' + str(rank))