def fit(self, X: list, y: list, n_jobs: int, max_iter: int = 50) -> None: """ Perform bayesian optimization on hyperparameter space. Args: x : Array containing training input data. y : Array containing data labels. n_jobs: Number of threads to run the algorithms. max_iter : Number of iterations to perform bayesian optimization. """ self.X, self.y = shuffle(X, y) self.n_jobs = n_jobs num_features = X.shape[1] for model_name in self.available_models: self.current_model = model_name print(self.current_model) opt = BayesianOptimization( f=self.perform_cv, domain=self.grids[self.current_model]) opt.run_optimization(max_iter=max_iter)
def fit(self, X: list, y: list, metric: str = None, max_iter: int = 50, silence=False) -> None: """ Perform bayesian optimization on hyperparameter space. Args: x : Array containing training input data. y : Array containing data labels. max_iter : Number of iterations to perform bayesian optimization. """ if metric not in self.metrics: msg = "Unsupported metric '{}'".format(metric) msg += "\nSupported metrics are: {}".format(self.metrics) raise ValueError(msg) self.X = X self.y = y self.core_metric = metric opt = BayesianOptimization(f=self.run_iteration, domain=self.grid) opt.run_optimization(max_iter=max_iter)
def run(_run=0): true_theta = load_true_theta(_run) true_y = load_true_y(_run) theta_priors = { 'a': LogNormal(-11.86, 0.1), 'b': Normal(1.2e-4, 1e-6), 'k': LogNormal(-0.35, 0.01) } def f_of_(X, device=GLOBAL_DEVICE): a = theta_priors['a'].icdf(tp.tensor(X[0,0]).to(device)) b = theta_priors['b'].icdf(tp.tensor(X[0,1]).to(device)) k = theta_priors['k'].icdf(tp.tensor(X[0,2]).to(device)) sample = simulate({'a':a, 'b':b, 'k':k}, device=device) return loss(true_y, sample).cpu().item() domain = [ {'name': 'a', 'type':'continuous','domain':(0.0001, 0.9999)}, {'name': 'b', 'type':'continuous','domain':(0.0001, 0.9999)}, {'name': 'k', 'type':'continuous','domain':(0.0001, 0.9999)} ] experiment = BayesianOptimization(f = lambda x: f_of_(x), domain=domain) experiment.run_optimization(max_iter=35) return {'y':true_y, 'theta':true_theta}, experiment
def runOracle(self, budget): self.data, self.result = self.initialise() self.result_list = [] # Initialize wts and probs print("Running ", self.policy, " with budget ", budget) for i in tqdm(range(self.C)): self.cat = i print("Processing arm: ", self.cat) myBopt = BayesianOptimization(f=self.my_func, domain=self.bounds) myBopt.run_optimization(max_iter=budget) self.result_list.append(myBopt.Y_best) self.opt_vals = [self.result_list[ii][-1] for ii in range(self.C)] self.opt_ht = np.argmin(self.opt_vals) maxlen = 0 for i in range(self.C): if len(self.result_list[i]) > maxlen: maxlen = len(self.result_list[i]) vals = np.zeros((maxlen, self.C)) for i in range(self.C): l = len(self.result_list[i]) vals[:, i][:l] = self.result_list[i][:l] # for i in range(self.C): # vals[:,i] = self.result_list[i] bv = self.result_list[self.opt_ht] * -1 df = pd.DataFrame(data=vals, columns=np.arange(self.C)) # print("Size of best vals in loop: ", bv.shape) return df, bv
def run_baeopt(opt_params): # set up opt params bounds = get_bounds(sm.run.ctl_str_ids, sm.run.n_ctl_steps) max_iter = opt_params.get('max_iter', 15) max_time = opt_params.get('max_time', 120) initial_guess = opt_params.get('initial_guess', []) if len(initial_guess) == 0: initial_guess = None else: initial_guess = np.array([initial_guess]) eps = opt_params.get('eps', 0.01) model_type = opt_params.get('model_type', 'GP') acquisition_type = opt_params.get('acquisition_type', 'EI') # instantiate object bae_opt = BayOpt( ev.evaluate, domain=bounds, model_type=model_type, acquisition_type='EI', X=initial_guess, evaluator_type='local_penalization', num_cores=opt_params['num_cores'], batch_size=opt_params['num_cores'], ) bae_opt.run_optimization(max_iter, max_time, eps) return bae_opt.x_opt, bae_opt.fx_opt
def optimize_local(output=output): # if self.allocationMethod == 'variablePrecision': # importing CMAES libarary import cma # setting up parameters and running the optimization x0 = 50 + 15 * np.random.randn(12 * (self.depth - 1)) res = cma.fmin(obj_func, x0, 30, options={'bounds':[1,100],\ 'tolfun':1, 'maxfevals': int(1e4)}) sigma_opt = res[0] elif self.allocationMethod == 'equalPrecision': # importing Bayesian optimization libraries import GPy, GPyOpt from GPyOpt.methods import BayesianOptimization # setting up parameters and running the optimization kernel = GPy.kern.Matern52(input_dim=1) domain = [{ 'name': 'var_1', 'type': 'continuous', 'domain': (0, 100) }] optimizer = BayesianOptimization(obj_func, domain=domain, kernel=kernel) optimizer.run_optimization(max_iter=50) sigma_opt = optimizer.X[optimizer.Y.argmin()] # appending the result (scalar sigma) to output queue output.put(sigma_opt)
def Bayesdefinition(self): self.opt_search = BayesianOptimization(self.func.objective_function, domain=self.domain, model_type='GP', acquisition_type='EI', num_cores=-1, verbosity=False)
def test_save_gp_2d(self): k = GPy.kern.Matern52(input_dim=2) m = GPModel(kernel=k) myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d, model=m) myBopt.run_optimization(max_iter=1, verbosity=False) myBopt.save_models(self.outfile_path) self.check_output_model_file(['Iteration'])
def text_cnn_train(args, train_path): # load data print("\nLoading data...") text_field = data.Field(lower=True) label_field = data.Field(sequential=False) #train_iter, dev_iter = mr(text_field, label_field, device=-1, repeat=False) train_iter, dev_iter = msw_text(text_field, label_field, train_path, device=-1, repeat=False) # batch = next(iter(train_iter)) # print(type(batch)) # print(batch.text) # train_iter, dev_iter, test_iter = sst(text_field, label_field, device=-1, repeat=False) # update args and print args.embed_num = len( text_field.vocab) # .vocab을 해주면 단어 집합을 만들어 주는거 같다. 일단 추정 args.class_num = len(label_field.vocab) - 1 args.cuda = (not False) and torch.cuda.is_available() kerns = '3,4,5' args.kernel_sizes = [int(k) for k in kerns.split(',')] re_train_path = train_path.split('/')[1][:-4] save_path = os.path.join(args.save_dir, re_train_path) print("\nParameters:") for attr, value in sorted(args.__dict__.items()): print("\t{}={}".format(attr.upper(), value)) # train or predict if args.baye: print('옵티마이즈 세팅') myBopt = BayesianOptimization(f=baye, domain=domain, initial_design_numdata=5) print('옵티마이즈 시작') myBopt.run_optimization(max_iter=10) print('옵티마이즈 결과 : ', myBopt.x_opt) print('최적의 하이퍼 파라미터의 결과의 파라미터', args.lr, args.dropout) else: try: cnn = model.CNN_Text(args) if args.snapshot is not None: print('\nLoading model from {}...'.format(args.snapshot)) cnn.load_state_dict(torch.load(args.snapshot)) if args.cuda: torch.cuda.set_device(args.device) cnn = cnn.cuda() train.train(train_iter, dev_iter, cnn, args, save_path) except KeyboardInterrupt: print('\n' + '-' * 89) print('Exiting from training early')
def break_grid_bo(w_, garden_index, grids): # first compute convex hull of grids # parametrize problem by just the center of the grid num_poles = grids.shape[0] bounds = [] for i in range(num_poles): name_s = "p" + str(i) lower_x = grids[i, 0] - 0.8 upper_x = grids[i, 0] + 0.8 lower_y = grids[i, 1] - 0.8 upper_y = grids[i, 1] + 0.8 domainx = (lower_x, upper_x) domainy = (lower_y, upper_y) dx = {'name': name_s + "x", 'type': 'continuous', 'domain': domainx} dy = {'name': name_s + "y", 'type': 'continuous', 'domain': domainy} bounds.append(dx) bounds.append(dy) Y_init, pole_platform, cc, _ = all_constraints(w_, garden_index, grids) loss = 1000 current_iter = 0 X_step = np.zeros((1, num_poles * 2)) Y_step = np.empty((1, 1)) Y_step[0, 0] = Y_init context = {} while loss > 2: if current_iter >= 200: return grids bo_step = BayesianOptimization(f=None, model="GP", domain=bounds, X=X_step, Y=Y_step, maximize=False, acquisition="LCB") try: x_next = bo_step.suggest_next_locations(context=context) except: x_next = bo_step.suggest_next_locations() grids = x_next.reshape(num_poles, 2) loss, pole_platform, cc, good_ind = all_constraints( w_, garden_index, grids) for i in good_ind: context["p{}x".format(i)] = grids[i, 0] context["p{}y".format(i)] = grids[i, 1] print("Iteration {} has loss {}".format(current_iter, loss)) print("POLE2PLAT {}".format(pole_platform)) y_next = loss y_tmp = np.empty((1, 1)) y_tmp[0, 0] = y_next X_step = np.vstack((X_step, x_next)) Y_step = np.vstack((Y_step, y_tmp)) loss = y_next current_iter += 1 return grids
def train( self, bo_model=None, bo_space=None, bo_acquisition=None, X_init=None, Y_init=None, BO_init_num=5, BO_max_num=20, verbosity=False): """ Uses Bayesian optimisation to find the optimal design. The objective function is the mutual information lower bound at a particular design, obtained by training a MINE model. Parameters ---------- bo_model: bo_space: bo_acquisition: BO_init_num: int The number of initial BO evaluations used to initialise the GP. (default is 5) BO_max_num: int The maximum number of BO evaluations after the initialisation. (default is 20) verbosity: boolean Turn off/on output to the command line. (default is False) """ if verbosity: print('Initialize Probabilistic Model') if bo_model and bo_space and bo_acquisition: raise NotImplementedError('Custom BO model not yet implemented.') elif all(v is None for v in [bo_model, bo_space, bo_acquisition]): pass else: raise ValueError( 'Either all BO arguments or none need to be specified.') # Define GPyOpt Bayesian Optimization object self.bo_obj = BayesianOptimization( f=self._objective, domain=self.domain, constraints=self.constraints, model_type='GP', acquisition_type='EI', normalize_Y=False, initial_design_numdata=BO_init_num, acquisition_jitter=0.01, maximize=True, X=X_init, Y=Y_init) # TODO: Implement a more modular approach with GPy model as input. if verbosity: print('Start Bayesian Optimisation') # run the bayesian optimisation self.bo_obj.run_optimization( max_iter=BO_max_num, verbosity=verbosity, eps=1e-5) # find optimal design from posterior GP model; stored as d_opt self._compute_optimal_design(self.bo_obj)
def forward(self, output, target, bayes=True, best_params=None): self.output = output self.target = target self.bayes = bayes # vest params is None when it is first train. if best_params is None: var_args = [ 0.5, 0.5, 0.5, 0.5, # threshold 0.0, 0.0, 0.0, 0.0, # minsizes 0.2, ] else: var_args = [ best_params['threshold_0'], best_params['threshold_1'], best_params['threshold_2'], best_params['threshold_3'], best_params['min_size_0'], best_params['min_size_1'], best_params['min_size_2'], best_params['min_size_3'], best_params['mask_threshold'], ] if bayes is True: pbounds = [ {'name': 'threshold_0', 'type': 'continuous', 'domain': (0, 1.0)}, {'name': 'threshold_1', 'type': 'continuous', 'domain': (0, 1.0)}, {'name': 'threshold_2', 'type': 'continuous', 'domain': (0, 1.0)}, {'name': 'threshold_3', 'type': 'continuous', 'domain': (0, 1.0)}, {'name': 'min_size_0', 'type': 'continuous', 'domain': (100, 10000)}, {'name': 'min_size_1', 'type': 'continuous', 'domain': (100, 10000)}, {'name': 'min_size_2', 'type': 'continuous', 'domain': (100, 10000)}, {'name': 'min_size_3', 'type': 'continuous', 'domain': (100, 10000)}, {'name': 'mask_threshold', 'type': 'continuous', 'domain': (0, 1.0)}, ] optimizer = BayesianOptimization( f=self.calc_loss, domain=pbounds, num_cores=16, maximize=True, verbosity=False, initial_design_numdata=2 ) optimizer.run_optimization(max_iter=18, verbosity=False) dice_coefficient = -1 * (optimizer.fx_opt) best_params = optimizer.x_opt else: dice_coefficient = self.calc_loss(var_args) return dice_coefficient, best_params
def test_one_initial_data_point(self): """Make sure BO still works with only one initial data point.""" bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (-1, 1)}] # Remove hyperparameter optimization, not needed here model = GPyOpt.models.GPModel(max_iters=0) opt = BayesianOptimization(lambda x: x, bounds, model=model, initial_design_numdata=1) # Make sure run_optimization works opt.run_optimization(max_iter=1) assert len(opt.Y) > 1
def run_lf_opt(self): def f_lf(X): return(structure(self.lf_model, self.n, X[0], self.EI_req).score) self.lf_opt = BayesianOptimization(f_lf, domain=self.domain, acquisition_type="EI", model_type='GP', exact_feval=True) self.lf_opt.run_optimization(max_iter = self.lf_max_iter, eps=1e-6) self.X_L, self.y_L = self.lf_opt.get_evaluations()
def test_save_gp_2d_ard(self): """ This was previously an edge-case, when some parameters were vectors, the naming of the columns was incorrect """ k = GPy.kern.Matern52(input_dim=2, ARD=True) m = GPModel(kernel=k) myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d, model=m) myBopt.run_optimization(max_iter=1, verbosity=False) myBopt.save_models(self.outfile_path) self.check_output_model_file(['Iteration'])
def test_one_initial_data_point(self): """Make sure BO still works with only one initial data point.""" bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (-1, 1)}] opt = BayesianOptimization(lambda x: x, bounds, initial_design_numdata=1) # Make sure run_optimization works opt.run_optimization(max_iter=1) assert len(opt.Y) > 1
def bo_voronoi_directed(exp): if not exp.initialised: exp.setParameters() G = getVoronoiDirectedGraph(occupancy_grid=exp.occupancy_grid, nodes=exp.nodes, edges=exp.edges_dir, start_nodes=exp.start_nodes, end_nodes=exp.end_nodes) simulateObj = Simulator(G, exp.start_nodes, exp.end_nodes) np.random.RandomState(42) kern_eq = GPy.kern.RBF(input_dim=2, ARD=True) + GPy.kern.White(input_dim=2, variance=1) kern_bias = GPy.kern.Bias(input_dim=2) kern = kern_eq + kern_bias domain = [] k = 0 assigned = {} for n in G.nodes: for neighbor in G.neighbors(n): if n != neighbor and frozenset( (n, neighbor)) not in assigned.keys(): unused_percentage = { 'name': k, 'type': 'continuous', 'domain': (0, 1) } direction_percentage = { 'name': k + 1, 'type': 'continuous', 'domain': (0, 1) } domain.append(unused_percentage) domain.append(direction_percentage) assigned[frozenset( (n, neighbor))] = 1 #tbc whether this will cause errors k += 2 domain.append({'name': k + 1, 'type': 'continuous', 'domain': (0, 0.05)}) opt = BayesianOptimization(f = simulateObj.run_simulator, maximize=False, \ domain=domain, model_type='GP', \ initial_design_numdata = constant.BO_INITIAL_SAMPLES,\ kernel=kern, acquisition_type='EI') out_name = "bo-results.txt" opt.run_optimization(max_iter=constant.BO_OPT_SAMPLES, report_file=out_name) return opt, G
def bayesian_opt(): optimizer = BayesianOptimization(f=best_fitness_score, domain=bounds(), model_type='GP', acquisition_type ='EI', acquisition_jitter = 0.05, exact_feval=True, maximize=False) # Only 20 iterations because we have 5 initial random points optimizer.run_optimization(max_iter=20) print(np.maximum.accumulate(-optimizer.Y).ravel())
def gpyopt(parallel, config_path, conversations_list, params): from GPyOpt.methods import BayesianOptimization global results results = [] utility = make_utility_function_gpyopt(parallel, config_path, conversations_list) bo = BayesianOptimization(utility, params, verbosity=True, initial_design_numdata=len(params) * 2) bo.run_optimization(max_iter=200, verbosity=True) print(f"done. best: {bo.fx_opt}") return results
def main_with_random_search(): bounds = [ {'name': 'batch_size', 'type': 'discrete', 'domain': (16, 32, 64)}, {'name': 'fc1_size', 'type': 'discrete', 'domain': (512, 1024, 2048, 4096)}, {'name': 'fc2_size', 'type': 'discrete', 'domain': (128, 256, 512, 1024)}] optimizer = BayesianOptimization(f=function_to_optimize, domain=bounds) optimizer.run_optimization(max_iter=10) logger.info('optimized parameters: {}'.format(optimizer.x_opt)) logger.info('optimized accuracy: {}'.format(optimizer.fx_opt))
def propose_hypers(bayes_opt_results, bayes_opt_setup): """Proposes hyperparameters given the current results. Args: results: instance of BayesOptResults. bayes_opt_setup: instance of BayesOptSetup Returns: x: the proposed point. """ bayes_opt = BayesianOptimization(f=None, domain=bayes_opt_setup.domain(), X=bayes_opt_results.xs, Y=bayes_opt_results.ys) proposed_x = bayes_opt.suggest_next_locations() return proposed_x, bayes_opt
def test_next_locations_pending(self): func = GPyOpt.objective_examples.experiments1d.forrester() domain =[{'name': 'var1', 'type': 'continuous', 'domain': (0,1)}] X_init = np.array([[0.0],[0.5],[1.0]]) Y_init = func.f(X_init) np.random.seed(1) bo_no_pending = BayesianOptimization(f = None, domain = domain, X = X_init, Y = Y_init) x_no_pending = bo_no_pending.suggest_next_locations() np.random.seed(1) bo_pending = BayesianOptimization(f = None, domain = domain, X = X_init, Y = Y_init, de_duplication = True) x_pending = bo_pending.suggest_next_locations(pending_X = x_no_pending) self.assertFalse(np.isclose(x_pending, x_no_pending))
def _optimize(self, f, variables, X_init, Y_init, maxiter, maxeval, iter_callback): from GPyOpt.methods import BayesianOptimization from GPyOpt.core.task.objective import SingleObjective maxeval = get_maxeval_for_bo(maxeval, maxiter) ndim = len(variables) if len(Y_init) > 0: x_best = X_init[0] y_best = Y_init[0] iter_callback(x_best, y_best, X_init, Y_init) kernel = self.get_kernel(ndim) gpy_domain = self.get_domain(variables) Y_init = np.array(Y_init).reshape(len(Y_init), -1) X_init = np.array(X_init, dtype=float) bo = BayesianOptimization(f=f, domain=gpy_domain, model_type='GP', acquisition_type=self.acquisition_type, kernel=kernel, X=np.array(X_init), Y=np.array(Y_init), exact_feval=True, verbosity=True, ) bo.num_acquisitions = self.run_info.result.n_eval self.run_info.bo_obj = bo def f_in_gpyopt(X): Y = [] x_best = self.transform(self.run_info.result.x) y_best = self.sign * self.run_info.result.y for x in X: y = f(x) if y_best is None or y < y_best: x_best = x y_best = y Y.append(y) iter_callback(x=x_best, y=y_best, X_iter=X, Y_iter=Y) return np.array(Y).reshape(len(Y), -1) bo.f = bo._sign(f_in_gpyopt) bo.objective = SingleObjective( bo.f, bo.batch_size, bo.objective_name) bo.run_optimization(max_iter=maxeval-len(X_init), eps=0, verbosity=False) result = OptimizerResult.from_GPyOpt_OptimizerResult(bo) self.run_info.result.success = True self.run_info.status = result.status self.run_info.message = result.message return self.run_info.result
def optimize(): """ the parameters are: pontrol_gain min_periods_for_inference min_move SE0KLengthScale SE0KSignalVariance PKLengthScale PKPeriodLength PKSignalVariance SE1KLengthScale SE1KSignalVariance min_periods_for_period_estimation points_for_approximation prediction_gain """ # the bounds are used to select which parameters to optimize, # and in which range to optimize. If the range is zero (e.g., (1.0,1.0)) # the parameter isn't optimized at all. bounds = [(0.7, 0.7), (2.0, 2.0), (0.2, 0.2), (700, 700), (18, 18), (10, 10), (200, 200), (20, 20), (25, 25), (10, 10), (2, 2), (100, 100), (0.5, 0.5)] bounds = np.log(bounds) # evaluate the current parameter set opt_params = [0.7, 2, 0.2, 700, 18, 10, 200, 20, 25, 10, 2, 100, 0.5] print eval_func([np.log(opt_params)]) # the initial_design_numdata defines how many random values are tested. # apparently this is the most important value and should be as high as affordable. # num_cores and batch_size define how many evaluations run simultaneously. myBopt = BayesianOptimization(f=eval_func, bounds=bounds, num_cores=6, batch_size=6, maximize=True, initial_design_numdata=24) myBopt.run_optimization(verbosity=True, maximize=True) print -myBopt.fx_opt opt_params = np.exp(myBopt.x_opt) print "{:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f} {:.2f}".format( opt_params[0], opt_params[1], opt_params[2], opt_params[3], opt_params[4], opt_params[5], opt_params[6], opt_params[7], opt_params[8], opt_params[9], opt_params[10], opt_params[11], opt_params[12])
def maximize_varmax_given_xi(xi, GP_model, PPBO_settings): bounds = [{ 'name': 'var_' + str(d), 'type': 'continuous', 'domain': (0, 1) } for d in range(1, GP_model.D + 1)] BO = BayesianOptimization( lambda x: -varmax(xi, x, GP_model, PPBO_settings.mc_samples), domain=bounds, optimize_restarts=0, normalize_Y=True) BO.run_optimization(max_iter=PPBO_settings.BO_maxiter) x_next = BO.x_opt zero_coords = list(np.where(xi != 0)[0]) x_next[zero_coords] = 0 return x_next
def estimate_opt(x, y, my_pwlf): # define the lower and upper bound for the number of line segments bounds = [{ 'name': 'var_1', 'type': 'discrete', 'domain': np.arange(2, 40) }] def my_obj(x): # define some penalty parameter l # you'll have to arbitrarily pick this # it depends upon the noise in your data, # and the value of your sum of square of residuals l = y.mean() * 0.001 f = np.zeros(x.shape[0]) for i, j in enumerate(x): my_pwlf.fit(j[0]) f[i] = my_pwlf.ssr + (l * j[0]) return f np.random.seed(12121) myBopt = BayesianOptimization(my_obj, domain=bounds, model_type='GP', initial_design_numdata=10, initial_design_type='latin', exact_feval=True, verbosity=True, verbosity_model=False) max_iter = 30 # perform the bayesian optimization to find the optimum number # of line segments myBopt.run_optimization(max_iter=max_iter, verbosity=True) print('\n \n Opt found \n') print('Optimum number of line segments:', myBopt.x_opt) print('Function value:', myBopt.fx_opt) myBopt.plot_acquisition() myBopt.plot_convergence() # perform the bayesian optimization to find the optimum number # of line segments myBopt.run_optimization(max_iter=max_iter, verbosity=True) return myBopt.x_opt
def optimisation(self, init_num=5, max_iter=10): # Define GPyOpt Bayesian Optimization object myBopt = BayesianOptimization(f=self._objective, domain=self.domain, constraints=self.constraints, acquisition_type='EI', normalize_Y=True, initial_design_numdata=init_num, evaluator_type='local_penalization', batch_size=int(self.num_cores), num_cores=int(self.num_cores), acquisition_jitter=0.01) # run the bayesian optimisation myBopt.run_optimization(max_iter=max_iter) self.bo_obj = myBopt # Select method to get optimum #optmethod='point' # take optimum from BO evaluations optmethod = 'interpol' # use posterior predictive to find optimum if optmethod == 'point': d_opt = self.bo_obj.x_opt elif optmethod == 'interpol': d_opt = methods.get_GP_optimum(self.bo_obj) else: raise NotImplementedError() # Take some real-world data at optimum y_obs = self.simobj.observe(d_opt)[0] if self.utiltype == 'MI': # Compute ratios r_obs and coefficients b_obs for final observation r_obs, b_obs = self.utilobj.compute_final(d_opt, y_obs, num_cores=self.num_cores) self.savedata = { 'd_opt': d_opt, 'y_obs': y_obs, 'r_obs': r_obs, 'b_obs': b_obs } else: raise NotImplementedError()
def train_bayesian_opt(eval=True): X_train, y_train, X_val, y_val = loadData() knn_params = [{ 'name': 'n_neighbors', 'type': 'discrete', 'domain': (3, 5, 7, 9, 11, 13) }, { 'name': 'p', 'type': 'discrete', 'domain': (1, 2, 3) }] # Optimization objective def cv_score(parameters): parameters = parameters[0] score = cross_val_score(KNeighborsClassifier(n_neighbors=int( parameters[0]), weights='uniform', algorithm='brute', p=int(parameters[1])), X_train, y_train, scoring='accuracy').mean() score = np.array(score) return score optimizer = BayesianOptimization(f=cv_score, domain=knn_params, model_type='GP', acquisition_type='EI', acquisition_jitter=0.001, maximize=True) optimizer.run_optimization(max_iter=20) logger.info(str(optimizer.Y)) logger.info(str(optimizer.X)) logger.info(str(optimizer.X[-1])) if eval == True: train(params={ 'n_neighbors': int(optimizer.X[-1][0]), 'p': int(optimizer.X[-1][1]) })
def optimise_sampler(num_samples, max_tuning_runs, hyper): # tune hyper print("TUNING HYPER-PARAMETERS for hyper=", hyper) if (hyper.n_parameters() > 0): # myBopt = BayesianOptimization(f=hyper, domain=hyper.bounds(), num_cores=os.environ['OMP_NUM_THREADS']) myBopt = BayesianOptimization(f=hyper, domain=hyper.bounds()) myBopt.run_optimization(max_iter=max_tuning_runs) x_opt = myBopt.x_opt else: x_opt = [] # take samples print("TAKING SAMPLES") p = multiprocessing.Pool(int(os.environ['OMP_NUM_THREADS'])) args = zip(range(num_samples), repeat(hyper), repeat(x_opt)) results = p.starmap(optimise, args) return np.array(results)
def setUp(self): np.random.seed(123) domain = [{'name': 'var1', 'type': 'continuous', 'domain': (-5, 5), 'dimensionality': 5}] space = Design_space(domain) func = alpine1(input_dim=5, bounds=space.get_bounds()) bo = BayesianOptimization(f=func.f, domain=domain) context = {'var1_1': 0.3, 'var1_2': 0.4} context_manager = ContextManager(space, context) x0 = np.array([[0, 0, 0, 0, 0]]) # initialize the model in a least intrusive way possible bo.suggest_next_locations() f = bo.acquisition.acquisition_function f_df = bo.acquisition.acquisition_function_withGradients self.problem_with_context = OptimizationWithContext(x0=x0, f=f, df=None, f_df=f_df, context_manager=context_manager) self.x = np.array([[3, -3, 3]])
def bayesian_optimization(objective): bds = [ { 'name': 'alpha', 'type': 'discrete', 'domain': np.arange(0.05, 0.4, 0.05) }, { 'name': 'epsilon', 'type': 'discrete', 'domain': np.arange(0.05, 0.4, 0.05) } # {'name': 'gamma', 'type': 'discrete', 'domain': np.arange(0.5, 1, 0.05)} ] # define el optimizador optimizer = BayesianOptimization(f=objective, domain=bds, model_type='GP', acquisition_type='EI', acquisition_jitter=0.05, verbosity=True, maximize=True) # realiza las 20 iteraciones de la optimizacion optimizer.run_optimization(max_iter=30) print(optimizer.X) print(optimizer.Y) fig = plt.figure() ax = fig.add_subplot(111, projection='3d') xx = optimizer.X[:, 0].reshape(len(optimizer.X[:, 0]), 1).reshape(-1) yy = optimizer.X[:, 1].reshape(len(optimizer.X[:, 1]), 1).reshape(-1) zz = -optimizer.Y.reshape(-1) surf = ax.plot_trisurf(xx, yy, zz, cmap='viridis') fig.colorbar(surf) plt.xlabel('Alpha') plt.ylabel('Epsilon') plt.title('MountainCarContinuos V0 Optimization') plt.show()
def test_normalization(self): """Make sure normalization works with wrong model.""" np.random.seed(1) bounds = [{'name': 'var_1', 'type': 'continuous', 'domain': (1, 2)}] # We get a first measurement at 1, but the model is so harshly # violated that we will just continue to get measurements at 1. # The minimum is at 2. x0 = np.array([[1]]) f = lambda x: -1000 * x # Remove hyperparameter optimization, not needed here model = GPyOpt.models.GPModel(max_iters=0) opt = BayesianOptimization(f, bounds, X=x0, model=model, normalize_Y=True) opt.run_optimization(max_iter=1) # Make sure that we did not sample the same point again assert np.linalg.norm(opt.X[0] - opt.X[1]) > 1e-2
def optimizer_func(X, Y, BatchSize): ''' Bayesian Optimizer Function BatchSize is the number of suggestions for the next rounds X should be the input variables Y should be the metric to be optimized ''' bds = [ { 'name': 'x1', 'type': 'continuous', 'domain': (0, 1) }, { 'name': 'x2', 'type': 'continuous', 'domain': (0, 1) }, { 'name': 'x3', 'type': 'continuous', 'domain': (0, 1) }, ] constraints = [ { 'name': 'constr_1', 'constraint': 'x[:,0] + x[:,1] + x[:,2] -(1 + 0.005)' }, ###<= 0 { 'name': 'constr_2', 'constraint': '(1- 0.005) - (x[:,0] + x[:,1] + x[:,2]) ' } ] ###<= 0 kernel = GPy.kern.Matern52(input_dim=len(bds), ARD=True) optimizer = BayesianOptimization( f=None, domain=bds, constraints=constraints, model_type='GP', acquisition_type='EI', acquisition_jitter=0.1, X=X, Y=Y, evaluator_type='local_penalization', batch_size=BatchSize, normalize_Y=True, #noise_var = 0.02**2, kernel=kernel) return optimizer
def optimize_theta(self): ''' Optimizes all hyper-parameters (including noise) by maximizing the evidence ''' if self.verbose: print("Hyperparameter optimization begins...") start = time.time() #BayesianOptimization #Higher lengthscale generates more accurate GP mean (and location of maximizer of mean) #However, higher lengthscale also generates less accurate argmax distribution (argmax samples are widepread) ''' Bounds for hyperparameters: When COVARIANCE_SHRINKAGE = 1e-6''' bounds = [{'name': 'sigma', 'type': 'continuous', 'domain': (0.0001,0.1)}, #Too low noise gives non-PSD covariance matrix {'name': 'leghtscale', 'type': 'continuous', 'domain': (0.05,0.9)}, #Theoretical l max: 4*np.sqrt(self.D) {'name': 'sigma_l', 'type': 'continuous', 'domain': (0.01,0.5)}] # since f is a utility function this parameter make no much sense. BO = BayesianOptimization(lambda theta: -self.evidence(theta[0],self.fMAP), #theta[0] since need to unnest list one level domain=bounds, optimize_restarts=3, normalize_Y=True, initial_design_numdata=20) BO.run_optimization(max_iter = 50) if self.verbose: print('Optimization of hyperparameters took ' + str(time.time()-start) + ' seconds.') self.theta = BO.x_opt if self.verbose: print("The optimized theta is "+ str(self.theta))
def test_save_gp_no_filename(self): myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d) myBopt.run_optimization(max_iter=1, verbosity=False) # Need to at least pass in filename or buffer self.assertRaises(TypeError, lambda: myBopt.save_models())
def test_save_gp_default(self): myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d) myBopt.run_optimization(max_iter=1, verbosity=False) myBopt.save_models(self.outfile_path) self.check_output_model_file(['Iteration'])
run_id = sys.argv[2] else: raise Exception('Must specify a second run_id argument.') print('run_id: %s' % run_id) print('constraints:') print(mixed_domain) G, labels_matrix = load_blogcat() input_dim = len(mixed_domain) func = DwOpt(input_dim, bounds=mixed_domain, default_params=default_params, run_id=run_id) bo = BayesianOptimization(f=func.f, # function to optimize domain=mixed_domain, # box-constrains of the problem initial_design_numdata=20, # number data initial design acquisition_type='EI', # Expected Improvement exact_feval=True, # True evaluations00 verbosity=True) max_iter = 40 max_time = None bo.run_optimization(max_iter, max_time) save_opt(bo, run_id) func.shutdown() print('\nNumber of fails: %i' % func.fails) if action == 'load': if len(sys.argv) >= 3: run_id = sys.argv[2] else:
def test_save_gp_default_no_iters(self): myBopt = BayesianOptimization(f=self.f_2d, domain=self.domain_2d) # Exception should be raised as no iterations have been carried out yet self.assertRaises(ValueError, lambda: myBopt.save_models(self.outfile_path))
def evalBOLikelihood( self, w, testdata, c0, c1, c_eval=0, c_min=0.01, c_max=0.2, use_log=False, true_dist=False, vars_g=None, npoints=50, samples_ids=None, weights_func=None): ''' Find minimum of likelihood on testdata using decomposed ratios and the weighted orthogonal morphing method to find the bases ''' if true_dist: vars = ROOT.TList() for var in vars_g: vars.Add(w.var(var)) x = ROOT.RooArgSet(vars) else: x = None score = ROOT.RooArgSet(w.var('score')) if use_log: evaluateRatio = self.evaluateLogDecomposedRatio post = 'log' else: evaluateRatio = self.evaluateDecomposedRatio post = '' # Compute bases if they don't exist for this range if not os.path.isfile( '3doubleindexes_{0:.2f}_{1:.2f}_{2:.2f}_{3:.2f}_{4}.dat'.format( c_min[0], c_min[1], c_max[0], c_max[1], npoints)): self.pre2DDoubleBasis(c_min=c_min, c_max=c_max, npoints=npoints) all_indexes = np.loadtxt( '3doubleindexes_{0:.2f}_{1:.2f}_{2:.2f}_{3:.2f}_{4}.dat'.format( c_min[0], c_min[1], c_max[0], c_max[1], npoints)) all_indexes = np.array([[int(x) for x in rows] for rows in all_indexes]) # Bkg used in the fit # TODO: Harcoded this have to be changed basis_value = 1 # Pre evaluate the values for each distribution pre_pdf = [[range(self.nsamples) for _ in range(self.nsamples)], [ range(self.nsamples) for _ in range(self.nsamples)]] pre_dist = [[range(self.nsamples) for _ in range(self.nsamples)], [ range(self.nsamples) for _ in range(self.nsamples)]] # Only precompute distributions that will be used unique_indexes = set() for indexes in all_indexes: unique_indexes |= set(indexes) # change this enumerates unique_indexes = list(unique_indexes) for k in range(len(unique_indexes)): for j in range(len(unique_indexes)): index_k, index_j = (unique_indexes[k], unique_indexes[j]) # This save some time by only evaluating the needed samples if index_k != basis_value: continue print 'Pre computing {0} {1}'.format(index_k, index_j) if k != j: f0pdf = w.function( 'bkghistpdf_{0}_{1}'.format( index_k, index_j)) f1pdf = w.function( 'sighistpdf_{0}_{1}'.format( index_k, index_j)) data = testdata if self.preprocessing: data = preProcessing(testdata, self.dataset_names[min( k, j)], self.dataset_names[max(k, j)], self.scaler) # outputs = # predict('{0}/model/{1}/{2}/{3}_{4}_{5}.pkl'.format(self.dir,self.model_g, outputs = predict( '/afs/cern.ch/work/j/jpavezse/private/{0}_{1}_{2}.pkl'.format( self.model_file, index_k, index_j), data, model_g=self.model_g) f0pdfdist = np.array( [self.evalDist(score, f0pdf, [xs]) for xs in outputs]) f1pdfdist = np.array( [self.evalDist(score, f1pdf, [xs]) for xs in outputs]) pre_pdf[0][index_k][index_j] = f0pdfdist pre_pdf[1][index_k][index_j] = f1pdfdist else: pre_pdf[0][index_k][index_j] = None pre_pdf[1][index_k][index_j] = None if true_dist: f0 = w.pdf('f{0}'.format(index_k)) f1 = w.pdf('f{0}'.format(index_j)) if len(testdata.shape) > 1: f0dist = np.array([self.evalDist(x, f0, xs) for xs in testdata]) f1dist = np.array([self.evalDist(x, f1, xs) for xs in testdata]) else: f0dist = np.array([self.evalDist(x, f0, [xs]) for xs in testdata]) f1dist = np.array([self.evalDist(x, f1, [xs]) for xs in testdata]) pre_dist[0][index_k][index_j] = f0dist pre_dist[1][index_k][index_j] = f1dist indices = np.ones(testdata.shape[0], dtype=bool) samples = [] # Usefull values to inspect after the training target = self.F1_couplings[:] def compute_one_alpha_part(weights, xs): c1s_1 = np.multiply(weights,xs) c1s_1 = np.multiply(weights,c1s_1) alpha1 = c1s_1.sum() return alpha1 exp_basis_weights = True def vectorize(func): def wrapper(X): v = np.zeros(len(X)) for i, x_i in enumerate(X): v[i] = func(x_i) return v.reshape(-1, 1) return wrapper morph = MorphingWrapper() morph.setSampleData( nsamples=15, ncouplings=3, types=[ 'S', 'S', 'S'], ncomb=19, morphed=self.F1_couplings, samples=self.all_couplings) def objective(theta): n_effs = np.zeros(2) alpha = np.zeros(2) cs = [] cross_section_list = [] couplings_list = [] for ix,ind in enumerate(all_indexes): target[1] = theta[0] target[2] = theta[1] print target morph.resetTarget(target) ind = np.array(ind) morph.resetBasis([self.all_couplings[int(k)] for k in ind]) couplings = np.array(morph.getWeights()) cross_section = np.array(morph.getCrossSections()) cross_section_list.append(cross_section) couplings_list.append(couplings) # Compute F1 couplings and cross sections c1_ = np.multiply(couplings, cross_section) n_eff = c1_.sum() n_tot = np.abs(c1_).sum() n_effs[ix] = n_eff / n_tot cs.append(c1_ / c1_.sum()) if exp_basis_weights == False: alpha[ind] = compute_one_alpha_part(couplings, cross_section) if exp_basis_weights == True: neff2 = 1./n_effs[1] neff1 = 1./n_effs[0] alpha1 = np.exp(-neff1**(1./3.)) alpha2 = np.exp(-neff2**(1./3.)) alpha[0] = alpha1/(alpha1 + alpha2) alpha[1] = alpha2/(alpha1 + alpha2) else: alpha[0] = (1/2.)*(alpha2/(alpha1+alpha2)) alpha[1] = (1/2.)*(alpha1/(alpha1+alpha2)) # Compute Bkg weights c0_arr_1 = np.zeros(15) c0_arr_2 = np.zeros(15) c0_arr_1[np.where(all_indexes[0] == basis_value)[0][0]] = 1. c0_arr_2[np.where(all_indexes[1] == basis_value)[0][0]] = 1. c0_arr_1 = c0_arr_1 / c0_arr_1.sum() c0_arr_2 = c0_arr_2 / c0_arr_2.sum() c1s = np.append(alpha[0] * cs[0], alpha[1] * cs[1]) c0_arr = np.append(0.5 * c0_arr_1, 0.5 * c0_arr_2) n_eff_ratio = (alpha[0] * n_effs[0] + alpha[1] * n_effs[1]) cross_section = np.append(cross_section_list[0], cross_section_list[1]) indexes = np.append(all_indexes[0], all_indexes[1]) completeRatios, trueRatios = evaluateRatio(w, testdata, x=x, plotting=False, roc=False, c0arr=c0_arr, c1arr=c1s, true_dist=true_dist, pre_dist=pre_dist, pre_evaluation=pre_pdf, cross_section=cross_section, indexes=indexes) completeRatios = 1. / completeRatios if not use_log: if n_eff_ratio < 0.3: # TODO: Harcoded number decomposedLikelihood = 20000 else: decomposedLikelihood = -2.*np.mean(np.log(completeRatios)) else: decomposedLikelihood = -np.mean(completeRatios.mean()) print completeRatios[completeRatios < 0.].shape return decomposedLikelihood bounds = [(c_min[0],c_max[0]),(c_min[1],c_max[1])] solver = BayesianOptimization(vectorize(objective), bounds) solver.run_optimization(max_iter=150, true_gradients=False) approx_MLE = solver.x_opt print("Approx. MLE =", approx_MLE) return ((0.,0.),approx_MLE)