def gp_fit_sklearn(x_input, x_tar, y_input, y_tar, params=None, title=''): k1 = kernels.DotProduct(sigma_0=1, sigma_0_bounds=(1e-05, 5)) k2 = kernels.RBF(length_scale=10, length_scale_bounds=(1e-3, x_tar[-1])) k3 = kernels.RationalQuadratic(alpha=1, length_scale=10, length_scale_bounds=(1e-3, x_tar[-1])) kernel = k1 * k2 * k3 gp1 = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, normalize_y=True, alpha=0) if params: gp1.set_params(params) gp1.fit(x_input.reshape(-1, 1), y_input) pred, std = gp1.predict(x_tar.reshape(-1, 1), return_std=True) plt.plot(x_input, y_input, 'bo', label='Input', alpha=0.4) plt.plot(x_tar, y_tar, 'go', label='Target', alpha=0.4) plt.plot(x_tar, pred, 'ro', label='Prediction', alpha=0.4) plt.gca().fill_between(x_tar, pred.reshape(-1) - 2 * std, pred.reshape(-1) + 2 * std, color='lightblue', alpha=0.5, label=r"$2\sigma$") plt.title(title) plt.legend() plt.show() return gp1, pred
def get_fitted_gaussian_processor(X_train, y_train, constraint_upper, standardize_y=True, **gp_params): # Initialize gaussian process regressor gp = GaussianProcessRegressor() gp.set_params(**gp_params) logger.debug('Instantiated gaussian processor for objective function:\n' + f'{gp}') logger.debug(f"Fitting gaussian processor") if standardize_y: if constraint_upper is not None: y_train = scale(np.hstack((y_train, constraint_upper))) scaled_constraint_upper = y_train[-1] y_train = y_train[:-1] else: y_train = scale([i for i in y_train]) scaled_constraint_upper = None gp.constraint_upper = scaled_constraint_upper else: gp.constraint_upper = constraint_upper logger.debug(f'X_train:\n{X_train}') logger.debug(f'y_train\n{y_train}') logger.debug(f'constraint_upper: {gp.constraint_upper}') if gp_params is None or gp_params.get('alpha') is None: # Find unique rows of X to avoid GP from breaking ur = unique_rows(X_train) gp.fit(X_train[ur], y_train[ur]) else: gp.fit(X_train, y_train) return gp
def _gp_fit(self, X, y, kernel=None): if kernel is None: kernel = Matern(nu=2.5) gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, normalize_y=True, random_state=self.random_state) gp.set_params(**self.gp_params) gp.fit(X, y) return gp
def train_and_save_final_model(X, y, X_train, y_train, params, save_model_file_path, test_data): gpr = GaussianProcessRegressor(random_state=0) gpr.set_params(**params) if test_data == None: gpr.fit(X_train, y_train) else: gpr.fit(X, y) #save model model_file_path = save_model_file_path + 'gpr.sav' pickle.dump(gpr, open(model_file_path, 'wb'))
class Gaussian_Process(): def __init__(self, kernel, mode): self.kernel = kernel if mode == 'OPT': self.gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=10, normalize_y=True) elif mode == 'MAP': self.gp = GaussianProcessRegressor(kernel=kernel, optimizer = None, n_restarts_optimizer=10, normalize_y=True) else: raise Exception('Wrong GP initial mode!!!') ''' def optimise__with_splice_sampling(self, initial_theta, num_iters, sigma, burn_in, X, y): self.fit(X, y) slice_sampler = Slice_sampler(num_iters = num_iters, sigma = sigma, burn_in = burn_in, gp = self) samples = slice_sampler.sample(init = initial_theta) theta_opt = [np.mean(samples[0]),np.mean(samples[1])] self.gp.set_params(**{"kernel__k1__length_scale": np.mean(samples[0]), "kernel__k2__noise_level": np.power(np.mean(samples[0]),2)}) ''' def set_params(self, theta_set): self.gp.set_params(**{"kernel__k1__noise_level": np.abs(theta_set[0]), "kernel__k2__k1__constant_value": np.abs(theta_set[1]), "kernel__k2__k2__length_scale": theta_set[2:]}) def log_marginal_likelihood(self,theta): return self.gp.log_marginal_likelihood(theta) def log_prior_parameters(self, theta): return np.sum(np.log([ss.norm(0, 1).pdf(theta_k) for theta_k in theta])) def log_joint_unnorm(self, theta): return self.log_marginal_likelihood(theta) + self.log_prior_parameters(theta) def fit(self, X, y): self.gp.fit(X, y) def predict(self, X): mu, sigma = self.gp.predict(X, return_std =True) return mu, sigma
class GP(object): def __init__(self, kernel_name, n_restarts_optimizer, random_state): if kernel_name=='matern': self.kernel = Matern(nu=2.5) self.gpr_ = GaussianProcessRegressor(kernel=self.kernel, n_restarts_optimizer=n_restarts_optimizer, random_state = random_state) self.ymax = None def fit(self, x, y): """ gaussian process fit Parameters: ------------ x: 2d-array must be 2d-array, shape = (smaple_num, feature_dim) y: 2d-array must be 2d-array, shape = (sample_num, 1) """ assert x.ndim==2, 'x must be 2D array' assert y.ndim==2, 'y must be 2D array' return self.gpr_.fit(x,y) def predict(self, x): """ predict by gaussian process fit Parameters: ------------ x: 2d-array must be 2d-array, shape=[sample_num, feature_dim] return_std: bool Returns: mu: 2d-array must be 2d-array, shape = (sample_num, output_dim) sigma: 1d-array shape = (sample_num,) """ assert x.ndim==2, 'x must be 2D array' mu, sigma = self.gpr_.predict(x, return_std=True) return mu, sigma def set_params(self, **gp_params): self.gpr_.set_params(**gp_params)
class GPR(object): def __init__(self, kername, n_restarts_optimizer, random_state=None): if kername == 'matern': kernel = Matern(nu=2.5) self.gpr_ = GaussianProcessRegressor( kernel=kernel, n_restarts_optimizer=n_restarts_optimizer, random_state=random_state) def fit(self, x, y): return self.gpr_.fit(x, y) def predict(self, x, return_std=True): return self.gpr_.predict(x, return_std=return_std) def set_params(self, **gp_params): self.gpr_.set_params(**gp_params)
class GaussianProcessOptimizer(object): def __init__(self, sample_function, kappa=20, seed=None, init_points=5, xi=0.01, **gp_params): self.f = sample_function # Seed the random number generator if seed was provided rand = np.random.RandomState( ) if seed is None else np.random.RandomState(seed) # GP Regression self.gp = GaussianProcessRegressor( #kernel=Matern(nu=2.5), kernel=RationalQuadratic(), n_restarts_optimizer=25, random_state=rand) self.gp.set_params(**gp_params) # Utility function parameter self.kappa = kappa self.xi = xi self._acq = lambda p: -self._expected_improvement(p) #self._acq = lambda p: - self._upper_confidence_bound(p) self.v_max = None # initial points, form { p : f(p) } self.points = { p: self.f(p) for p in [(_R(), _R()) for _ in range(init_points)] } self.v_max = max(self.points.values()) self.update_fit() self.last_point = None def _upper_confidence_bound(self, p): mu, sigma = self.gp.predict(np.array([np.asarray(p)]), return_std=True) return mu + (self.kappa * sigma) def _expected_improvement(self, p): mu, sigma = self.gp.predict(np.array([np.asarray(p)]), return_std=True) z = (mu - self.v_max - self.xi) / sigma return (mu - self.v_max - self.xi) * norm.cdf(z) + sigma * norm.pdf(z) def update_fit(self): keys = [k for k in self.points.keys()] X = np.array([np.asarray(k) for k in keys]) y = np.array([self.points[k] for k in keys]) print(X, y) self.gp.fit(X, y) def _umax(self, i_points=5): ores = scipy.optimize.minimize(self._acq, [_R(), _R()], bounds=[(0, 1), (0, 1)]) print(ores) return (ores.x[0], ores.x[1]) def predict(self, p): r = self.gp.predict(np.array([np.asarray(p)]), return_std=True) return (r[0][0], r[1][0]) def maximize(self, N=50): for _ in range(N): p = self._umax() i = 0 while p in self.points: p = (_R(), _R()) i += 1 if i > 5: print("Something nefarious is afoot.") v = self.f(p) if self.v_max is None or v > self.v_max: self.v_max = v self.points[p] = v self.update_fit() self.last_point = p
class BayesianOptimization(object): def __init__(self, f, pbounds, verbose=1): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds # Get the name of the parameters self.keys = list(pbounds.keys()) # Find number of parameters self.dim = len(pbounds) # Create an array with parameters bounds self.bounds = [] for key in self.pbounds.keys(): self.bounds.append(self.pbounds[key]) self.bounds = np.asarray(self.bounds) # Some function to be optimized self.f = f # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Numpy array place holders self.X = None self.Y = None # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), n_restarts_optimizer=25, ) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # Verbose self.verbose = verbose # set the parameter mesh # self.set_parameter_mesh() def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Generate random points l = [ np.random.uniform(x[0], x[1], size=init_points) for x in self.bounds ] # Concatenate new random points to possible existing # points from self.explore method. self.init_points += list(map(list, zip(*l))) # Create empty list to store the new values of the function y_init = [] # Evaluate target function at all initialization # points (random + explore) for x in self.init_points: y_init.append(self.f(**dict(zip(self.keys, x)))) if self.verbose: self.plog.print_step(x, y_init[-1]) # Append any other points passed by the self.initialize method (these # also have a corresponding target value passed by the user). self.init_points += self.x_init # Append the target value of self.initialize method. y_init += self.y_init # Turn it into np array and store. self.X = np.asarray(self.init_points) self.Y = np.asarray(y_init) # Updates the flag self.initialized = True def explore(self, points_dict): """Method to explore user defined points :param points_dict: """ # Consistency check param_tup_lens = [] for key in self.keys: param_tup_lens.append(len(list(points_dict[key]))) if all([e == param_tup_lens[0] for e in param_tup_lens]): pass else: raise ValueError('The same number of initialization points ' 'must be entered for every parameter.') # Turn into list of lists all_points = [] for key in self.keys: all_points.append(points_dict[key]) # Take transpose of list self.init_points = list(map(list, zip(*all_points))) def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) # Loop through the all bounds and reset the min-max bound matrix for row, key in enumerate(self.pbounds.keys()): # Reset all entries, even if the same. self.bounds[row] = self.pbounds[key] def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Upper Confidence Bound. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing """ # Reset timer self.plog.reset_timer() # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points) y_max = self.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False if np.any((self.X - x_max).sum(axis=1) == 0): x_max = np.random.uniform(self.bounds[:, 0], self.bounds[:, 1], size=self.bounds.shape[0]) pwarning = True # Append most recently generated values to X and Y arrays self.X = np.vstack((self.X, x_max.reshape((1, -1)))) self.Y = np.append(self.Y, self.f(**dict(zip(self.keys, x_max)))) # Updating the GP. ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Update maximum value to search for next probe point. if self.Y[-1] > y_max: y_max = self.Y[-1] # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds) # Print stuff if self.verbose: self.plog.print_step(self.X[-1], self.Y[-1], warning=pwarning) # Keep track of total number of iterations self.i += 1 self.res['max'] = { 'max_val': self.Y.max(), 'max_params': dict(zip(self.keys, self.X[self.Y.argmax()])) } self.res['all']['values'].append(self.Y[-1]) self.res['all']['params'].append(dict(zip(self.keys, self.X[-1]))) # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.X, np.expand_dims(self.Y, axis=1))) header = ', '.join(self.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',') def plot_stuff(self, params, resolution): ''' Nest function for plotting. :param params A dictionary that contains the parameter values/ranges for plotting, where 'None' indicates to plot over that variable example 1: for target function p(x,y,z), params = { x: 1, y: 2, z: None } projects the parameter space onto a 1d hyperplane defined by p = f(x=1,y=2,z) example 2: for target function p(x,y,z), params = { x: 1, y: None, z: None } projects the parameter space onto a 2d hyperplane defined by p = f(x=1,y,z) :param resolution The length to use for the plotting variable(s) indicated in 'params' ''' mesh, mu, sigma, utility = self.posterior(params, resolution) plot_params = [k for k, v in params.iteritems() if v == None] if len(plot_params) == 1: self.plot1d(mesh, params, plot_params[0], mu, sigma, resolution) elif len(plot_params) == 2: self.plot2d(mesh, params, plot_params, mu, sigma, utility, resolution) else: print('invalid number of plotting variables passed in') def plot1d(self, mesh, params, plot_param, mu, sigma, resolution): ''' plot 1d thing Parameters ---------- :param mesh a parameter mesh passed from get_parameter_mesh() :param params passed from plot_stuff function :param plot_param the parameter being plotted over :param mu GP predicted mean :param sigma GP predicted uncertainty :param resolution plotting resolution ''' x = mesh[:, self.keys.index(plot_param)].flatten() fontsize = 9 fig = plt.figure(figsize=(7, 5)) rest_string = 'Gaussian Process and Utility Function After {} Steps, projected onto {}\n'.format( len(self.X), param_mapping_helper(plot_param, 0, just_param=True)) for k, v in params.iteritems(): if v is not None: kn, vn = param_mapping_helper(k, v) rest_string += '{} = {}, '.format(kn, vn) rest_string = rest_string[:-2] fig.suptitle(rest_string, fontdict={'size': fontsize + 2}) gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1]) axis = plt.subplot(gs[0]) acq = plt.subplot(gs[1]) # axis.plot(x, y, linewidth=3, label='Target') axis.plot(self.X[:, self.keys.index(plot_param)].flatten(), self.Y, 'D', markersize=8, label=u'Observations (projected)', color='r') axis.plot(x, mu, '--', color='k', label='Prediction') axis.fill(np.concatenate([x, x[::-1]]), np.concatenate( [mu - 1.9600 * sigma, (mu + 1.9600 * sigma)[::-1]]), alpha=.6, fc='c', ec='None', label='95% confidence interval') # axis.set_xlim((-2, 10)) axis.set_ylim((-0.2, 1.2)) axis.set_ylabel('Area Under Precision Recall Curve', fontdict={'size': fontsize}) # axis.set_xlabel(r'$log(\tau)$', fontdict={'size':fontsize}) utility = self.util.utility(mesh, self.gp, 0) acq.plot(x, utility, label='Utility Function', color='purple') acq.plot(x[np.argmax(utility)], np.max(utility), '*', markersize=15, label=u'Next Best Guess', markerfacecolor='gold', markeredgecolor='k', markeredgewidth=1) # acq.set_xlim((-2, 10)) acq.set_ylim((0, np.max(utility) + 0.5)) acq.set_ylabel('Utility', fontdict={'size': fontsize}) acq.set_xlabel(param_mapping_helper(plot_param), fontdict={'size': fontsize}) axis.legend(loc=4) acq.legend(loc=4) print(mu) print(sigma) plt.show() def plot2d(self, mesh, params, plot_params, mu, sigma, utility, resolution): ''' plot 2d thing Parameters ---------- :param mesh a parameter mesh passed from get_parameter_mesh() :param params passed from plot_stuff function :param plot_params list of the 2 parameter being plotted over :param mu GP predicted mean :param sigma GP predicted uncertainty :param resolution plotting resolution ''' fontsize = 13 # prepare shit. make the plot objects and get the range that you're plotting over fig, ax = plt.subplots(2, 2, figsize=(14, 10)) rest_string = 'Gaussian Process and Acquisition Function after {} Steps, projected onto ({},{})\n'.format( len(self.X), param_mapping_helper(plot_params[0]), param_mapping_helper(plot_params[1])) for k, v in params.iteritems(): if v is not None: kn, vn = param_mapping_helper(k, v) rest_string += '{} = {}, '.format(kn, vn) fig.suptitle(rest_string, fontdict={'size': fontsize + 4}) x = mesh[:, self.keys.index(plot_params[0])].flatten() y = mesh[:, self.keys.index(plot_params[1])].flatten() plots = [] # GP mean ax[0][0].set_title('GP Predicted Mean', fontdict={'size': fontsize}) im00 = ax[0][0].hexbin(x, y, C=mu, gridsize=int(resolution * 2 / 3), vmin=0, vmax=1) plots.append((ax[0][0], im00)) # GP variance ax[1][0].set_title('GP Variance', fontdict={'size': fontsize}) im10 = ax[1][0].hexbin(x, y, C=sigma, gridsize=int(resolution * 2 / 3), vmin=np.min(sigma), vmax=np.max(sigma)) plots.append((ax[1][0], im10)) # Utility function ax[0][1].set_title('Acquisition Function', fontdict={'size': fontsize}) im01 = ax[0][1].hexbin(x, y, C=utility, gridsize=int(resolution * 2 / 3), vmin=np.min(utility), vmax=np.max(utility)) plots.append((ax[0][1], im01)) # Observations ax[1][1].set_title('Observations', fontdict={'size': fontsize}) ax[1][1].plot(self.X[:, self.keys.index(plot_params[0])].flatten(), self.X[:, self.keys.index(plot_params[1])].flatten(), 'x', markersize=4, color='k', label='Observations') plots.append((ax[1][1], None)) # for every plot, do some labeling shit lims = [ self.pbounds[plot_params[i]][j] for i in [0, 1] for j in [0, 1] ] for axis, im in plots: axis.axis(lims) axis.set_xlabel(param_mapping_helper(plot_params[0])) axis.set_ylabel(param_mapping_helper(plot_params[1])) axis.set_aspect('equal') if axis.get_title() != 'Observations': cb = fig.colorbar(im, ax=axis) # show plt.show() def get_parameter_mesh(self, param_inputs, resolution): ''' A helper function for 'plot_stuff' method. Takes parameter range that is being plotted over and returns a set of mesh-grid points for that range Parameters ---------- :param param_inputs passed from plot_stuff function :param resolution passed from plot_stuff function Returns _______ :return X np.ndarray of shape (nb plotting points, nb target function features) ''' param_inputs_copy = param_inputs.copy() for k, v in param_inputs_copy.iteritems(): if v == None: param_inputs_copy[k] = np.linspace(self.pbounds[k][0], self.pbounds[k][1], resolution) else: param_inputs_copy[k] = np.array(v) mesh_guys = np.meshgrid(*[param_inputs_copy[i] for i in self.keys]) mesh_flats = [guy.ravel() for guy in mesh_guys] X = np.vstack(mesh_flats).T return X def posterior(self, param_inputs, resolution): ''' Computes posterior over the parameter space Parameters __________ :param param_inputs passed from plot_stuff function :param resolution passed from plot_stuff function Returns _______ :return mesh passed from function get_parameter_mesh, np.ndarray of shape (nb plotting points, nb target function features) :return mu GP predicted mean, np.ndarray of shape (nb plotting points,) :return sigma GP predicted uncertainty, np.ndarray of shape (nb plotting points,) :return utility acquisition function, np.ndarray of shape (nb plotting points,) ''' mesh = self.get_parameter_mesh(param_inputs, resolution) self.gp.fit(self.X, self.Y) mu, sigma = self.gp.predict(mesh, return_std=True) utility = self.util.utility(mesh, self.gp, self.Y.max()) # print mu.shape # print sigma.shape # print utility.shape return mesh, mu, sigma, utility
class BayesianOptimization(object): def __init__(self, f, pbounds, opt_value, random_state=None): """ :param f: Function to be maximized. :param pbounds: Array with minimum and maximum values for each dimensions' range. :param opt_value: Function's real optimum value. """ self.opt_value = opt_value if random_state is None: self.random_state = np.random.RandomState() elif isinstance(random_state, int): self.random_state = np.random.RandomState(random_state) else: self.random_state = random_state self.bounds = pbounds self.dx = self.bounds.shape[1] self.f = f # Initialization lists --- stores starting points before process begins self.init_points = [] # Numpy array place holders self.X = None self.Y = None # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state) # Utility Function placeholder self.util = None def __call__(self, x): """ Method to get function's evaluations. :param x: Point where the function needs to be evaluated. """ return self.f(x) def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Generate random points self.init_points = self.random_state.uniform(self.bounds[0], self.bounds[1], size=(init_points, self.dx)) # Create empty arrays to store the new points and values of the function. self.X = np.empty((0, self.dx)) self.Y = np.empty(0) # Evaluate target function at all initialization points self.X = np.vstack((self.X, self.init_points)) for y in range(init_points): self.Y = np.append(self.Y, self.f(self.X[y])) def maximize(self, init_points=5, n_iter=25, acq='ei', kappa=2.576, xi=0.0, **gp_params): """ Bayesian Optimization method. :param init_points: Number of random points to probe. :param n_iter: Number of iterations. :param acq: Acquisition function to use. :param kappa: Matérn kernel's parameter. :param xi: Noise for supporting exploration. :param **gp_params: Parameters to build Gaussian Process. """ # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialization self.init(init_points) y_max = self.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking ur = unique_rows(self.X) # Build Gaussian Process self.gp.fit(self.X[ur], self.Y[ur]) # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds, random_state=self.random_state, dx=self.dx) # Placeholders simple_regret = np.empty(n_iter) y_best = np.empty(n_iter) tot_time = np.empty(n_iter) # Start iterations for i in range(n_iter): # Set initial computaitonal time init_time = time.time() pwarning = False if np.any((self.X - x_max).sum(axis=1) == 0): x_max = self.random_state.uniform(self.bounds[:, 0], self.bounds[:, 1], size=self.bounds.shape[0]) pwarning = True # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds, random_state=self.random_state, dx=self.dx) # Append most recently generated values to X and Y arrays self.X = np.vstack((self.X, x_max.reshape((1, -1)))) self.Y = np.append(self.Y, self.f(x_max)) # Updating the GP. ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Update maximum value to search for next probe point. if self.Y[-1] > y_max: y_max = self.Y[-1] # Update placeholders y_best[i] = -y_max simple_regret[i] = -y_max - self.opt_value if i == 0: tot_time[i] = time.time() - init_time else: tot_time[i] = tot_time[i - 1] + time.time() - init_time return y_best, simple_regret, tot_time
class BayesianOptimization(Observable): def __init__(self, f, pbounds, random_state=None, verbose=2, lazy_gpr=False, lag=10000): """ this is a comment """ self._random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self._space = TargetSpace(f, pbounds, random_state) # queue self._queue = Queue() if (lazy_gpr): # Internal lazy GP regressor self._gp = GaussianProcessRegressor_lazy( kernel=Matern(nu=2.5), alpha=1e-6, normalize_y=True, n_restarts_optimizer=1, random_state=self._random_state, lag=lag #optimizer=None ) else: # Internal GP regressor self._gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), alpha=1e-6, normalize_y=True, n_restarts_optimizer=1, random_state=self._random_state #optimizer=None ) self._verbose = verbose super(BayesianOptimization, self).__init__(events=DEFAULT_EVENTS) @property def space(self): return self._space @property def max(self): return self._space.max() @property def res(self): return self._space.res() def get_lower_L(self): return self._gp.L def register(self, params, target): """Expect observation with known target""" self._space.register(params, target) self.dispatch(Events.OPTMIZATION_STEP) def probe(self, params, lazy=True): """Probe target of x""" x = 0 if lazy: self._queue.add(params) else: x = self._space.probe(params) self.dispatch(Events.OPTMIZATION_STEP) return x def suggest(self, utility_function): """Most promissing point to probe next""" if len(self._space) == 0: return self._space.array_to_params(self._space.random_sample()) # Sklearn's GP throws a large number of warnings at times, but # we don't really need to see them here. with warnings.catch_warnings(): warnings.simplefilter("ignore") self._gp.fit(self._space.params, self._space.target) # Finding argmax of the acquisition function. suggestion = acq_max(ac=utility_function.utility, gp=self._gp, y_max=self._space.target.max(), bounds=self._space.bounds, random_state=self._random_state) return self._space.array_to_params(suggestion) def _prime_queue(self, init_points): """Make sure there's something in the queue at the very beginning.""" if self._queue.empty and self._space.empty: init_points = max(init_points, 1) for _ in range(init_points): self._queue.add(self._space.random_sample()) def _prime_subscriptions(self): if not any([len(subs) for subs in self._events.values()]): _logger = _get_default_logger(self._verbose) self.subscribe(Events.OPTMIZATION_START, _logger) self.subscribe(Events.OPTMIZATION_STEP, _logger) self.subscribe(Events.OPTMIZATION_END, _logger) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.01, samples=None, eps=0.2, solution=1, **gp_params): """Mazimize your function""" self._prime_subscriptions() self.dispatch(Events.OPTMIZATION_START) self._prime_queue(init_points) #add random points to the queue self.set_gp_params(**gp_params) util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) iteration = 0 total_time = 0 while not self._queue.empty or iteration < n_iter: try: x_probe = next(self._queue) # print(self._gp.kernel.theta) except StopIteration: tstart = time.time() x_probe = self.suggest(util) tend = time.time() total_time += (tend - tstart) iteration += 1 # print(self._gp.kernel.theta) y = self.probe(x_probe, lazy=False) if (abs(y - solution) < eps): print("Breaking the loop now ...") break # if samples != None : # with open("samples.pickle", "rb") as f: # epochs, wd, lr, m, acc = pickle.load(f) # for _ in range(len(epochs)): # self._space.register_seeds(x, params) # self.dispatch(Events.OPTMIZATION_STEP) self.dispatch(Events.OPTMIZATION_END) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds Parameters ---------- new_bounds : dict A dictionary with the parameter name and its new bounds """ self._space.set_bounds(new_bounds) def set_gp_params(self, **params): self._gp.set_params(**params)
class BayesianOptimization(object): def __init__(self, f, options, random_state=None): """ :param f: Function to be maximized. :param options: Dictionary with parameters names as keys and their values. """ self.options = options self.opt_value = options['opt_value'] if random_state is None: self.random_state = np.random.RandomState() elif isinstance(random_state, int): self.random_state = np.random.RandomState(random_state) else: self.random_state = random_state self.method = options['method'] self.bounds = options['pbounds'] self.dx = self.bounds.shape[1] # Some function to be optimized self.f = f # Initialization lists --- stores starting points before process begins self.init_points = [] # Numpy array place holders self.X = None self.Y = None # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state) # Utility Function placeholder self.util = None self.n_iter = options['n_iter'] self.B = options['B'] self.n_bo = options['n_bo'] self.n_percent = options['n_percent'] self.init(options['init_points']) self.acq = options['acq_fun'] def __call__(self, x): """ :param x: Point the function needs to be evaluated. """ return self.f(x) def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Generate random points self.init_points = self.random_state.uniform(self.bounds[0], self.bounds[1], size=(init_points, self.dx)) # Create empty arrays to store the new points and values of the function. self.X = np.empty((0, self.dx)) self.Y = np.empty(0) # Evaluate target function at all initialization self.X = np.vstack((self.X, self.init_points)) for y in range(init_points): self.Y = np.append(self.Y, self.f(self.X[y])) def maximize(self, **gp_params): if self.method == 'mondrian': from mondrian_classic import Decomposition elif self.method == 'dbscan': from dbscan import Decomposition elif self.method == 'kmeans': from kmeans import Decomposition else: from HDBSCAN import Decomposition # Set acquisition function self.util = UtilityFunction(kind=self.acq, kappa=self.options['k'], xi=self.options['xi']) # Initialize x, y and find current y_max y_best = self.Y.max() x_best = self.X[self.Y.argmax()] # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Finding argmax of the acquisition function. x_new = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_best, bounds=self.bounds, random_state=self.random_state, n_bo=self.n_bo, B=self.B, n_percent=self.n_percent, dx=self.dx) # Update optimum value found x_max = x_new[0] if self.f(x_max) > y_best: y_best = self.f(x_max) x_best = x_max # Placeholders simple_regret = np.empty(self.n_iter) y_bests = np.empty(self.n_iter) tot_time = np.empty(self.n_iter) # Start iterations for i in range(self.n_iter): # Set initial time init_time = time.time() newX = np.empty((0, self.dx)) newY = np.empty(0) tree = Decomposition(self.X, self.Y, self.bounds) # builds tree's leaves leaves1 = tree.building() tot_eval = np.ceil(2.0 * self.B) tot_volumn = np.array([n.volumn for n in leaves1]).sum() leaves = [e for e in leaves1 if e.X.shape[0] != 0] for l in leaves: # Define l-th partition data and range X = l.X Y = l.y bounds = l.x_range self.n_bo = np.maximum( self.n_bo, np.ceil((tot_eval * l.volumn / tot_volumn)).astype(int)) # Maximize acquisition function to find next probing point ur = unique_rows(X) self.gp.fit(X[ur], Y[ur]) x_new = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_best, bounds=bounds, random_state=self.random_state, n_bo=self.n_bo, B=self.B, n_percent=self.n_percent, dx=self.dx) x_max = x_new[0] newX = np.vstack((newX, x_new)) for k in x_new: newY = np.append(newY, self.f(k)) index = newY.argsort()[::-1][:self.B] newX = newX[index] newY = newY[index] # Update optimum value found if newY[0] > y_best: y_best = newY[0] x_best = newX[0] self.X = np.vstack((self.X, newX)) self.Y = np.hstack((self.Y, newY)) simple_regret[i] = -y_best - self.opt_value y_bests[i] = y_best if i == 0: tot_time[i] = time.time() - init_time else: tot_time[i] = tot_time[i - 1] + time.time() - init_time return -y_bests, simple_regret, tot_time
class BayesianOptimization(object): def __init__(self, pbounds, random_state=None, verbose=1): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds self.random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self.space = TargetSpace(pbounds, random_state) # Initialization flag self.initialized = False self.y_max=0 # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state ) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.space.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # non-public config for maximizing the aquisition function # (used to speedup tests, but generally leave these as is) self._acqkw = {'n_warmup': 100000, 'n_iter': 250} # Verbose self.verbose = verbose def init(self, x,y): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Concatenate new random points to possible existing # points from self.explore method. self.init_points.extend(x) # Evaluate target function at all initialization points self._observe_point(x,y) # Add the points from `self.initialize` to the observations if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) if self.verbose: self.plog.print_step(x, y) # Updates the flag self.initialized = True def _observe_point(self, x,y): self.space.observe_point(x,y) #if self.verbose: #self.plog.print_step(x, y) def explore(self, points_dict, eager=False): """Method to explore user defined points. :param points_dict: :param eager: if True, these points are evaulated immediately """ if eager: self.plog.reset_timer() if self.verbose: self.plog.print_header(initialization=True) points = self.space._dict_to_points(points_dict) for x in points: self._observe_point(x) else: points = self.space._dict_to_points(points_dict) self.init_points = points def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.space.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.space.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def conf_diction(self,conf): return self.space.config_to_dic(conf) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) self.space.set_bounds(new_bounds) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max y_max = self.space.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking self.gp.fit(self.space.X, self.space.Y) # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) #---------!!!!!!!!!!-----!!!!!!-----Spark environment if not self.space.validate_conf(x_max) : x_max = self.space.rechoose_conf(x_max) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False while x_max in self.space: print("Change X_MAX") x_max = self.space.random_points(1)[0] pwarning = True # Append most recently generated values to X and Y arrays y = self.space.observe_point(x_max) if self.verbose: self.plog.print_step(x_max, y, pwarning) # Updating the GP. self.gp.fit(self.space.X, self.space.Y) # Update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append(dict(zip(self.space.keys, x_max))) # Update maximum value to search for next probe point. if self.space.Y[-1] > y_max: y_max = self.space.Y[-1] # Maximize acquisition function to find next probing point if n_iter == 1: break x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) #-----------!!!!!!!!!---------- #if not self.space.validate_conf(x_max) : # x_max = self.space.rechoose_conf(x_max) # Keep track of total number of iterations self.i += 1 # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def newMax(self, acq='ucb', kappa=2.576, xi=0.0, **gp_params): print("into newMax") self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) self.y_max = self.space.Y.max() self.gp.set_params(**gp_params) self.gp.fit(self.space.X, self.space.Y) x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=self.y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) #如果对CPU和RAM之间的比例没有特殊需求的话,可以不用对选出的配置进行重新选择 ''' if not self.space.validate_conf(x_max) : x_max = self.space.rechoose_conf(x_max) ''' # Print new header if self.verbose: self.plog.print_header(initialization=False) pwarning = False while x_max in self.space: print("Change X_MAX") x_max = self.space.random_points(1)[0] pwarning = True # Append most recently generated values to X and Y arrays print("out newMax") return x_max def compute(self,x_max,y): # Updating the GP. self.space.observe_point(x_max,y) self.gp.fit(self.space.X, self.space.Y) # Update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append(dict(zip(self.space.keys, x_max))) # Update maximum value to search for next probe point. if self.space.Y[-1] >self. y_max: self.y_max = self.space.Y[-1] # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=self.y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) return x_max def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.space.X, np.expand_dims(self.space.Y, axis=1))) header = ', '.join(self.space.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',') # --- API compatibility --- @property def X(self): warnings.warn("use self.space.X instead", DeprecationWarning) return self.space.X @property def Y(self): warnings.warn("use self.space.Y instead", DeprecationWarning) return self.space.Y @property def keys(self): warnings.warn("use self.space.keys instead", DeprecationWarning) return self.space.keys @property def f(self): warnings.warn("use self.space.target_func instead", DeprecationWarning) return self.space.target_func @property def bounds(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.bounds @property def dim(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.dim
class BayesianOptimization(object): def __init__(self, f, pbounds, random_state=None, verbose=1): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds if random_state is None: self.random_state = np.random.RandomState() elif isinstance(random_state, int): self.random_state = np.random.RandomState(random_state) else: self.random_state = random_state # Get the name of the parameters self.keys = list(pbounds.keys()) # Find number of parameters self.dim = len(pbounds) # Create an array with parameters bounds self.bounds = [] for key in self.pbounds.keys(): self.bounds.append(self.pbounds[key]) self.bounds = np.asarray(self.bounds) # Some function to be optimized self.f = f # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Numpy array place holders self.X = None self.Y = None # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # Verbose self.verbose = verbose def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Generate random points l = [ self.random_state.uniform(x[0], x[1], size=init_points) for x in self.bounds ] # Concatenate new random points to possible existing # points from self.explore method. self.init_points += list(map(list, zip(*l))) # Create empty arrays to store the new points and values of the function. self.X = np.empty((0, self.bounds.shape[0])) self.Y = np.empty(0) # Evaluate target function at all initialization # points (random + explore) for x in self.init_points: self.X = np.vstack((self.X, np.asarray(x).reshape((1, -1)))) self.Y = np.append(self.Y, self.f(**dict(zip(self.keys, x)))) self.res['max'] = { 'max_val': self.Y.max(), 'max_params': dict(zip(self.keys, self.X[self.Y.argmax()])) } self.res['all']['values'].append(self.Y[-1]) self.res['all']['params'].append(dict(zip(self.keys, self.X[-1]))) if self.verbose: self.plog.print_step(x, self.Y[-1]) # TODO:include the following into res # Append any other points passed by the self.initialize method (these # also have a corresponding target value passed by the user). self.init_points += self.x_init self.X = np.vstack( (self.X, np.asarray(self.x_init).reshape(-1, self.X.shape[1]))) # Append the target value of self.initialize method. self.Y = np.concatenate((self.Y, self.y_init)) # Updates the flag self.initialized = True def explore(self, points_dict): """Method to explore user defined points :param points_dict: """ # Consistency check param_tup_lens = [] for key in self.keys: param_tup_lens.append(len(list(points_dict[key]))) if all([e == param_tup_lens[0] for e in param_tup_lens]): pass else: raise ValueError('The same number of initialization points ' 'must be entered for every parameter.') # Turn into list of lists all_points = [] for key in self.keys: all_points.append(points_dict[key]) # Take transpose of list self.init_points = list(map(list, zip(*all_points))) def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) # Loop through the all bounds and reset the min-max bound matrix for row, key in enumerate(self.pbounds.keys()): # Reset all entries, even if the same. self.bounds[row] = self.pbounds[key] def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Upper Confidence Bound. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing """ # Reset timer self.plog.reset_timer() # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points) if acq == 'rnd': x_max = self.random_state.uniform(self.bounds[:, 0], self.bounds[:, 1], size=self.bounds.shape[0]) else: y_max = self.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds, random_state=self.random_state) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False if np.any((self.X - x_max).sum(axis=1) == 0): x_max = self.random_state.uniform(self.bounds[:, 0], self.bounds[:, 1], size=self.bounds.shape[0]) pwarning = True # Append most recently generated values to X and Y arrays self.X = np.vstack((self.X, x_max.reshape((1, -1)))) self.Y = np.append(self.Y, self.f(**dict(zip(self.keys, x_max)))) if acq == 'rnd': x_max = self.random_state.uniform(self.bounds[:, 0], self.bounds[:, 1], size=self.bounds.shape[0]) else: # Updating the GP. ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Update maximum value to search for next probe point. if self.Y[-1] > y_max: y_max = self.Y[-1] # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds, random_state=self.random_state) # Print stuff if self.verbose: self.plog.print_step(self.X[-1], self.Y[-1], warning=pwarning) # Keep track of total number of iterations self.i += 1 self.res['max'] = { 'max_val': self.Y.max(), 'max_params': dict(zip(self.keys, self.X[self.Y.argmax()])) } self.res['all']['values'].append(self.Y[-1]) self.res['all']['params'].append(dict(zip(self.keys, self.X[-1]))) # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.X, np.expand_dims(self.Y, axis=1))) header = ', '.join(self.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',')
class SS_model_GP(SS_model_base): def __init__(self, data, x_headers, y_header, scale=30**2, RBF_scale=[2.91, 174, 12.8], noise=0.617): super().__init__(data, x_headers, y_header) self.scale = scale self.RBF_scale = RBF_scale self.noise = noise # self.RBF_scale = [[0.885, 169, 1.25], # [2.91, 174, 12.8]] self.kernel = self.scale * RBF(length_scale=self.RBF_scale, length_scale_bounds=(1e-3, 100000.0))\ + WhiteKernel(noise_level=self.noise, noise_level_bounds=(1e-10, 1e+10)) self.model = GaussianProcessRegressor(kernel=self.kernel) def update_hyperparameters(self, param): self.scale = param['param1'] self.RBF_scale = [param['param2'], param['param3'], param['param4']] self.noise = param['param5'] self.kernel = self.scale * RBF(length_scale=self.RBF_scale, length_scale_bounds=(1e-3, 100000.0))\ + WhiteKernel(noise_level=self.noise, noise_level_bounds=(1e-10, 1e+10)) self.model.set_params(kernel=self.kernel) def get_hyperparameters(self): return self.model.kernel_ def model_predict(self): self.y_pred, self.y_std = self.model.predict(self.X_pred, return_std=True) def fit(self): self.model.fit(self.X_train, self.y_train) def grid_search(self): scale = [50**2, 100**2, 300**2] RBF_scale = [[0.1, 1, 5], [100, 300], [1, 10, 50]] noise = [1e-1, 1] param_grid = { 'param1': scale, 'param2': RBF_scale[0], 'param3': RBF_scale[1], 'param4': RBF_scale[2], 'param5': noise } grid = ParameterGrid(param_grid) best_settings = [] best_test_error = 99999 num_grid_search = len(grid) for i, param in enumerate(grid): self.update_hyperparameters(param) test_error, train_error = self.train_with_cross_validation() if test_error < best_test_error: best_test_error = test_error best_settings = param print("\nsearching: " + str(i) + " / " + str(num_grid_search)) print('params: ') print(param) print(self.get_hyperparameters()) print("train_error: " + str(train_error)) print("test_error: " + str(test_error)) self.update_hyperparameters(best_settings)
class BayesianOptimization(object): def __init__(self, f, pbounds, random_state=None, verbose=1, **kwargs): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds self.random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self.space = TargetSpace(f, pbounds, random_state) # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Counter of iterations self.i = 0 #NEW FS TO IMPLEMENT DIFFERENT KERNELS kernel = kwargs.get('kernel') whiteNoise = kwargs.get('whiteNoise', 0.1) scaling = kwargs.get('scalingKer', 0.1) self.mp = kwargs.get('mp_obj', None) if (kernel is None): kernel = Matern(nu=2.5) elif (kernel[:6] == 'matern'): kernel = Matern(nu=float(kernel[6:])) elif (kernel[:3] == 'ard'): #pdb.set_trace() if (kernel[3:] == 'aniso'): init_length = np.ones(len(pbounds.keys())) init_length_bounds = [(1e-5, 1e5) for _ in init_length] kernel = ConstantKernel(constant_value=1.0) * RBF( length_scale=init_length, length_scale_bounds=init_length_bounds) else: kernel = ConstantKernel(constant_value=1.0) * RBF( length_scale=1.0) else: raise NotImplementedError() if (scaling is not None): kernel *= ConstantKernel(constant_value=scaling) if (whiteNoise is not None): kernel += WhiteKernel(float(whiteNoise)) # Internal GP regressor self.gp = GaussianProcessRegressor(kernel=kernel, n_restarts_optimizer=25, random_state=self.random_state) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.space.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # NOW public config for maximizing the aquisition function # (used to speedup tests, but generally leave these as is) #self._acqkw = {'n_warmup': 100000, 'n_iter': 250} n_warmup, n_acq_iter = kwargs.get('gp_warmup'), kwargs.get( 'gp_acq_iter') if (n_warmup is None): n_warmup = 100000 if (n_acq_iter is None): n_acq_iter = 250 self._acqkw = {'n_warmup': n_warmup, 'n_iter': n_acq_iter} # Verbose self.verbose = verbose #NewFS self._nb_ev = 0 def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ #_time = time.time() # Concatenate new random points to possible existing # points from self.explore method. rand_points = self.space.random_points(init_points) self.init_points.extend(rand_points) # Evaluate target function at all initialization points for x in self.init_points: y = self._observe_point(x) # Add the points from `self.initialize` to the observations if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) if self.verbose: self.plog.print_step(x, y) #_time_elapsed = time.time() - self._time_init #_y_max = self.space.Y[-1] #self._bestfom_fev.append([self._nb_ev, _y_max ]) #self._bestfom_time.append([_time_elapsed, _y_max]) # Updates the flag self.initialized = True def _observe_point(self, x): y = self.space.observe_point(x) if self.verbose: self.plog.print_step(x, y) self._nb_ev += 1 return y def explore(self, points_dict, eager=False): """Method to explore user defined points. :param points_dict: :param eager: if True, these points are evaulated immediately """ if eager: self.plog.reset_timer() if self.verbose: self.plog.print_header(initialization=True) points = self.space._dict_to_points(points_dict) for x in points: self._observe_point(x) else: points = self.space._dict_to_points(points_dict) self.init_points = points def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.space.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.space.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) self.space.set_bounds(new_bounds) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Upper Confidence Bound. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing Example: >>> xs = np.linspace(-2, 10, 10000) >>> f = np.exp(-(xs - 2)**2) + np.exp(-(xs - 6)**2/10) + 1/ (xs**2 + 1) >>> bo = BayesianOptimization(f=lambda x: f[int(x)], >>> pbounds={"x": (0, len(f)-1)}) >>> bo.maximize(init_points=2, n_iter=25, acq="ucb", kappa=1) """ # Reset timer self.plog.reset_timer() # Set acquisition function if (hasattr(kappa, '__call__')): kappa_init = kappa(0) elif (hasattr(kappa, '__iter__')): kappa_init = kappa[0] else: kappa_init = kappa self.util = UtilityFunction(kind=acq, kappa=kappa_init, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points) y_max = self.space.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking self.gp.fit(self.space.X, self.space.Y) if (self.mp is None): pool = None else: pool = self.mp.pool # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, pool=pool, **self._acqkw) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False while x_max in self.space: x_max = self.space.random_points(1)[0] pwarning = True # Append most recently generated values to X and Y arrays y = self.space.observe_point(x_max) if self.verbose: self.plog.print_step(x_max, y, pwarning) # Updating the GP. self.gp.fit(self.space.X, self.space.Y) # Update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append(dict(zip(self.space.keys, x_max))) # Update maximum value to search for next probe point. if self.space.Y[-1] > y_max: y_max = self.space.Y[-1] # Maximize acquisition function to find next probing point if (hasattr(kappa, '__call__')): self.util.UpdateKappa(kappa(i)) elif (hasattr(kappa, '__iter__')): self.util.UpdateKappa(kappa[i]) x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, pool=pool, **self._acqkw) # Keep track of total number of iterations self.i += 1 #New #_time_elapsed = self._time_init - time.time() #self._bestfom_fev.append([self._nb_ev, y_max ]) #self._bestfom_time.append([_time_elapsed, y_max]) # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.space.X, np.expand_dims(self.space.Y, axis=1))) header = ', '.join(self.space.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',') # --- API compatibility --- @property def X(self): warnings.warn("use self.space.X instead", DeprecationWarning) return self.space.X @property def Y(self): warnings.warn("use self.space.Y instead", DeprecationWarning) return self.space.Y @property def keys(self): warnings.warn("use self.space.keys instead", DeprecationWarning) return self.space.keys @property def f(self): warnings.warn("use self.space.target_func instead", DeprecationWarning) return self.space.target_func @property def bounds(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.bounds @property def dim(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.dim
class Optimizer(): def __init__(self, f, bounds, random_state=None, verbose=1): self.bounds = bounds self.random_state = ensure_rng(random_state) self.gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state) self.space = TargetSpace(f, bounds, self.random_state) self.initialized = False self.i = 0 # used for iterations self.util = None #utility function self._acqkw = {'n_warmup': 100000, 'n_iter': 250} self.init_points = [] self.x_init = [] self.y_init = [] self.res = {} #output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} self.verbose = verbose def init(self, init_points): #init_points is number of randomly sampled points to probe rand_points = self.space.random_points(init_points) self.init_points.extend(rand_points) for x in self.init_points: y = self.space.observe_point(x) if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) self.initialized = True def explore(self, points_dict, eager=False): if eager: points = self.space._dict_to_points(points_dict) for x in points: self.space.observe_point(x) else: points = self.space._dict_to_points(points_dict) self.init_points = points def maximize(self, init_points=5, n_iter=25, acq='ei', kappa=2.576, xi=0.0, **gp_params): '''main optimization method''' self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) if not self.initialized: self.init(init_points) y_max = self.space.getY.max() self.gp.set_params(**gp_params) self.gp.fit(self.space.getX, self.space.getY) x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, yt=self.space.getY, xt=self.space.getX, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) for i in range(n_iter): while x_max in self.space: x_max = self.space.random_points(1)[0] y = self.space.observe_point(x_max) #update the gp self.gp.fit(self.space.getX, self.space.getY) #update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append(dict(zip(self.space.keys, x_max))) if self.space.getY[-1] > y_max: y_max = self.space.getY[-1] #maximize the acquisition function to find the next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, yt=self.space.getY, xt=self.space.getX, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) self.i += 1
class BayesianOptimization(Observable): def __init__(self, f, pbounds, yrange=1, random_state=None, verbose=2, alpha=1e-6, nu=2.5, noisy=False, parameter_normalizer=0.5, parall_option=1, print_timing=False): #parameter normalizer is a parameter that decided within how many standard deviation the y limists should be, advised to be between 1 and 3 (lower is better as it avoids getting stuck due to low uncertainty where no data is avaiable) #parall_option is to chose the level of parallilazation of the optimizer: 0 is no parallelization, 1 is low level (only valid for NEI, parallel montecarlo estimation) and 2 is high level (when NEI parallel montecarlo estimation during grid search, and then swich to parallelize optimizer) """""" self._random_state = ensure_rng(random_state) self.parall_option = parall_option # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self.print_timing = print_timing # queue self._queue = Queue() if (isinstance(yrange, int)): #set an expected confidence interval _yrange = yrange else: _yrange = yrange[1] - yrange[0] self._norm_constant = _yrange / ( parameter_normalizer * 2 ) #1.96 = 95% confidence intervall (0.05 probability of getting elements outside this range) self._original_f = f self._space = TargetSpace(self.normalizedF, pbounds, random_state, norm_constant=self._norm_constant, noisy=noisy) # lambda **params:f(**params)/self._norm_constant if (not noisy): try: assert alpha < 0.0001 except AssertionError: raise ValueError( "When working with non-noisy functions use an alpha smaller then 0.00001 (used {} instead) do " .format(alpha)) else: #normalize GP noise with respect to the expecter y range. alpha = alpha / self._norm_constant # Internal GP regressor self._gp = GaussianProcessRegressor( kernel=augKernel(nu=nu, discrete=self._space._discrete), alpha=alpha, normalize_y=True, n_restarts_optimizer=5, random_state=self._random_state, ) self._verbose = verbose super(BayesianOptimization, self).__init__(events=DEFAULT_EVENTS) def normalizedF(self, **params): return self._original_f(**params) / self._norm_constant @property def space(self): return self._space @property def max(self): mx = self._space.max() mx['target'] = mx['target'] * self._norm_constant return mx @property def res(self): return self._space.res() def register(self, params, target): """Expect observation with known target""" self._space.register(params, target) self.dispatch(Events.OPTMIZATION_STEP) def probe(self, params, lazy=True): """Probe target of x""" if lazy: self._queue.add(params) else: self._space.probe(params) self.dispatch(Events.OPTMIZATION_STEP) def suggest(self, utility_function, optimizer_best_trials, optimizer_random_trials, optimizer_n_warmups): """Most promissing point to probe next""" if len(self._space) == 0: return self._space.array_to_params(self._space.random_sample()) # Sklearn's GP throws a large number of warnings at times, but # we don't really need to see them here. with warnings.catch_warnings(): warnings.simplefilter("ignore") self._gp.fit(self._space.params, self._space.target) # Finding argmax of the acquisition function. acquisitor = Acquisitor(ac=utility_function.utility, opt=self, random_state=self._random_state) if (self.parall_option == 2): tp2 = time() suggestion = acquisitor.parall_acq_max( n_best_iter=optimizer_best_trials, n_rand_iter=optimizer_random_trials, n_warmup=optimizer_n_warmups) tp2 = time() - tp2 if (self.print_timing): print(tp2) else: t = time() if (self.parall_option == 0): suggestion = acquisitor.acq_max( low_level_parall=False, n_best_iter=optimizer_best_trials, n_rand_iter=optimizer_random_trials, n_warmup=optimizer_n_warmups) elif (self.parall_option == 1): suggestion = acquisitor.acq_max( low_level_parall=True, n_best_iter=optimizer_best_trials, n_rand_iter=optimizer_random_trials, n_warmup=optimizer_n_warmups) else: raise ValueError( "Parallelization option should be 0 (none) 1 (low level) or 2 (high level) (used {} instead) do " .format(self.parall_option)) t = time() - t if (self.print_timing): print(t) return suggestion def _prime_queue(self, init_points): """Make sure there's something in the queue at the very beginning.""" if self._queue.empty and self._space.empty: init_points = max(init_points, 1) for _ in range(init_points): self._queue.add(self._space.random_sample()) def _prime_subscriptions(self): if not any([len(subs) for subs in self._events.values()]): _logger = _get_default_logger(self._verbose) self.subscribe(Events.OPTMIZATION_START, _logger) self.subscribe(Events.OPTMIZATION_STEP, _logger) self.subscribe(Events.OPTMIZATION_END, _logger) def maximize(self, init_points=5, n_iter=25, acq='ei', kappa=2.576, xi=0.0, N_QMC=None, optimizer_best_trials=4, optimizer_random_trials=10, optimizer_n_warmups=10000, **gp_params): """Mazimize your function inputs: xi = offset of the expected improvement. would suggest not to tuch. kappa = decides the ratio of mean to standard deviation for UCB N_QMC = for NEI, decides the number of iterations of monte carlo simulation (higher has better precision but will take longer) optimizer_best_trials = how many trials to let the optimizer from the best results of the grid search optimizer_random_trials = how many trials to let the optimizer from randomly selected points """ self._prime_subscriptions() self.dispatch(Events.OPTMIZATION_START) self._prime_queue(init_points) self.set_gp_params(**gp_params) # if(self.space.noisy): # try: # assert acq == 'nei' # except AssertionError: # raise ValueError( # "If working with a noisy function, please use 'nei' acquisition (using {} instead) do ".format(acq) # ) util = UtilityFunction(kind=acq, kappa=kappa, xi=xi, N_QMC=N_QMC) self.util = util iteration = 0 while not self._queue.empty or iteration < n_iter: try: x_probe = next(self._queue) except StopIteration: x_probe = self.suggest(util, optimizer_best_trials, optimizer_random_trials, optimizer_n_warmups) iteration += 1 self.probe(x_probe, lazy=False) self.dispatch(Events.OPTMIZATION_END) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds Parameters ---------- new_bounds : dict A dictionary with the parameter name and its new bounds """ self._space.set_bounds(new_bounds) def set_gp_params(self, **params): self._gp.set_params(**params)
class BayesianOptimization(Observable): def __init__(self, f, pbounds, random_state=None, verbose=2, constraints=[]): """""" self._random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self._space = TargetSpace(f, pbounds, random_state) # queue self._queue = Queue() # Internal GP regressor self._gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), alpha=1e-6, normalize_y=True, n_restarts_optimizer=25, random_state=self._random_state, ) self._verbose = verbose # Key constraints correspond to literal keyword names # array constraints correspond to point in array row self._key_constraints = constraints self._array_constraints = self.array_like_constraints() super(BayesianOptimization, self).__init__(events=DEFAULT_EVENTS) @property def space(self): return self._space @property def max(self): return self._space.max() @property def res(self): return self._space.res() @property def constraints(self): return self._array_constraints @property def verbose(self): return self._verbose def register(self, params, target): """Expect observation with known target""" self._space.register(params, target) self.dispatch(Events.OPTMIZATION_STEP) def probe(self, params, lazy=True): """Probe target of x""" if isinstance(params, list): for param in params: if lazy: self._queue.add(param) else: self._space.probe(param) self.dispatch(Events.OPTMIZATION_STEP) else: if lazy: self._queue.add(params) else: self._space.probe(params) self.dispatch(Events.OPTMIZATION_STEP) def suggest(self, utility_function): """Most promissing point to probe next""" if len(self._space) == 0: return self._space.array_to_params(self._space.random_sample()) # Sklearn's GP throws a large number of warnings at times, but # we don't really need to see them here. with warnings.catch_warnings(): warnings.simplefilter("ignore") self._gp.fit(self._space.params, self._space.target) # Finding argmax of the acquisition function. suggestion = acq_max(ac=utility_function.utility, gp=self._gp, y_max=self._space.target.max(), bounds=self._space.bounds, random_state=self._random_state) return self._space.array_to_params(suggestion) def reset_rng(self, random_state=None): self._random_state = ensure_rng(random_state) def _prime_queue(self, init_points): """Make sure there's something in the queue at the very beginning.""" if self._queue.empty and self._space.empty: init_points = max(init_points, 1) for _ in range(init_points): self._queue.add(self._space.random_sample()) def _prime_subscriptions(self): if not any([len(subs) for subs in self._events.values()]): _logger = _get_default_logger(self._verbose) self.subscribe(Events.OPTMIZATION_START, _logger) self.subscribe(Events.OPTMIZATION_STEP, _logger) self.subscribe(Events.OPTMIZATION_END, _logger) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): """Mazimize your function""" self._prime_subscriptions() self.dispatch(Events.OPTMIZATION_START) self._prime_queue(init_points) self.set_gp_params(**gp_params) util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) iteration = 0 while not self._queue.empty or iteration < n_iter: try: x_probe = next(self._queue) except StopIteration: x_probe = self.suggest(util) iteration += 1 self.probe(x_probe, lazy=False) self.dispatch(Events.OPTMIZATION_END) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds Parameters ---------- new_bounds : dict A dictionary with the parameter name and its new bounds """ self._space.set_bounds(new_bounds) def set_gp_params(self, **params): self._gp.set_params(**params) def array_like_constraints(self): ''' Takes list of logical constraints in terms of space keys, and replaces the constraints in terms of array indicies. This allows direct evaluation in the acquisition function. Parameters ---------- constraints: list of string constraints ''' keys = self.space.keys array_like = [] for constraint in self._key_constraints: tmp = constraint for idx, key in enumerate(keys): # tmp = tmp.replace(key,'x[0][{}]'.format(idx)) tmp = tmp.replace(key, 'x[{}]'.format(idx)) array_like.append(tmp) return array_like def get_constraint_dict(self): ''' Develops inequality constraints ONLY. (>=0) ''' dicts = [] funcs = [] for idx, constraint in enumerate(self.constraints): st = "def f_{}(x): return pd.eval({})\nfuncs.append(f_{})".format( idx, constraint, idx) exec(st) dicts.append({'type': 'ineq', 'fun': funcs[idx]}) return dicts def output_space(self, path): """ Outputs complete space as csv file. Simple function for testing Parameters ---------- path Returns ------- """ df = pd.DataFrame(data=self.space.params, columns=self.space.keys) df['Target'] = self.space.target df.to_csv(path)
class BayesianOptimization(Observable): """ This class takes the function to optimize as well as the parameters bounds in order to find which values for the parameters yield the maximum value using bayesian optimization. Parameters ---------- f: function Function to be maximized. pbounds: dict Dictionary with parameters names as keys and a tuple with minimum and maximum values. random_state: int or numpy.random.RandomState, optional(default=None) If the value is an integer, it is used as the seed for creating a numpy.random.RandomState. Otherwise the random state provieded it is used. When set to None, an unseeded random state is generated. verbose: int, optional(default=2) The level of verbosity. bounds_transformer: DomainTransformer, optional(default=None) If provided, the transformation is applied to the bounds. Methods ------- probe() Evaluates the function on the given points. Can be used to guide the optimizer. maximize() Tries to find the parameters that yield the maximum value for the given function. set_bounds() Allows changing the lower and upper searching bounds """ def __init__(self, f, pbounds, random_state=None, verbose=2, bounds_transformer=None): self._random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self._space = TargetSpace(f, pbounds, random_state) self._queue = Queue() # Internal GP regressor self._gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), alpha=1e-6, normalize_y=True, n_restarts_optimizer=5, random_state=self._random_state, ) self._verbose = verbose self._bounds_transformer = bounds_transformer if self._bounds_transformer: try: self._bounds_transformer.initialize(self._space) except (AttributeError, TypeError): raise TypeError('The transformer must be an instance of ' 'DomainTransformer') super(BayesianOptimization, self).__init__(events=DEFAULT_EVENTS) @property def space(self): return self._space @property def max(self): return self._space.max() @property def res(self): return self._space.res() def register(self, params, target): """Expect observation with known target""" self._space.register(params, target) self.dispatch(Events.OPTIMIZATION_STEP) def probe(self, params, lazy=True): """ Evaluates the function on the given points. Useful to guide the optimizer. Parameters ---------- params: dict or list The parameters where the optimizer will evaluate the function. lazy: bool, optional(default=True) If True, the optimizer will evaluate the points when calling maximize(). Otherwise it will evaluate it at the moment. """ if lazy: self._queue.add(params) else: self._space.probe(params) self.dispatch(Events.OPTIMIZATION_STEP) def suggest(self, utility_function): """Most promising point to probe next""" if len(self._space) == 0: return self._space.array_to_params(self._space.random_sample()) # Sklearn's GP throws a large number of warnings at times, but # we don't really need to see them here. with warnings.catch_warnings(): warnings.simplefilter("ignore") self._gp.fit(self._space.params, self._space.target) # Finding argmax of the acquisition function. suggestion = acq_max( ac=utility_function.utility, gp=self._gp, y_max=self._space.target.max(), bounds=self._space.bounds, random_state=self._random_state ) return self._space.array_to_params(suggestion) def _prime_queue(self, init_points): """Make sure there's something in the queue at the very beginning.""" if self._queue.empty and self._space.empty: init_points = max(init_points, 1) for _ in range(init_points): self._queue.add(self._space.random_sample()) def _prime_subscriptions(self): if not any([len(subs) for subs in self._events.values()]): _logger = _get_default_logger(self._verbose) self.subscribe(Events.OPTIMIZATION_START, _logger) self.subscribe(Events.OPTIMIZATION_STEP, _logger) self.subscribe(Events.OPTIMIZATION_END, _logger) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, kappa_decay=1, kappa_decay_delay=0, xi=0.0, **gp_params): """ Probes the target space to find the parameters that yield the maximum value for the given function. Parameters ---------- init_points : int, optional(default=5) Number of iterations before the explorations starts the exploration for the maximum. n_iter: int, optional(default=25) Number of iterations where the method attempts to find the maximum value. acq: {'ucb', 'ei', 'poi'} The acquisition method used. * 'ucb' stands for the Upper Confidence Bounds method * 'ei' is the Expected Improvement method * 'poi' is the Probability Of Improvement criterion. kappa: float, optional(default=2.576) Parameter to indicate how closed are the next parameters sampled. Higher value = favors spaces that are least explored. Lower value = favors spaces where the regression function is the highest. kappa_decay: float, optional(default=1) `kappa` is multiplied by this factor every iteration. kappa_decay_delay: int, optional(default=0) Number of iterations that must have passed before applying the decay to `kappa`. xi: float, optional(default=0.0) [unused] """ self._prime_subscriptions() self.dispatch(Events.OPTIMIZATION_START) self._prime_queue(init_points) self.set_gp_params(**gp_params) util = UtilityFunction(kind=acq, kappa=kappa, xi=xi, kappa_decay=kappa_decay, kappa_decay_delay=kappa_decay_delay) iteration = 0 while not self._queue.empty or iteration < n_iter: try: x_probe = next(self._queue) except StopIteration: util.update_params() x_probe = self.suggest(util) iteration += 1 self.probe(x_probe, lazy=False) if self._bounds_transformer: self.set_bounds( self._bounds_transformer.transform(self._space)) self.dispatch(Events.OPTIMIZATION_END) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds Parameters ---------- new_bounds : dict A dictionary with the parameter name and its new bounds """ self._space.set_bounds(new_bounds) def set_gp_params(self, **params): """Set parameters to the internal Gaussian Process Regressor""" self._gp.set_params(**params)
class BayesianOptimization(object): def __init__(self, f, pbounds, random_state=None): self.pbounds = pbounds self.random_state = ensure_rng(random_state) self.gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state) self.init_points = [] self.space = TargetSpace(f, pbounds, random_state) self.x_init = [] self.y_init = [] self.initialized = False self._acqkw = {'n_warmup': 100000, 'n_iter': 250} def init(self, init_points): #init points rand_points = self.space.random_points(init_points) self.init_points.extend(rand_points) #evaluate target function at all init points for x in self.init_points: y = self._observe_point(x) #add the points to the obsevations if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) self.initialized = True def _observe_point(self, x): y = self.space.observe_point(x) return y def initialize(self, points_dict): self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.space.keys: all_points.append(points_dict[key][i]) self.x_init_append(all_points) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): #get acquisition function self.util = AcquisitionFunction(kind=acq, kappa=kappa, xi=xi) #initialize if not self.initialized: self.init(init_points) y_max = self.space.Y.max() #set gp parameters self.gp.set_params(**gp_params) #gaussian process fit self.gp.fit(self.space.X, self.space.Y) #find argmax of the acquisition function x_max = acq_max(ac=self.util.acqf, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) #Iterative process for i in range(n_iter): #Append most recently generated values to X and Y arrays y = self.space.observe_point(x_max) #updatging the Gp self.gp.fit(self.space.X, self.space.Y) #update maximum value to search for next probe point if self.space.Y[-1] > y_max: y_max = self.space.Y[-1] #Maximum acquasition function to find next probing point x_max = acq_max(ac=self.util.acqf, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) @property def X(self): return self.space.X @property def Y(self): return self.space.Y
class BayesianOptimization(Observable): def __init__(self, f, pbounds, ptypes=None, random_state=None, verbose=2): """""" self._random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self._space = TargetSpace(f, pbounds, ptypes, random_state) # queue self._queue = Queue() # Internal GP regressor self._gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), alpha=1e-6, normalize_y=True, n_restarts_optimizer=5, random_state=self._random_state, ) self._verbose = verbose super(BayesianOptimization, self).__init__(events=DEFAULT_EVENTS) @property def space(self): return self._space @property def max(self): return self._space.max() @property def res(self): return self._space.res() def register(self, params, target): """Expect observation with known target""" self._space.register(params, target) self.dispatch(Events.OPTMIZATION_STEP) def probe(self, params, lazy=True): """Probe target of x""" if lazy: self._queue.add(params) else: self._space.probe(params) self.dispatch(Events.OPTMIZATION_STEP) def suggest(self, utility_function): """Most promissing point to probe next""" if len(self._space) == 0: return self._space.array_to_params(self._space.random_sample()) # Sklearn's GP throws a large number of warnings at times, but # we don't really need to see them here. with warnings.catch_warnings(): warnings.simplefilter("ignore") self._gp.fit(self._space.params, self._space.target) # Finding argmax of the acquisition function. suggestion = acq_max( ac=utility_function.utility, gp=self._gp, y_max=self._space.target.max(), bounds=self._space.bounds, btypes=self._space.btypes, random_state=self._random_state ) return self._space.array_to_params(suggestion) def _prime_queue(self, init_points): """Make sure there's something in the queue at the very beginning.""" if self._queue.empty and self._space.empty: init_points = max(init_points, 1) for _ in range(init_points): self._queue.add(self._space.random_sample()) def _prime_subscriptions(self): if not any([len(subs) for subs in self._events.values()]): _logger = _get_default_logger(self._verbose) self.subscribe(Events.OPTMIZATION_START, _logger) self.subscribe(Events.OPTMIZATION_STEP, _logger) self.subscribe(Events.OPTMIZATION_END, _logger) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): """Mazimize your function""" self._prime_subscriptions() self.dispatch(Events.OPTMIZATION_START) self._prime_queue(init_points) self.set_gp_params(**gp_params) util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) iteration = 0 while not self._queue.empty or iteration < n_iter: try: x_probe = next(self._queue) except StopIteration: x_probe = self.suggest(util) iteration += 1 self.probe(x_probe, lazy=False) self.dispatch(Events.OPTMIZATION_END) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds Parameters ---------- new_bounds : dict A dictionary with the parameter name and its new bounds """ self._space.set_bounds(new_bounds) def set_gp_params(self, **params): self._gp.set_params(**params)
class BayesianOptimization(object): def __init__(self, f, pbounds, random_state=None, verbose=1, if_cost=False, cost_function=None, cost_max=0.0): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. :param cost_function: The cost function of f, proportional to the (appox) running time of f, should have exact same parameter list with f. This function should fairly cheap to evaluate. :param cost_init: init points cost, if neccessary. """ # Store the original dictionary self.pbounds = pbounds self.random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self.space = TargetSpace(f, pbounds, random_state) # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state, ) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.space.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # non-public config for maximizing the aquisition function # (used to speedup tests, but generally leave these as is) self._acqkw = {'n_warmup': 100000, 'n_iter': 250} # Verbose self.verbose = verbose # Cost function self.if_cost = if_cost if cost_function is not None: assert (inspect.getfullargspec(f).args == inspect.getfullargspec( cost_function).args) self.cost_function = cost_function self.cost_init = 0.0 self.cost_max = cost_max self.cost_acc = 0.0 def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Concatenate new random points to possible existing # points from self.explore method. rand_points = self.space.random_points(init_points) self.init_with_params(rand_points) def init_with_params(self, params): def cost(x): if self.cost_function is None: return 0.0 x_ = np.asarray(x).ravel() params = dict(zip(self.space.keys, x_)) return self.cost_function(**params) self.init_points.extend(params) # Evaluate target function at all initialization points for x in self.init_points: y = self._observe_point(x) self.cost_init += cost(x) self.cost_acc = self.cost_init # Add the points from `self.initialize` to the observations if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) if self.verbose: self.plog.print_step(x, y) # Updates the flag self.initialized = True def _observe_point(self, x): y = self.space.observe_point(x) if self.verbose: self.plog.print_step(x, y) return y def explore(self, points_dict, eager=False): """Method to explore user defined points. :param points_dict: :param eager: if True, these points are evaulated immediately """ if eager: self.plog.reset_timer() if self.verbose: self.plog.print_header(initialization=True) points = self.space._dict_to_points(points_dict) for x in points: self._observe_point(x) else: points = self.space._dict_to_points(points_dict) self.init_points = points def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.space.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.space.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) self.space.set_bounds(new_bounds) def maximize(self, init_points=5, n_iter=25, acq='ei', kappa=2.576, xi=0.0, n_jobs=1, acq_type='others', threshhold=0.1, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Expected Improvement. :param n_jobs: Maximum experiments on f that we can run in parallel. Main optimization function with parallel evaluation enabled when n_jobs > 1. This method enables parallel evaluation via batch Bayesian Optimization, which picks multiple samples per iteration. The target function f should be thread-safe to use this parallel method. Here're some methods available for picking samples according to the acquisition function: - q-EI method - Iterative maximize method :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object :param acq_type: bo: nomal EI, others:prefer lower cost when similar EI :param threshhold: when acq_type!=bo Returns ------- :return: Nothing Example: >>> xs = np.linspace(-2, 10, 10000) >>> f = np.exp(-(xs - 2)**2) + np.exp(-(xs - 6)**2/10) + 1/ (xs**2 + 1) >>> bo = BayesianOptimization(f=lambda x: f[int(x)], >>> pbounds={"x": (0, len(f)-1)}) >>> bo.maximize(init_points=2, n_iter=25, acq="ucb", kappa=1) """ # Reset timer self.plog.reset_timer() # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points) y_max = self.space.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking self.gp.fit(self.space.X, self.space.Y) # self.candidates = [] # Finding argmax of the acquisition function. def ac(x, gp, y_max): x_ = np.asarray(x).ravel() params = dict(zip(self.space.keys, x_)) #print("(1){}\n(2){}\n".format(x_,self.util.utility(x, gp, y_max))) #print("{}\n".format(self.util.utility(x, gp, y_max) / ( # self.cost_function(**params) if self.cost_function is not None else 1))) if self.if_cost is False or acq_type == 'switch': return self.util.utility(x, gp, y_max) return self.util.utility(x, gp, y_max) / (self.cost_function( **params) if acq_type == 'div' else self.cost_function( **params)**((self.cost_max - self.cost_acc) / (self.cost_max - self.cost_init))) #return self.util.utility(x, gp, y_max) def cost(x): x_ = np.asarray(x).ravel() params = dict(zip(self.space.keys, x_)) return self.cost_function(**params) x_max = acq_max(ac=ac, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, alpha=(self.cost_max - self.cost_acc) / (self.cost_max - self.cost_init), cost_func=(cost if self.cost_function is not None and acq_type == 'switch' else None), **self._acqkw) print("acc:{}, init:{}\n".format(self.cost_acc, self.cost_init)) if self.cost_function is not None: self.cost_acc += cost(x_max) #print("x_max_before:{}\n".format(x_max)) #print("x_max_after:{}\n".format(x_max)) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False while x_max in self.space: x_max = self.space.random_points(1)[0] pwarning = True # Append most recently generated values to X and Y arrays y = self.space.observe_point(x_max) if self.verbose: self.plog.print_step(x_max, y, pwarning) # Updating the GP. self.gp.fit(self.space.X, self.space.Y) # Update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append(dict(zip(self.space.keys, x_max))) # Update maximum value to search for next probe point. if self.space.Y[-1] > y_max: y_max = self.space.Y[-1] self.candidates = [] # Maximize acquisition function to find next probing point x_max = acq_max(ac=ac, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, alpha=(self.cost_max - self.cost_acc) / (self.cost_max - self.cost_init), cost_func=(cost if self.cost_function is not None and acq_type == 'switch' else None), **self._acqkw) if self.cost_function is not None: self.cost_acc += cost(x_max) print("acc:{}\n".format(self.cost_acc)) if self.cost_acc >= self.cost_max: break #print("x_max_before:{}\n".format(x_max)) #print("x_max_after:{}\n".format(x_max)) # Keep track of total number of iterations self.i += 1 # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.space.X, np.expand_dims(self.space.Y, axis=1))) header = ','.join(self.space.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',', comments='') # --- API compatibility --- @property def X(self): warnings.warn("use self.space.X instead", DeprecationWarning) return self.space.X @property def Y(self): warnings.warn("use self.space.Y instead", DeprecationWarning) return self.space.Y @property def keys(self): warnings.warn("use self.space.keys instead", DeprecationWarning) return self.space.keys @property def f(self): warnings.warn("use self.space.target_func instead", DeprecationWarning) return self.space.target_func @property def bounds(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.bounds @property def dim(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.dim
def envelope_gp_cnn_theta(sources, data, target, target_points_to_start, bounds, loss,\ search_grid, n_restarts=10, number_of_iterations=100, sigma_msr=1e-10, forget=100,\ forget_upd=0.8, tao=1e-2, nu=1, gamma=None,\ lr_bandit_weights=None, strategy = 'exp3'): forget_theta_reward_coef = 0.8 net_bias = 0 forget_grid = 10.**np.arange(-2,2) policy_shape = forget_grid.shape forget_grid = forget_grid.ravel() net_dim = forget_grid.shape[0] net = nn.Sequential(nn.Linear(net_dim,net_dim), nn.Tanh()) net_const_ = torch.Tensor([1]*forget_grid.shape[0])[np.newaxis, np.newaxis, :] net_h_ = None forget_= 0 optimizer = torch.optim.Adam(net.parameters(), lr=0.5*1e-2) target_data = target_points_to_start.copy() #envelope_gp sigma_s = [sigma_msr] + [nu/(tao+1)]*len(sources) # добавим пустой источник sources_ = [lambda data: None] + sources # добавим пустой источник gp = GaussianProcessRegressor() gps = GaussianProcessRegressor() #MAB log_weights = np.array([0.0] * len(sources_)) history = [] dim = data.shape[1] K = len(sources_) #theorem 1 exp3_ix if gamma is None: if strategy == 'exp3-IX': gamma = np.sqrt(2*np.log(K)/(K*number_of_iterations)) if strategy == 'exp3-auer': gamma = 0.9 if strategy == 'exp3': gamma = 0 if lr_bandit_weights is None: if strategy == 'exp3-IX': lr_bandit_weights = 2*gamma if strategy == 'exp3-auer': lr_bandit_weights = gamma if strategy == 'exp3': lr_bandit_weights = np.sqrt(2*np.log(K)/(K*number_of_iterations)) #main for _ in tqdm_notebook(range(number_of_iterations), leave=False): #distribution update corrected_weights = log_weights - np.max(log_weights) theSum = logsumexp(corrected_weights) if strategy == 'exp3-auer': probabilityDistribution = \ (1.0 - gamma) * np.exp(corrected_weights - theSum) + (gamma / log_weights.shape[0]) elif strategy == 'exp3-IX' or strategy == 'exp3': probabilityDistribution = np.exp(corrected_weights - theSum) #draw arm = algo.draw(probabilityDistribution) alpha=np.vstack((sigma_s[arm]*np.ones((data.shape[0],1)),\ sigma_msr*np.ones((target_data.shape[0],1)))).ravel() if arm != 0: gp.set_params(alpha=alpha) y = np.vstack((sources_[arm](data)[:,np.newaxis], target(target_data)[:,np.newaxis])).ravel() gp.fit(np.vstack((data, target_data)), y) gps.fit(data, sources_[arm](data)) else: gp.set_params(alpha=sigma_msr) y = target(target_data) gp.fit(target_data, y) gps = deepcopy(gp) policy = nn.Softmax(0)(net(net_const_).reshape(-1)).reshape(-1) m = Categorical(policy) forget_ = m.sample() log_policy = m.log_prob(forget_) forget_ = forget_grid[forget_.tolist()] expected_improvement = algo.get_gp_ucb_simple_sklearn(search_grid, gp, forget_) min_val= -np.max(expected_improvement) new_point = search_grid[np.argmax(expected_improvement)] for x0 in np.random.uniform(bounds[:, 0], bounds[:, 1], size=(n_restarts, dim)): min_obj = lambda x: -algo.get_gp_ucb_simple_sklearn([x], gp, forget_) res = minimize(min_obj, x0, method='L-BFGS-B', bounds=bounds) if res.fun < min_val: min_val = res.fun new_point = res.x theReward = -loss(gps.predict(new_point[np.newaxis,:]),target(new_point))**2 #net to control theta optimize step net_reward = -(1 - target(new_point))**2 if isinstance(net_reward, np.ndarray): net_reward = net_reward[0] net_loss = (net_reward - net_bias) * (-log_policy) net_bias = (net_bias+net_reward)*forget_theta_reward_coef optimizer.zero_grad() net_loss.backward() optimizer.step() history += [(arm, probabilityDistribution, theReward, policy.reshape(policy_shape), net_reward, net_bias, forget_)] log_weights[arm] += theReward * lr_bandit_weights / \ (probabilityDistribution[arm] + gamma) target_data = np.vstack((target_data, np.array([new_point]))) tao += 0.5 nu += ((target(new_point) - gps.predict(new_point[np.newaxis,:]))**2)/2 sigma_s[arm] = nu/(tao + 1) return target_data, history, gp
def maximize(self, init_points=5, restarts=50, n_iter=25, acq='ei', **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param restarts: The number of times minimation if to be repeated. Larger number of restarts improves the chances of finding the true maxima. :param n_iter: Total number of times the process is to reapeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Expected Improvement. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing """ # Start a timer total_time = datetime.now() # Create instance of printer object printI = PrintInfo(self.verbose) # Set acquisition function AC = AcquisitionFunction() ac_types = {'ei': AC.EI, 'pi': AC.PoI, 'ucb': AC.UCB} ac = ac_types[acq] # Initialize x, y and find current ymax if not self.initialized: self.init(init_points) ymax = self.Y.max() # ------------------------------ // ------------------------------ // ------------------------------ # # Fitting the gaussian process. # Since scipy 0.16 passing lower and upper bound to theta seems to be # broken. However, there is a lot of development going on around GP # is scikit-learn. So I'll pick the easy route here and simple specify # only theta0. gp = GP( ) # TODO: arguments not accepted in scikit-learn v 0.21.2 (theta0=numpy.random.uniform(0.001, 0.05, self.dim), random_start=25) gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking ur = unique_rows(self.X) gp.fit(self.X[ur], self.Y[ur]) # Finding argmax of the acquisition function. x_max = acq_max(ac, gp, ymax, restarts, self.bounds) # ------------------------------ // ------------------------------ // ------------------------------ # # Iterative process of searching for the maximum. At each round the most recent x and y values # probed are added to the X and Y arrays used to train the Gaussian Process. Next the maximum # known value of the target function is found and passed to the acq_max function. The arg_max # of the acquisition function is found and this will be the next probed value of the tharget # function in the next round. for i in range(n_iter): op_start = datetime.now() # Append most recently generated values to X and Y arrays self.X = numpy.concatenate((self.X, x_max.reshape((1, self.dim))), axis=0) self.Y = numpy.append(self.Y, self.f(**dict(zip(self.keys, x_max)))) # Updating the GP. ur = unique_rows(self.X) gp.fit(self.X[ur], self.Y[ur]) # Update maximum value to search for next probe point. if self.Y[-1] > ymax: ymax = self.Y[-1] # Maximize acquisition function to find next probing point x_max = acq_max(ac, gp, ymax, restarts, self.bounds) # Print stuff printI.print_info(op_start, i, x_max, ymax, self.X, self.Y, self.keys) # ------------------------------ // ------------------------------ // ------------------------------ # # Output dictionary self.res = {} self.res['max'] = { 'max_val': self.Y.max(), 'max_params': dict(zip(self.keys, self.X[self.Y.argmax()])) } self.res['all'] = {'values': [], 'params': []} # Fill values for t, p in zip(self.Y, self.X): self.res['all']['values'].append(t) self.res['all']['params'].append(dict(zip(self.keys, p))) # Print a final report if verbose active. if self.verbose: tmin, tsec = divmod((datetime.now() - total_time).total_seconds(), 60) print('Optimization finished with maximum: %8f, at position: %8s.' % (self.res['max']['max_val'],\ self.res['max']['max_params'])) print('Time taken: %i minutes and %s seconds.' % (tmin, tsec))
class BayesianOptimization(object): def __init__(self, parameter_bounds, gp_kernel=Matern(), verbose=1): """ :param parameter_bounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.parameter_bounds = parameter_bounds # Get the name of the parameters self.keys = list(parameter_bounds.keys()) # Find number of parameters self.dim = len(parameter_bounds) # Create an array with parameters bounds self.bounds = [] for key in self.parameter_bounds.keys(): self.bounds.append(self.parameter_bounds[key]) self.bounds = np.asarray(self.bounds) # Initialization flag self.initialized = False self.hasSetup = False # Numpy array placeholders self.X = None self.Y = None # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor( kernel=gp_kernel, n_restarts_optimizer=25, ) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.keys) # Output dictionary self.res = { 'max': { 'max_val': None, 'max_params': None }, 'all': { 'values': [], 'params': [] } } # Output dictionary # Verbose self.verbose = verbose self.selected_groups_scores = [] self.average_batch_scores = [] def initialize(self, points): """ Method to introduce labelled points :param points: np.array with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 label must be in column 0 :return: """ if not self.hasSetup: raise RuntimeError("BO has not been set up yet.") # print("ORIGINAL: ", points) self.X = np.delete(points, 0, 1) self.Y = points[0] # print("X: ", self.X) # print("Y: ", self.Y) # Update GP with unique rows of X to prevent GP from breaking unique_rows = get_unique_rows(self.X) self.gp.fit(self.X[unique_rows - 1], self.Y[unique_rows - 1]) self.initialized = True def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.parameter_bounds.update(new_bounds) # Loop through the all bounds and reset the min-max bound matrix for row, key in enumerate(self.parameter_bounds.keys()): # Reset all entries, even if the same. self.bounds[row] = self.parameter_bounds[key] def setup(self, acq='poi', kappa=2.576, xi=0.0, **gp_params): """ Customize BO :param acq: Acquisition function to be used, defaults to Probability of Improvement. :param kappa: For Upper Confidence Bound :param xi: For Expected Improvement and Probability of Improvement :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object :return: """ self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Set parameters if any was passed self.gp.set_params(**gp_params) self.hasSetup = True def minimize(self, n_batches): """ Main optimization method. :param n_batches: np.array [batch][group][sample][feature] :return: Nothing """ n_iter = len(n_batches) # Reset timer self.plog.reset_timer() y_max = self.Y.max() # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Find argmax of the acquisition function. results = acq_max(ac=self.util.utility, gp=self.gp, groups=n_batches[i], y_max=y_max, bounds=self.bounds) selected_group, selected_group_score, average_batch_score = results self.selected_groups_scores.append(selected_group_score) self.average_batch_scores.append(average_batch_score) # Append most recently generated values to X and Y arrays new_Xs = selected_group[:, 1:] new_Ys = selected_group[:, 0] self.X = np.vstack((self.X, new_Xs)) self.Y = np.append(self.Y, new_Ys) # Update GP unique_rows = get_unique_rows(self.X) - 1 self.gp.fit(self.X[unique_rows], self.Y[unique_rows]) # Update maximum value to search for next probe point. highest_new_y = new_Ys.max() if highest_new_y > y_max: y_max = highest_new_y # Print stuff if self.verbose: for j in range(len(new_Ys)): self.plog.print_step(new_Xs[j], new_Ys[j]) # Keep track of total number of iterations self.i += 1 self.res['max'] = { 'max_val': self.Y.max(), 'max_params': dict(zip(self.keys, self.X[self.Y.argmax()])) } self.res['all']['values'].extend(new_Ys) for j in range(len(new_Xs)): self.res['all']['params'].append( dict(zip(self.keys, new_Xs[j]))) # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of file where points will be saved in csv format :return: None """ points = np.hstack((self.X, np.expand_dims(self.Y, axis=1))) header = ', '.join(self.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',')
class BayesianOptimization(object): def __init__(self, f, pbounds, random_state=None, verbose=1): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds self.random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self.space = TargetSpace(f, pbounds, random_state) # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor(kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.space.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # non-public config for maximizing the aquisition function # (used to speedup tests, but generally leave these as is) self._acqkw = {'n_warmup': 100000, 'n_iter': 250} # Verbose self.verbose = verbose def init(self, init_points, fixed_params=None): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Concatenate new random points to possible existing # points from self.explore method. bounds = self.space.bounds if fixed_params is None: fixed_params = np.nan * np.zeros(len(bounds), dtype=float) inds_fixed = np.where(~np.isnan(fixed_params))[0] def fix_full(x): if len(x) == 0: return x x = x.reshape((x.shape[0], -1)) if len(fixed_params) != 0: x[:, inds_fixed] = fixed_params[inds_fixed] return x rand_points = fix_full(self.space.random_points(init_points)) self.init_points.extend(rand_points) # Evaluate target function at all initialization points for x in self.init_points: y = self._observe_point(x) # Add the points from `self.initialize` to the observations if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) if self.verbose: self.plog.print_step(x, y) # Updates the flag self.initialized = True def _observe_point(self, x): y = self.space.observe_point(x) if self.verbose: self.plog.print_step(x, y) return y def explore(self, points_dict, eager=False): """Method to explore user defined points. :param points_dict: :param eager: if True, these points are evaulated immediately """ if eager: self.plog.reset_timer() if self.verbose: self.plog.print_header(initialization=True) points = self.space._dict_to_points(points_dict) for x in points: self._observe_point(x) else: points = self.space._dict_to_points(points_dict) self.init_points = points def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.space.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.space.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) self.space.set_bounds(new_bounds) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, fit=True, update=True, fixed_params=None, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Upper Confidence Bound. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing Example: >>> xs = np.linspace(-2, 10, 10000) >>> f = np.exp(-(xs - 2)**2) + np.exp(-(xs - 6)**2/10) + 1/ (xs**2 + 1) >>> bo = BayesianOptimization(f=lambda x: f[int(x)], >>> pbounds={"x": (0, len(f)-1)}) >>> bo.maximize(init_points=2, n_iter=25, acq="ucb", kappa=1) """ assert n_iter != 0 or fit # Reset timer self.plog.reset_timer() # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points, fixed_params) y_max = self.space.Y.max() if fit: # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking self.gp.fit(self.space.X, self.space.Y) # # Finding argmax of the acquisition function. # if n_iters == 0: # x_max = acq_max(ac=self.util.utility, # gp=self.gp, # y_max=y_max, # bounds=self.space.bounds, # random_state=self.random_state, # **self._acqkw) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. x_sampled = [] for i in range(n_iter): # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, fixed_params=fixed_params, **self._acqkw) if update: # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False while x_max in self.space: x_max = self.space.random_points(1)[0] pwarning = True # Append most recently generated values to X and Y arrays y = self.space.observe_point(x_max) if self.verbose: self.plog.print_step(x_max, y, pwarning) # Updating the GP. self.gp.fit(self.space.X, self.space.Y) # Update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append( dict(zip(self.space.keys, x_max))) # Update maximum value to search for next probe point. if self.space.Y[-1] > y_max: y_max = self.space.Y[-1] x_sampled.append(x_max) # Keep track of total number of iterations self.i += 1 # Print a final report if verbose active. if self.verbose: self.plog.print_summary() return x_sampled def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.space.X, np.expand_dims(self.space.Y, axis=1))) header = ','.join(self.space.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',', comments='') # --- API compatibility --- @property def X(self): warnings.warn("use self.space.X instead", DeprecationWarning) return self.space.X @property def Y(self): warnings.warn("use self.space.Y instead", DeprecationWarning) return self.space.Y @property def keys(self): warnings.warn("use self.space.keys instead", DeprecationWarning) return self.space.keys @property def f(self): warnings.warn("use self.space.target_func instead", DeprecationWarning) return self.space.target_func @property def bounds(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.bounds @property def dim(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.dim
class BayesianOptimization(object): def __init__(self, f, pbounds, verbose=1): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds # Get the name of the parameters self.keys = list(pbounds.keys()) # Find number of parameters self.dim = len(pbounds) # Create an array with parameters bounds self.bounds = [] for key in self.pbounds.keys(): self.bounds.append(self.pbounds[key]) self.bounds = np.asarray(self.bounds) # Some function to be optimized self.f = f # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Numpy array place holders self.X = None self.Y = None # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor( kernel=Matern(), n_restarts_optimizer=25, ) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # Verbose self.verbose = verbose def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Generate random points l = [ np.random.uniform(x[0], x[1], size=init_points) for x in self.bounds ] # Concatenate new random points to possible existing # points from self.explore method. self.init_points += list(map(list, zip(*l))) # Create empty list to store the new values of the function y_init = [] # Evaluate target function at all initialization # points (random + explore) for x in self.init_points: y_init.append(self.f(**dict(zip(self.keys, x)))) if self.verbose: self.plog.print_step(x, y_init[-1]) # Append any other points passed by the self.initialize method (these # also have a corresponding target value passed by the user). self.init_points += self.x_init # Append the target value of self.initialize method. y_init += self.y_init # Turn it into np array and store. self.X = np.asarray(self.init_points) self.Y = np.asarray(y_init) # Updates the flag self.initialized = True def explore(self, points_dict): """ Method to explore user defined points :param points_dict: :return: """ # Consistency check param_tup_lens = [] for key in self.keys: param_tup_lens.append(len(list(points_dict[key]))) if all([e == param_tup_lens[0] for e in param_tup_lens]): pass else: raise ValueError('The same number of initialization points ' 'must be entered for every parameter.') # Turn into list of lists all_points = [] for key in self.keys: all_points.append(points_dict[key]) # Take transpose of list self.init_points = list(map(list, zip(*all_points))) def initialize(self, points_dict): """ Method to introduce point for which the target function value is known :param points_dict: :return: """ for target in points_dict: self.y_init.append(target) all_points = [] for key in self.keys: all_points.append(points_dict[target][key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) # Loop through the all bounds and reset the min-max bound matrix for row, key in enumerate(self.pbounds.keys()): # Reset all entries, even if the same. self.bounds[row] = self.pbounds[key] def maximize(self, init_points=5, n_iter=25, acq='ei', kappa=2.576, xi=0.0, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Expected Improvement. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing """ # Reset timer self.plog.reset_timer() # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points) y_max = self.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False if np.any((self.X - x_max).sum(axis=1) == 0): x_max = np.random.uniform(self.bounds[:, 0], self.bounds[:, 1], size=self.bounds.shape[0]) pwarning = True # Append most recently generated values to X and Y arrays self.X = np.vstack((self.X, x_max.reshape((1, -1)))) self.Y = np.append(self.Y, self.f(**dict(zip(self.keys, x_max)))) # Updating the GP. ur = unique_rows(self.X) self.gp.fit(self.X[ur], self.Y[ur]) # Update maximum value to search for next probe point. if self.Y[-1] > y_max: y_max = self.Y[-1] # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.bounds) # Print stuff if self.verbose: self.plog.print_step(self.X[-1], self.Y[-1], warning=pwarning) # Keep track of total number of iterations self.i += 1 self.res['max'] = { 'max_val': self.Y.max(), 'max_params': dict(zip(self.keys, self.X[self.Y.argmax()])) } self.res['all']['values'].append(self.Y[-1]) self.res['all']['params'].append(dict(zip(self.keys, self.X[-1]))) # Print a final report if verbose active. if self.verbose: self.plog.print_summary()
class BayesianOptimization(object): def __init__(self, f, pbounds, random_state=None, verbose=1): """ :param f: Function to be maximized. :param pbounds: Dictionary with parameters names as keys and a tuple with minimum and maximum values. :param verbose: Whether or not to print progress. """ # Store the original dictionary self.pbounds = pbounds self.random_state = ensure_rng(random_state) # Data structure containing the function to be optimized, the bounds of # its domain, and a record of the evaluations we have done so far self.space = TargetSpace(f, pbounds, random_state) # Initialization flag self.initialized = False # Initialization lists --- stores starting points before process begins self.init_points = [] self.x_init = [] self.y_init = [] # Counter of iterations self.i = 0 # Internal GP regressor self.gp = GaussianProcessRegressor( kernel=Matern(nu=2.5), n_restarts_optimizer=25, random_state=self.random_state ) # Utility Function placeholder self.util = None # PrintLog object self.plog = PrintLog(self.space.keys) # Output dictionary self.res = {} # Output dictionary self.res['max'] = {'max_val': None, 'max_params': None} self.res['all'] = {'values': [], 'params': []} # non-public config for maximizing the aquisition function # (used to speedup tests, but generally leave these as is) self._acqkw = {'n_warmup': 100000, 'n_iter': 250} # Verbose self.verbose = verbose def init(self, init_points): """ Initialization method to kick start the optimization process. It is a combination of points passed by the user, and randomly sampled ones. :param init_points: Number of random points to probe. """ # Concatenate new random points to possible existing # points from self.explore method. rand_points = self.space.random_points(init_points) self.init_points.extend(rand_points) # Evaluate target function at all initialization points for x in self.init_points: y = self._observe_point(x) # Add the points from `self.initialize` to the observations if self.x_init: x_init = np.vstack(self.x_init) y_init = np.hstack(self.y_init) for x, y in zip(x_init, y_init): self.space.add_observation(x, y) if self.verbose: self.plog.print_step(x, y) # Updates the flag self.initialized = True def _observe_point(self, x): y = self.space.observe_point(x) if self.verbose: self.plog.print_step(x, y) return y def explore(self, points_dict, eager=False): """Method to explore user defined points. :param points_dict: :param eager: if True, these points are evaulated immediately """ if eager: self.plog.reset_timer() if self.verbose: self.plog.print_header(initialization=True) points = self.space._dict_to_points(points_dict) for x in points: self._observe_point(x) else: points = self.space._dict_to_points(points_dict) self.init_points = points def initialize(self, points_dict): """ Method to introduce points for which the target function value is known :param points_dict: dictionary with self.keys and 'target' as keys, and list of corresponding values as values. ex: { 'target': [-1166.19102, -1142.71370, -1138.68293], 'alpha': [7.0034, 6.6186, 6.0798], 'colsample_bytree': [0.6849, 0.7314, 0.9540], 'gamma': [8.3673, 3.5455, 2.3281], } :return: """ self.y_init.extend(points_dict['target']) for i in range(len(points_dict['target'])): all_points = [] for key in self.space.keys: all_points.append(points_dict[key][i]) self.x_init.append(all_points) def initialize_df(self, points_df): """ Method to introduce point for which the target function value is known from pandas dataframe file :param points_df: pandas dataframe with columns (target, {list of columns matching self.keys}) ex: target alpha colsample_bytree gamma -1166.19102 7.0034 0.6849 8.3673 -1142.71370 6.6186 0.7314 3.5455 -1138.68293 6.0798 0.9540 2.3281 -1146.65974 2.4566 0.9290 0.3456 -1160.32854 1.9821 0.5298 8.7863 :return: """ for i in points_df.index: self.y_init.append(points_df.loc[i, 'target']) all_points = [] for key in self.space.keys: all_points.append(points_df.loc[i, key]) self.x_init.append(all_points) def set_bounds(self, new_bounds): """ A method that allows changing the lower and upper searching bounds :param new_bounds: A dictionary with the parameter name and its new bounds """ # Update the internal object stored dict self.pbounds.update(new_bounds) self.space.set_bounds(new_bounds) def maximize(self, init_points=5, n_iter=25, acq='ucb', kappa=2.576, xi=0.0, **gp_params): """ Main optimization method. Parameters ---------- :param init_points: Number of randomly chosen points to sample the target function before fitting the gp. :param n_iter: Total number of times the process is to repeated. Note that currently this methods does not have stopping criteria (due to a number of reasons), therefore the total number of points to be sampled must be specified. :param acq: Acquisition function to be used, defaults to Upper Confidence Bound. :param gp_params: Parameters to be passed to the Scikit-learn Gaussian Process object Returns ------- :return: Nothing Example: >>> xs = np.linspace(-2, 10, 10000) >>> f = np.exp(-(xs - 2)**2) + np.exp(-(xs - 6)**2/10) + 1/ (xs**2 + 1) >>> bo = BayesianOptimization(f=lambda x: f[int(x)], >>> pbounds={"x": (0, len(f)-1)}) >>> bo.maximize(init_points=2, n_iter=25, acq="ucb", kappa=1) """ # Reset timer self.plog.reset_timer() # Set acquisition function self.util = UtilityFunction(kind=acq, kappa=kappa, xi=xi) # Initialize x, y and find current y_max if not self.initialized: if self.verbose: self.plog.print_header() self.init(init_points) y_max = self.space.Y.max() # Set parameters if any was passed self.gp.set_params(**gp_params) # Find unique rows of X to avoid GP from breaking self.gp.fit(self.space.X, self.space.Y) # Finding argmax of the acquisition function. x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) # Print new header if self.verbose: self.plog.print_header(initialization=False) # Iterative process of searching for the maximum. At each round the # most recent x and y values probed are added to the X and Y arrays # used to train the Gaussian Process. Next the maximum known value # of the target function is found and passed to the acq_max function. # The arg_max of the acquisition function is found and this will be # the next probed value of the target function in the next round. for i in range(n_iter): # Test if x_max is repeated, if it is, draw another one at random # If it is repeated, print a warning pwarning = False while x_max in self.space: x_max = self.space.random_points(1)[0] pwarning = True # Append most recently generated values to X and Y arrays y = self.space.observe_point(x_max) if self.verbose: self.plog.print_step(x_max, y, pwarning) # Updating the GP. self.gp.fit(self.space.X, self.space.Y) # Update the best params seen so far self.res['max'] = self.space.max_point() self.res['all']['values'].append(y) self.res['all']['params'].append(dict(zip(self.space.keys, x_max))) # Update maximum value to search for next probe point. if self.space.Y[-1] > y_max: y_max = self.space.Y[-1] # Maximize acquisition function to find next probing point x_max = acq_max(ac=self.util.utility, gp=self.gp, y_max=y_max, bounds=self.space.bounds, random_state=self.random_state, **self._acqkw) # Keep track of total number of iterations self.i += 1 # Print a final report if verbose active. if self.verbose: self.plog.print_summary() def points_to_csv(self, file_name): """ After training all points for which we know target variable (both from initialization and optimization) are saved :param file_name: name of the file where points will be saved in the csv format :return: None """ points = np.hstack((self.space.X, np.expand_dims(self.space.Y, axis=1))) header = ', '.join(self.space.keys + ['target']) np.savetxt(file_name, points, header=header, delimiter=',') # --- API compatibility --- @property def X(self): warnings.warn("use self.space.X instead", DeprecationWarning) return self.space.X @property def Y(self): warnings.warn("use self.space.Y instead", DeprecationWarning) return self.space.Y @property def keys(self): warnings.warn("use self.space.keys instead", DeprecationWarning) return self.space.keys @property def f(self): warnings.warn("use self.space.target_func instead", DeprecationWarning) return self.space.target_func @property def bounds(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.bounds @property def dim(self): warnings.warn("use self.space.dim instead", DeprecationWarning) return self.space.dim
def envelope_gp(sources, data, target, target_points_to_start, bounds, loss, search_grid, n_restarts=10,\ number_of_iterations=100, sigma_msr=1e-10, forget=100, forget_upd=0.8, tao=1e-2, nu=1, gamma=None,\ lr_bandit_weights=None, strategy = 'exp3'): forget_ = forget target_data = target_points_to_start.copy() #envelope_gp sigma_s = [sigma_msr ] + [nu / (tao + 1)] * len(sources) # добавим пустой источник sources_ = [lambda data: None] + sources # добавим пустой источник gp = GaussianProcessRegressor() gps = GaussianProcessRegressor() #MAB log_weights = np.array([0.0] * len(sources_)) history = [] dim = data.shape[1] K = len(sources_) #theorem 1 exp3_ix if gamma is None: if strategy == 'exp3-IX': gamma = np.sqrt(2 * np.log(K) / (K * number_of_iterations)) if strategy == 'exp3-auer': gamma = 0.9 if strategy == 'exp3': gamma = 0 if lr_bandit_weights is None: if strategy == 'exp3-IX': lr_bandit_weights = 2 * gamma if strategy == 'exp3-auer': lr_bandit_weights = gamma if strategy == 'exp3': lr_bandit_weights = np.sqrt(2 * np.log(K) / (K * number_of_iterations)) #main for _ in tqdm_notebook(range(number_of_iterations), leave=False): #distribution update corrected_weights = log_weights - np.max(log_weights) theSum = logsumexp(corrected_weights) if strategy == 'exp3-auer': probabilityDistribution = \ (1.0 - gamma) * np.exp(corrected_weights - theSum) + (gamma / log_weights.shape[0]) elif strategy == 'exp3-IX' or strategy == 'exp3': probabilityDistribution = np.exp(corrected_weights - theSum) #draw arm = draw(probabilityDistribution) alpha=np.vstack((sigma_s[arm]*np.ones((data.shape[0],1)),\ sigma_msr*np.ones((target_data.shape[0],1)))).ravel() if arm != 0: gp.set_params(alpha=alpha) y = np.vstack((sources_[arm](data)[:, np.newaxis], target(target_data)[:, np.newaxis])).ravel() gp.fit(np.vstack((data, target_data)), y) gps.fit(data, sources_[arm](data)) else: gp.set_params(alpha=sigma_msr) y = target(target_data) gp.fit(target_data, y) gps = deepcopy(gp) expected_improvement = get_gp_ucb_simple_sklearn( search_grid, gp, forget_) min_val = -np.max(expected_improvement) new_point = search_grid[np.argmax(expected_improvement)] for x0 in np.random.uniform(bounds[:, 0], bounds[:, 1], size=(n_restarts, dim)): min_obj = lambda x: -get_gp_ucb_simple_sklearn([x], gp, forget_) res = minimize(min_obj, x0, method='L-BFGS-B', bounds=bounds) if res.fun < min_val: min_val = res.fun new_point = res.x theReward = -loss(gps.predict(new_point[np.newaxis, :]), target(new_point))**2 history += [(arm, probabilityDistribution, theReward)] log_weights[arm] += theReward * lr_bandit_weights / \ (probabilityDistribution[arm] + gamma) target_data = np.vstack((target_data, np.array([new_point]))) tao += 0.5 nu += ( (target(new_point) - gps.predict(new_point[np.newaxis, :]))**2) / 2 sigma_s[arm] = nu / (tao + 1) forget_ *= forget_upd return target_data, history, gp,