def build_deep_gp(input_dimension, hidden_dimension, covariance_function): # GP going from input to hidden num_params_layer1, predict_layer1, log_marginal_likelihood_layer1 = \ make_gp_funs(covariance_function, num_cov_params=input_dimension + 1) # GP going from hidden to output num_params_layer2, predict_layer2, log_marginal_likelihood_layer2 = \ make_gp_funs(covariance_function, num_cov_params=hidden_dimension + 1) num_hidden_params = hidden_dimension * n_data total_num_params = num_params_layer1 + num_params_layer2 + num_hidden_params def unpack_all_params(all_params): layer1_params = all_params[:num_params_layer1] layer2_params = all_params[num_params_layer1:num_params_layer1+num_params_layer2] hiddens = all_params[num_params_layer1 + num_params_layer2:] return layer1_params, layer2_params, hiddens def combined_predict_fun(all_params, X, y, xs): layer1_params, layer2_params, hiddens = unpack_all_params(all_params) h_star_mean, h_star_cov = predict_layer1(layer1_params, X, hiddens, xs) y_star_mean, y_star_cov = predict_layer2(layer2_params, np.atleast_2d(hiddens).T, y, np.atleast_2d(h_star_mean).T) return y_star_mean, y_star_cov def log_marginal_likelihood(all_params): layer1_params, layer2_params, h = unpack_all_params(all_params) return log_marginal_likelihood_layer1(layer1_params, X, h) + \ log_marginal_likelihood_layer2(layer2_params, np.atleast_2d(h).T, y) predict_layer_funcs = [predict_layer1, predict_layer2] return total_num_params, log_marginal_likelihood, combined_predict_fun, unpack_all_params, \ predict_layer_funcs
def bayesian_optimize(func, domain_min, domain_max, num_iters=20, callback=None): D = len(domain_min) num_params, predict, log_marginal_likelihood = \ make_gp_funs(rbf_covariance, num_cov_params=D + 1) model_params = init_covariance_params(num_params) def optimize_gp_params(init_params, X, y): log_hyperprior = lambda params: np.sum(norm.logpdf(params, 0., 100.)) objective = lambda params: -log_marginal_likelihood(params, X, y) -log_hyperprior(params) return minimize(value_and_grad(objective), init_params, jac=True, method='CG').x def choose_next_point(domain_min, domain_max, acquisition_function, num_tries=15, rs=npr.RandomState(0)): """Uses gradient-based optimization to find next query point.""" init_points = rs.rand(num_tries, D) * (domain_max - domain_min) + domain_min grad_obj = value_and_grad(lambda x: -acquisition_function(x)) def optimize_point(init_point): print('.', end='') result = minimize(grad_obj, x0=init_point, jac=True, method='L-BFGS-B', options={'maxiter': 10}, bounds=list(zip(domain_min, domain_max))) return result.x, acquisition_function(result.x) optimzed_points, optimized_values = list(zip(*list(map(optimize_point, init_points)))) print() best_ix = np.argmax(optimized_values) return np.atleast_2d(optimzed_points[best_ix]) # Start by evaluating once in the middle of the domain. X = np.zeros((0, D)) y = np.zeros((0)) X = np.concatenate((X, np.reshape((domain_max - domain_min) / 2.0, (D, 1)))) y = np.concatenate((y, np.reshape(np.array(func(X)), (1,)))) for i in range(num_iters): if i > 1: print("Optimizing model parameters...") model_params = optimize_gp_params(model_params, X, y) print("Choosing where to look next", end='') def predict_func(xstar): mean, cov = predict(model_params, X, y, xstar) return mean, np.sqrt(np.diag(cov)) def acquisition_function(xstar): xstar = np.atleast_2d(xstar) # To work around a bug in scipy.minimize mean, std = predict_func(xstar) return expected_new_max(mean, std, defaultmax(y)) next_point = choose_next_point(domain_min, domain_max, acquisition_function) print("Evaluating expensive function...") new_value = func(next_point) X = np.concatenate((X, next_point)) y = np.concatenate((y, np.reshape(np.array(new_value), (1,)))) if callback: callback(X, y, predict_func, acquisition_function, next_point, new_value) best_ix = np.argmax(y) return X[best_ix, :], y[best_ix]
gradient = grad(variational_objective) return variational_objective, gradient, unpack_params if __name__ == '__main__': # Specify an inference problem by its unnormalized log-posterior. D = 10;dim=1;N=20;M=10; params = [0, - 6.32795237, - 0.69221531, - 0.24707744] # Build params = [0, - 6.32795237, - 0.69221531, - 0.24707744]model and objective function. num_params, predict, log_marginal_likelihood = \ gp.make_gp_funs(gp.rbf_covariance, num_cov_params=D + 1) X, y = gp.build_toy_dataset(D=dim,n_data=N) pseudo, blah = gp.build_toy_dataset(D=dim,n_data=M) out,blah=gp.build_toy_dataset(D=dim,n_data=100) objective = lambda params: -log_marginal_likelihood(params, X, y) def log_posterior(x,inputs ,len_sc,variance, t): N=x.shape sum_prob=0 params = [0, len_sc, variance, - 0.24707744] for i in range(N[0]): """An example 2D intractable distribution: a Gaussian evaluated at zero with a Gaussian prior on the log-variance."""
x = np.linspace(0.1, 1, num_per_class) xs = np.concatenate([rate *x * np.cos(angle + x * rate) + noise_std * rs.randn(num_per_class) for angle in spoke_angles]) ys = np.concatenate([rate *x * np.sin(angle + x * rate) + noise_std * rs.randn(num_per_class) for angle in spoke_angles]) return np.concatenate([np.expand_dims(xs, 1), np.expand_dims(ys,1)], axis=1) if __name__ == '__main__': data_dimension = 2 # Normally the data dimension would be much higher. latent_dimension = 2 # Build model and objective function. params_per_gp, predict, log_marginal_likelihood = \ make_gp_funs(rbf_covariance, num_cov_params=latent_dimension + 1) total_gp_params = data_dimension * params_per_gp data = make_pinwheel_data(5, 40) datalen = data.shape[0] num_latent_params = datalen * latent_dimension def unpack_params(params): gp_params = np.reshape(params[:total_gp_params], (data_dimension, params_per_gp)) latents = np.reshape(params[total_gp_params:], (datalen, latent_dimension)) return gp_params, latents def objective(params): gp_params, latents = unpack_params(params) gp_likelihood = sum([log_marginal_likelihood(gp_params[i], latents, data[:, i])
ys = np.concatenate([ rate * x * np.sin(angle + x * rate) + noise_std * rs.randn(num_per_class) for angle in spoke_angles ]) return np.concatenate( [np.expand_dims(xs, 1), np.expand_dims(ys, 1)], axis=1) if __name__ == '__main__': data_dimension = 2 # Normally the data dimension would be much higher. latent_dimension = 2 # Build model and objective function. params_per_gp, predict, log_marginal_likelihood = \ make_gp_funs(rbf_covariance, num_cov_params=latent_dimension + 1) total_gp_params = data_dimension * params_per_gp data = make_pinwheel_data(5, 40) datalen = data.shape[0] num_latent_params = datalen * latent_dimension def unpack_params(params): gp_params = np.reshape(params[:total_gp_params], (data_dimension, params_per_gp)) latents = np.reshape(params[total_gp_params:], (datalen, latent_dimension)) return gp_params, latents def objective(params):