def test_mean_var_interface_returns_same_as_cpp(self): """Test that the /gp/mean_var endpoint does the same thing as the C++ interface.""" tolerance = 1.0e-11 for test_case in self.gp_test_environments: python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() cpp_cov = SquareExponential(python_cov.hyperparameters) cpp_gp = GaussianProcess(cpp_cov, historical_data) points_to_evaluate = python_domain.generate_uniform_random_points_in_domain( 10) # mean and var from C++ cpp_mean = cpp_gp.compute_mean_of_points(points_to_evaluate) cpp_var = cpp_gp.compute_variance_of_points(points_to_evaluate) # mean and var from REST json_payload = self._build_json_payload( python_domain, python_cov, historical_data, points_to_evaluate.tolist()) resp = self.testapp.post(self.endpoint, json_payload) resp_schema = GpMeanVarResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) rest_mean = numpy.asarray(resp_dict.get('mean')) rest_var = numpy.asarray(resp_dict.get('var')) self.assert_vector_within_relative(rest_mean, cpp_mean, tolerance) self.assert_vector_within_relative(rest_var, cpp_var, tolerance)
def optimize(self, do_optimize=True, **kwargs): if self.prior is None: self.p0 = numpy.random.rand(1 + self.dim + self._num_derivatives + 1) else: self.p0 = self.prior.sample_from_prior(1) self.hypers = [optimize.minimize(self.nll, self.p0.ravel()).x] self.is_trained = True self._models = [] hypers_list = [] noises_list = [] for sample in self.hypers: print sample if numpy.any((-20 > sample) + (sample > 20)): continue sample = numpy.exp(sample) # Instantiate a GP for each hyperparameter configuration cov_hyps = sample[:(self.dim + 1)] hypers_list.append(cov_hyps) se = SquareExponential(cov_hyps) if self.noisy: noise = sample[(self.dim + 1):] else: noise = numpy.array((1 + self._num_derivatives) * [1.e-8]) noises_list.append(noise) model = GaussianProcess(se, noise, self._historical_data, self.derivatives) self._models.append(model) self._gaussian_process_mcmc = GaussianProcessMCMC( numpy.array(hypers_list), numpy.array(noises_list), self._historical_data, self.derivatives)
def test_python_and_cpp_return_same_cholesky_variance_and_gradient(self): """Compare chol_var/grad chol_var results from Python & C++, checking seeral random points per test case.""" num_tests_per_case = 2 var_tolerance = 3.0e-12 # TODO(GH-240): set RNG seed for this case and restore toleranace to 3.0e-12 or better grad_var_tolerance = 3.0e-10 for test_case in self.gp_test_environments: domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() cpp_cov = SquareExponential(python_cov.hyperparameters) cpp_gp = GaussianProcess(cpp_cov, historical_data) for num_to_sample in self.num_to_sample_list: for _ in xrange(num_tests_per_case): points_to_sample = domain.generate_uniform_random_points_in_domain(num_to_sample) cpp_var = cpp_gp.compute_cholesky_variance_of_points(points_to_sample) python_var = python_gp.compute_cholesky_variance_of_points(points_to_sample) self.assert_vector_within_relative(python_var, cpp_var, var_tolerance) cpp_grad_var = cpp_gp.compute_grad_cholesky_variance_of_points(points_to_sample) python_grad_var = python_gp.compute_grad_cholesky_variance_of_points(points_to_sample) self.assert_vector_within_relative(python_grad_var, cpp_grad_var, grad_var_tolerance)
def test_python_and_cpp_return_same_mu_and_gradient(self): """Compare mu/grad mu results from Python & C++, checking seeral random points per test case.""" num_tests_per_case = 4 mu_tolerance = 3.0e-13 grad_mu_tolerance = 3.0e-12 for test_case in self.gp_test_environments: domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() cpp_cov = SquareExponential(python_cov.hyperparameters) cpp_gp = GaussianProcess(cpp_cov, historical_data) for num_to_sample in self.num_to_sample_list: for _ in xrange(num_tests_per_case): points_to_sample = domain.generate_uniform_random_points_in_domain( num_to_sample) cpp_mu = cpp_gp.compute_mean_of_points(points_to_sample) python_mu = python_gp.compute_mean_of_points( points_to_sample) self.assert_vector_within_relative(python_mu, cpp_mu, mu_tolerance) cpp_grad_mu = cpp_gp.compute_grad_mean_of_points( points_to_sample) python_grad_mu = python_gp.compute_grad_mean_of_points( points_to_sample) self.assert_vector_within_relative(python_grad_mu, cpp_grad_mu, grad_mu_tolerance)
def test_sample_point_from_gp(self): """Test that sampling points from the GP works.""" point_one = SamplePoint([0.0, 1.0], -1.0, 0.0) point_two = SamplePoint([2.0, 2.5], 1.0, 0.1) covariance = SquareExponential([1.0, 1.0, 1.0]) historical_data = HistoricalData(len(point_one.point), [point_one, point_two]) gaussian_process = GaussianProcess(covariance, historical_data) out_values = numpy.zeros(3) for i in xrange(3): out_values[i] = gaussian_process.sample_point_from_gp( point_two.point, 0.001) gaussian_process._gaussian_process.reset_to_most_recent_seed() out_values_test = numpy.ones(3) for i in xrange(3): out_values_test[i] = gaussian_process.sample_point_from_gp( point_two.point, 0.001) # Exact match b/c we should've run over the exact same computations self.assert_vector_within_relative(out_values_test, out_values, 0.0) # Sampling from a historical point (that had 0 noise) should produce the same value associated w/that point value = gaussian_process.sample_point_from_gp(point_one.point, 0.0) self.assert_scalar_within_relative(value, point_one.value, numpy.finfo(numpy.float64).eps)
def test_interface_returns_same_as_cpp(self): """Test that the /gp/ei endpoint does the same thing as the C++ interface.""" tolerance = 1.0e-11 for test_case in self.gp_test_environments: python_domain, python_gp = test_case python_cov, historical_data = python_gp.get_core_data_copy() cpp_cov = SquareExponential(python_cov.hyperparameters) cpp_gp = GaussianProcess(cpp_cov, historical_data) points_to_evaluate = python_domain.generate_uniform_random_points_in_domain( 10) # EI from C++ expected_improvement_evaluator = ExpectedImprovement( cpp_gp, None, ) # TODO(GH-99): Change test case to have the right shape: # (num_to_evaluate, num_to_sample, dim) # Here we assume the shape is (num_to_evaluate, dim) so we insert an axis, making num_to_sample = 1. # Also might be worth testing more num_to_sample values (will require manipulating C++ RNG state). cpp_expected_improvement = expected_improvement_evaluator.evaluate_at_point_list( points_to_evaluate[:, numpy.newaxis, :], ) # EI from REST json_payload = self._build_json_payload( python_domain, python_cov, historical_data, points_to_evaluate.tolist()) resp = self.testapp.post(self.endpoint, json_payload) resp_schema = GpEiResponse() resp_dict = resp_schema.deserialize(json.loads(resp.body)) rest_expected_improvement = numpy.asarray( resp_dict.get('expected_improvement')) self.assert_vector_within_relative( rest_expected_improvement, cpp_expected_improvement, tolerance, )
def train(self, do_optimize=True, **kwargs): """ Performs MCMC sampling to sample hyperparameter configurations from the likelihood and trains for each sample a GP on X and y Parameters ---------- X: np.ndarray (N, D) Input data points. The dimensionality of X is (N, D), with N as the number of points and D is the number of features. y: np.ndarray (N,) The corresponding target values. do_optimize: boolean If set to true we perform MCMC sampling otherwise we just use the hyperparameter specified in the kernel. """ if do_optimize: # We have one walker for each hyperparameter configuration sampler = emcee.EnsembleSampler(self.n_chains, 1 + self.dim + self._num_derivatives + 1, self.compute_log_likelihood) # Do a burn-in in the first iteration if not self.burned: # Initialize the walkers by sampling from the prior if self.prior is None: self.p0 = numpy.random.rand(self.n_chains, 1 + self.dim + self._num_derivatives + 1) else: self.p0 = self.prior.sample_from_prior(self.n_chains) # Run MCMC sampling self.p0, _, _ = sampler.run_mcmc(self.p0, self.burnin_steps, rstate0=self.rng) self.burned = True # Start sampling pos, _, _ = sampler.run_mcmc(self.p0, self.chain_length, rstate0=self.rng) # Save the current position, it will be the start point in # the next iteration self.p0 = pos # Take the last samples from each walker self.hypers = sampler.chain[numpy.random.choice(self.n_chains, self.n_hypers), -1] self.is_trained = True self._models = [] hypers_list = [] noises_list = [] for sample in self.hypers: if numpy.any((-20 > sample) + (sample > 20)): continue sample = numpy.exp(sample) # Instantiate a GP for each hyperparameter configuration cov_hyps = sample[:(self.dim+1)] hypers_list.append(cov_hyps) se = SquareExponential(cov_hyps) if self.noisy: noise = sample[(self.dim+1):] else: noise = numpy.array((1+self._num_derivatives)*[1.e-8]) noises_list.append(noise) model = GaussianProcess(se, noise, self._historical_data, self.derivatives) self._models.append(model) self._gaussian_process_mcmc = GaussianProcessMCMC(numpy.array(hypers_list), numpy.array(noises_list), self._historical_data, self.derivatives)
} for i in range(len(hist_data)): result["data_{0}_points_sampled".format(i)] = hist_data[i].points_sampled result["data_{0}_points_sampled_value".format(i)] = hist_data[i].points_sampled_value for key in range(len(hist_data)): # Setup prior for MAP if key == 0: prior_mean = np.concatenate(([max(0.01, np.var(hist_data[key].points_sampled_value) - np.mean(hist_data[key].points_sampled_noise_variance))], [(d[1]-d[0]) for d in problem.obj_func_min._search_domain])) else: prior_mean = np.concatenate(([max(0.01, np.var(hist_data[key].points_sampled_value) - np.mean(hist_data[key].points_sampled_noise_variance) - np.mean(hist_data[0].points_sampled_noise_variance))], [(d[1]-d[0]) for d in problem.obj_func_min._search_domain])) prior_sig = np.diag(np.power(prior_mean/2., 2.0)) prior = NormalPrior(prior_mean, prior_sig) hyper_bounds = [(0.001, prior_mean[i]+2.*np.sqrt(prior_sig[i,i])) for i in range(problem.obj_func_min.getDim()+1)] hyperparam_search_domain = pythonTensorProductDomain([ClosedInterval(bound[0], bound[1]) for bound in hyper_bounds]) multistart_pts = hyperparam_search_domain.generate_uniform_random_points_in_domain(num_hyper_multistart) best_f = np.inf cov = SquareExponential(prior_mean) for i in range(num_hyper_multistart): hyper, f, output = hyper_opt(cov, data=hist_data[key], init_hyper=multistart_pts[i, :], hyper_bounds=hyper_bounds, approx_grad=False, hyper_prior=prior) # print output if f < best_f: best_hyper = hyper best_f = f result['prior_mean'].append(prior_mean) result['prior_sig'].append(np.diag(prior_sig)) result['hyper_bounds'].append(hyper_bounds) result['hyperparam'] = np.concatenate((result['hyperparam'], best_hyper)) result['loglikelihood'].append(-best_f) send_data_to_s3(bucket, problem.hyper_path, result)