def test_python_and_cpp_return_same_cholesky_variance_and_gradient(self):
"""Compare chol_var/grad chol_var results from Python & C++, checking seeral random points per test case."""
num_tests_per_case = 2
var_tolerance = 3.0e-12
# TODO(GH-240): set RNG seed for this case and restore toleranace to 3.0e-12 or better
grad_var_tolerance = 3.0e-10
for test_case in self.gp_test_environments:
domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
for num_to_sample in self.num_to_sample_list:
for _ in xrange(num_tests_per_case):
points_to_sample = domain.generate_uniform_random_points_in_domain(num_to_sample)
cpp_var = cpp_gp.compute_cholesky_variance_of_points(points_to_sample)
python_var = python_gp.compute_cholesky_variance_of_points(points_to_sample)
self.assert_vector_within_relative(python_var, cpp_var, var_tolerance)
cpp_grad_var = cpp_gp.compute_grad_cholesky_variance_of_points(points_to_sample)
python_grad_var = python_gp.compute_grad_cholesky_variance_of_points(points_to_sample)
self.assert_vector_within_relative(python_grad_var, cpp_grad_var, grad_var_tolerance)
def test_mean_var_interface_returns_same_as_cpp(self):
"""Test that the /gp/mean_var endpoint does the same thing as the C++ interface."""
tolerance = 1.0e-11
for test_case in self.gp_test_environments:
python_domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
points_to_evaluate = python_domain.generate_uniform_random_points_in_domain(
10)
# mean and var from C++
cpp_mean = cpp_gp.compute_mean_of_points(points_to_evaluate)
cpp_var = cpp_gp.compute_variance_of_points(points_to_evaluate)
# mean and var from REST
json_payload = self._build_json_payload(
python_domain, python_cov, historical_data,
points_to_evaluate.tolist())
resp = self.testapp.post(self.endpoint, json_payload)
resp_schema = GpMeanVarResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
rest_mean = numpy.asarray(resp_dict.get('mean'))
rest_var = numpy.asarray(resp_dict.get('var'))
self.assert_vector_within_relative(rest_mean, cpp_mean, tolerance)
self.assert_vector_within_relative(rest_var, cpp_var, tolerance)
def test_python_and_cpp_return_same_cholesky_variance_and_gradient(self):
"""Compare chol_var/grad chol_var results from Python & C++, checking seeral random points per test case."""
num_tests_per_case = 2
var_tolerance = 3.0e-12
# TODO(GH-240): set RNG seed for this case and restore toleranace to 3.0e-12 or better
grad_var_tolerance = 3.0e-10
for test_case in self.gp_test_environments:
domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
for num_to_sample in self.num_to_sample_list:
for _ in xrange(num_tests_per_case):
points_to_sample = domain.generate_uniform_random_points_in_domain(num_to_sample)
cpp_var = cpp_gp.compute_cholesky_variance_of_points(points_to_sample)
python_var = python_gp.compute_cholesky_variance_of_points(points_to_sample)
self.assert_vector_within_relative(python_var, cpp_var, var_tolerance)
cpp_grad_var = cpp_gp.compute_grad_cholesky_variance_of_points(points_to_sample)
python_grad_var = python_gp.compute_grad_cholesky_variance_of_points(points_to_sample)
self.assert_vector_within_relative(python_grad_var, cpp_grad_var, grad_var_tolerance)
def test_mean_var_interface_returns_same_as_cpp(self):
"""Test that the /gp/mean_var endpoint does the same thing as the C++ interface."""
tolerance = 1.0e-11
for test_case in self.gp_test_environments:
python_domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
points_to_evaluate = python_domain.generate_uniform_random_points_in_domain(10)
# mean and var from C++
cpp_mean = cpp_gp.compute_mean_of_points(points_to_evaluate)
cpp_var = cpp_gp.compute_variance_of_points(points_to_evaluate)
# mean and var from REST
json_payload = self._build_json_payload(python_domain, python_cov, historical_data, points_to_evaluate.tolist())
resp = self.testapp.post(self.endpoint, json_payload)
resp_schema = GpMeanVarResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
rest_mean = numpy.asarray(resp_dict.get('mean'))
rest_var = numpy.asarray(resp_dict.get('var'))
self.assert_vector_within_relative(rest_mean, cpp_mean, tolerance)
self.assert_vector_within_relative(rest_var, cpp_var, tolerance)
def test_python_and_cpp_return_same_mu_and_gradient(self):
"""Compare mu/grad mu results from Python & C++, checking seeral random points per test case."""
num_tests_per_case = 4
mu_tolerance = 3.0e-13
grad_mu_tolerance = 3.0e-12
for test_case in self.gp_test_environments:
domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
for num_to_sample in self.num_to_sample_list:
for _ in xrange(num_tests_per_case):
points_to_sample = domain.generate_uniform_random_points_in_domain(
num_to_sample)
cpp_mu = cpp_gp.compute_mean_of_points(points_to_sample)
python_mu = python_gp.compute_mean_of_points(
points_to_sample)
self.assert_vector_within_relative(python_mu, cpp_mu,
mu_tolerance)
cpp_grad_mu = cpp_gp.compute_grad_mean_of_points(
points_to_sample)
python_grad_mu = python_gp.compute_grad_mean_of_points(
points_to_sample)
self.assert_vector_within_relative(python_grad_mu,
cpp_grad_mu,
grad_mu_tolerance)
def test_sample_point_from_gp(self):
"""Test that sampling points from the GP works."""
point_one = SamplePoint([0.0, 1.0], -1.0, 0.0)
point_two = SamplePoint([2.0, 2.5], 1.0, 0.1)
covariance = SquareExponential([1.0, 1.0, 1.0])
historical_data = HistoricalData(len(point_one.point),
[point_one, point_two])
gaussian_process = GaussianProcess(covariance, historical_data)
out_values = numpy.zeros(3)
for i in xrange(3):
out_values[i] = gaussian_process.sample_point_from_gp(
point_two.point, 0.001)
gaussian_process._gaussian_process.reset_to_most_recent_seed()
out_values_test = numpy.ones(3)
for i in xrange(3):
out_values_test[i] = gaussian_process.sample_point_from_gp(
point_two.point, 0.001)
# Exact match b/c we should've run over the exact same computations
self.assert_vector_within_relative(out_values_test, out_values, 0.0)
# Sampling from a historical point (that had 0 noise) should produce the same value associated w/that point
value = gaussian_process.sample_point_from_gp(point_one.point, 0.0)
self.assert_scalar_within_relative(value, point_one.value,
numpy.finfo(numpy.float64).eps)
def optimize(self, do_optimize=True, **kwargs):
if self.prior is None:
self.p0 = numpy.random.rand(1 + self.dim + self._num_derivatives +
1)
else:
self.p0 = self.prior.sample_from_prior(1)
self.hypers = [optimize.minimize(self.nll, self.p0.ravel()).x]
self.is_trained = True
self._models = []
hypers_list = []
noises_list = []
for sample in self.hypers:
print sample
if numpy.any((-20 > sample) + (sample > 20)):
continue
sample = numpy.exp(sample)
# Instantiate a GP for each hyperparameter configuration
cov_hyps = sample[:(self.dim + 1)]
hypers_list.append(cov_hyps)
se = SquareExponential(cov_hyps)
if self.noisy:
noise = sample[(self.dim + 1):]
else:
noise = numpy.array((1 + self._num_derivatives) * [1.e-8])
noises_list.append(noise)
model = GaussianProcess(se, noise, self._historical_data,
self.derivatives)
self._models.append(model)
self._gaussian_process_mcmc = GaussianProcessMCMC(
numpy.array(hypers_list), numpy.array(noises_list),
self._historical_data, self.derivatives)
def test_sample_point_from_gp(self):
"""Test that sampling points from the GP works."""
point_one = SamplePoint([0.0, 1.0], -1.0, 0.0)
point_two = SamplePoint([2.0, 2.5], 1.0, 0.1)
covariance = SquareExponential([1.0, 1.0, 1.0])
historical_data = HistoricalData(len(point_one.point), [point_one, point_two])
gaussian_process = GaussianProcess(covariance, historical_data)
out_values = numpy.zeros(3)
for i in xrange(3):
out_values[i] = gaussian_process.sample_point_from_gp(point_two.point, 0.001)
gaussian_process._gaussian_process.reset_to_most_recent_seed()
out_values_test = numpy.ones(3)
for i in xrange(3):
out_values_test[i] = gaussian_process.sample_point_from_gp(point_two.point, 0.001)
# Exact match b/c we should've run over the exact same computations
self.assert_vector_within_relative(out_values_test, out_values, 0.0)
# Sampling from a historical point (that had 0 noise) should produce the same value associated w/that point
value = gaussian_process.sample_point_from_gp(point_one.point, 0.0)
self.assert_scalar_within_relative(value, point_one.value, numpy.finfo(numpy.float64).eps)
def test_gp_construction_singular_covariance_matrix(self):
"""Test that the GaussianProcess ctor indicates a singular covariance matrix when points_sampled contains duplicates (0 noise)."""
index = numpy.argmax(numpy.greater_equal(self.num_sampled_list, 1))
domain, gaussian_process = self.gp_test_environments[index]
point_one = SamplePoint([0.0] * domain.dim, 1.0, 0.0)
# points two and three have duplicate coordinates and we have noise_variance = 0.0
point_two = SamplePoint([1.0] * domain.dim, 1.0, 0.0)
point_three = point_two
historical_data = HistoricalData(len(point_one.point),
[point_one, point_two, point_three])
with pytest.raises(C_GP.SingularMatrixException):
GaussianProcess(gaussian_process.get_covariance_copy(),
historical_data)
def moe_compute_best_pt_info(moe_exp, covariance_info, confidence=None,
mean_fxn_info=None, sample_pts=None, debug=False):
if moe_exp.historical_data.num_sampled <= 0: return None, None
covar = SquareExponential(covariance_info['hyperparameters'])
cpp_gp = GaussianProcess(covar, moe_exp.historical_data,
mean_fxn_info=mean_fxn_info)
if (sample_pts == None):
sample_pts = np.array(moe_exp.historical_data.points_sampled)
#moe_pts_r = np.array(moe_exp.historical_data.points_sampled)
#moe_pts_d = moe_exp.domain.generate_uniform_random_points_in_domain(50)
#sample_pts = np.concatenate((moe_pts_r, moe_pts_d), axis=0)
cpp_mu = cpp_gp.compute_mean_of_points(sample_pts, debug)
cpp_var = np.diag(cpp_gp.compute_variance_of_points(sample_pts))
#print "sample_pts ", sample_pts, "\ncpp_mu ", cpp_mu, "\ncpp_var ", cpp_var
if confidence is None:
minidx = np.argmin(cpp_mu)
else:
upper_conf = scipy.stats.norm.interval(confidence, loc=cpp_mu,
scale=np.sqrt(cpp_var))[1]
minidx = np.argmin(upper_conf)
#print " cpp_var ", cpp_var[minidx], " upper_conf ", upper_conf[minidx]
#print "cpp_mu ", cpp_mu[minidx], " best_moe_pt ", sample_pts[minidx]
return [sample_pts[minidx], cpp_mu[minidx], cpp_var[minidx]]
def test_python_and_cpp_return_same_mu_and_gradient(self):
"""Compare mu/grad mu results from Python & C++, checking seeral random points per test case."""
num_tests_per_case = 4
mu_tolerance = 3.0e-13
grad_mu_tolerance = 3.0e-12
for test_case in self.gp_test_environments:
domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
for num_to_sample in self.num_to_sample_list:
for _ in xrange(num_tests_per_case):
points_to_sample = domain.generate_uniform_random_points_in_domain(num_to_sample)
cpp_mu = cpp_gp.compute_mean_of_points(points_to_sample)
python_mu = python_gp.compute_mean_of_points(points_to_sample)
self.assert_vector_within_relative(python_mu, cpp_mu, mu_tolerance)
cpp_grad_mu = cpp_gp.compute_grad_mean_of_points(points_to_sample)
python_grad_mu = python_gp.compute_grad_mean_of_points(points_to_sample)
self.assert_vector_within_relative(python_grad_mu, cpp_grad_mu, grad_mu_tolerance)
def _make_gp_from_params(params):
"""Create and return a C++ backed gaussian_process from the request params as a dict.
``params`` has the following form::
params = {
'gp_historical_info': <instance of :class:`moe.views.schemas.base_schemas.GpHistoricalInfo`>,
'domain_info': <instance of :class:`moe.views.schemas.base_schemas.DomainInfo`>,
'covariance_info': <instance of :class:`moe.views.schemas.base_schemas.CovarianceInfo`>,
}
:param params: The request params dict
:type params: dict
"""
# Load up the info
gp_historical_info = params.get("gp_historical_info")
domain_info = params.get("domain_info")
points_sampled = gp_historical_info.get('points_sampled')
sample_point_list = []
for point in points_sampled:
sample_point_list.append(
SamplePoint(
point['point'],
point['value'],
point['value_var'],
))
optimizer_info = params.get('optimizer_info', {})
optimizer_type = optimizer_info.get('optimizer_type', None)
if optimizer_type == L_BFGS_B_OPTIMIZER:
covariance_of_process = _make_covariance_of_process_from_params(
params, "python")
gaussian_process = pythonGaussianProcess(
covariance_of_process,
HistoricalData(domain_info.get('dim'), sample_point_list),
)
else:
covariance_of_process = _make_covariance_of_process_from_params(params)
gaussian_process = GaussianProcess(
covariance_of_process,
HistoricalData(domain_info.get('dim'), sample_point_list),
)
return gaussian_process
def test_interface_returns_same_as_cpp(self):
"""Test that the /gp/ei endpoint does the same thing as the C++ interface."""
tolerance = 1.0e-11
for test_case in self.gp_test_environments:
python_domain, python_gp = test_case
python_cov, historical_data = python_gp.get_core_data_copy()
cpp_cov = SquareExponential(python_cov.hyperparameters)
cpp_gp = GaussianProcess(cpp_cov, historical_data)
points_to_evaluate = python_domain.generate_uniform_random_points_in_domain(
10)
# EI from C++
expected_improvement_evaluator = ExpectedImprovement(
cpp_gp,
None,
)
# TODO(GH-99): Change test case to have the right shape:
# (num_to_evaluate, num_to_sample, dim)
# Here we assume the shape is (num_to_evaluate, dim) so we insert an axis, making num_to_sample = 1.
# Also might be worth testing more num_to_sample values (will require manipulating C++ RNG state).
cpp_expected_improvement = expected_improvement_evaluator.evaluate_at_point_list(
points_to_evaluate[:, numpy.newaxis, :], )
# EI from REST
json_payload = self._build_json_payload(
python_domain, python_cov, historical_data,
points_to_evaluate.tolist())
resp = self.testapp.post(self.endpoint, json_payload)
resp_schema = GpEiResponse()
resp_dict = resp_schema.deserialize(json.loads(resp.body))
rest_expected_improvement = numpy.asarray(
resp_dict.get('expected_improvement'))
self.assert_vector_within_relative(
rest_expected_improvement,
cpp_expected_improvement,
tolerance,
)
def train(self, do_optimize=True, **kwargs):
"""
Performs MCMC sampling to sample hyperparameter configurations from the
likelihood and trains for each sample a GP on X and y
Parameters
----------
X: np.ndarray (N, D)
Input data points. The dimensionality of X is (N, D),
with N as the number of points and D is the number of features.
y: np.ndarray (N,)
The corresponding target values.
do_optimize: boolean
If set to true we perform MCMC sampling otherwise we just use the
hyperparameter specified in the kernel.
"""
if do_optimize:
# We have one walker for each hyperparameter configuration
sampler = emcee.EnsembleSampler(self.n_chains, 1 + self.dim + self._num_derivatives + 1,
self.compute_log_likelihood)
# Do a burn-in in the first iteration
if not self.burned:
# Initialize the walkers by sampling from the prior
if self.prior is None:
self.p0 = numpy.random.rand(self.n_chains, 1 + self.dim + self._num_derivatives + 1)
else:
self.p0 = self.prior.sample_from_prior(self.n_chains)
# Run MCMC sampling
self.p0, _, _ = sampler.run_mcmc(self.p0, self.burnin_steps,
rstate0=self.rng)
self.burned = True
# Start sampling
pos, _, _ = sampler.run_mcmc(self.p0, self.chain_length,
rstate0=self.rng)
# Save the current position, it will be the start point in
# the next iteration
self.p0 = pos
# Take the last samples from each walker
self.hypers = sampler.chain[numpy.random.choice(self.n_chains, self.n_hypers), -1]
self.is_trained = True
self._models = []
hypers_list = []
noises_list = []
for sample in self.hypers:
if numpy.any((-20 > sample) + (sample > 20)):
continue
sample = numpy.exp(sample)
# Instantiate a GP for each hyperparameter configuration
cov_hyps = sample[:(self.dim+1)]
hypers_list.append(cov_hyps)
se = SquareExponential(cov_hyps)
if self.noisy:
noise = sample[(self.dim+1):]
else:
noise = numpy.array((1+self._num_derivatives)*[1.e-8])
noises_list.append(noise)
model = GaussianProcess(se, noise,
self._historical_data,
self.derivatives)
self._models.append(model)
self._gaussian_process_mcmc = GaussianProcessMCMC(numpy.array(hypers_list), numpy.array(noises_list),
self._historical_data, self.derivatives)
### update noise in historical data
updated_points_sampled_noise_variance = create_array_points_sampled_noise_variance(
current_hist_data.points_sampled, hyperparameters_noise)
### create new Historical data object with updated values
new_historical_data = HistoricalData(dim=problem.obj_func_min.getDim())
# new_historical_data.append_historical_data(current_hist_data.points_sampled,
# current_hist_data.points_sampled_value,
# updated_points_sampled_noise_variance)
new_historical_data.append_historical_data(
current_hist_data.points_sampled,
current_hist_data.points_sampled_value,
current_hist_data.points_sampled_noise_variance)
### Use new hyperparameters -- this requires instantiating a new GP object
kg_cov_cpp = cppCovariance(hyperparameters=hypers_GP)
kg_gp_cpp = GaussianProcess(kg_cov_cpp, new_historical_data)
# ================================================================================================================ #
# Find s and point that maximize KG/cost #
# ================================================================================================================ #
discrete_pts_x = problem.obj_func_min.get_moe_domain(
).generate_uniform_x_points_in_domain(num_discretization)
discrete_pts = np.hstack(
(problem.obj_func_min.getSearchDomain()[0, -1] * np.ones(
(num_discretization, 1)), discrete_pts_x))
all_mu = kg_gp_cpp.compute_mean_of_points(discrete_pts)
sorted_idx = np.argsort(all_mu)
all_S_xprime = discrete_pts[
sorted_idx[-num_x_prime:], :] # select the last num_x_prime samples
# ================================================================================================================ #