def test_grid_generation(self):
"""Test that ``generate_grid_points`` generates a uniform grid.
Test makes assumptions about the ordering of the output that may be invalidated by
changes to numpy.meshgrid.
"""
domain_bounds = ClosedInterval.build_closed_intervals_from_list([[0.0, 1.0], [-2.0, 3.0], [2.71, 3.14]])
points_per_dimension = [7, 11, 8]
# Test that all points are present
grid = generate_grid_points(points_per_dimension, domain_bounds)
per_axis_grid = [numpy.linspace(bounds.min, bounds.max, points_per_dimension[i])
for i, bounds in enumerate(domain_bounds)]
# Loop ordering assumes the output is ordered a certain way.
for i, y_coord in enumerate(per_axis_grid[1]):
for j, x_coord in enumerate(per_axis_grid[0]):
for k, z_coord in enumerate(per_axis_grid[2]):
truth = numpy.array([x_coord, y_coord, z_coord])
index = i * per_axis_grid[2].size * per_axis_grid[0].size + j * per_axis_grid[2].size + k
test = grid[index, ...]
self.assert_vector_within_relative(test, truth, 0.0)
# Also test that scalar points_per_dimension works
points_per_dimension = [5, 5, 5]
grid_truth = generate_grid_points(points_per_dimension, domain_bounds)
points_per_dimension = 5
grid_test = generate_grid_points(points_per_dimension, domain_bounds)
self.assert_vector_within_relative(grid_test, grid_truth, 0.0)
def fill_random_domain_bounds(lower_bound_interval, upper_bound_interval, dim):
r"""Generate a random list of dim ``[min_i, max_i]`` pairs.
The data is organized such that:
``min_i \in [lower_bound_interval.min, lower_bound_interval.max]``
``max_i \in [upper_bound_interval.min, upper_bound_interval.max]``
This is mainly useful for testing or when "random" data is needed so that we get more varied cases than the unit hypercube.
:param lower_bound_interval: an uniform range, ``[min, max]``, from which to draw the domain lower bounds, ``min_i``
:type lower_bound_interval: ClosedInterval
:param upper_bound_interval: an uniform range, ``[min, max]``, from which to draw the domain upper bounds, ``max_i``
:type upper_bound_interval: ClosedInterval
:param dim: the spatial dimension of a point (i.e., number of independent params in experiment)
:type dim: int > 0
:return: ClosedInterval objects with their min, max members initialized as described
:rtype: list of ClosedInterval
"""
domain_bounds = numpy.empty((dim, 2))
domain_bounds[..., 0] = numpy.random.uniform(lower_bound_interval.min,
lower_bound_interval.max)
domain_bounds[..., 1] = numpy.random.uniform(upper_bound_interval.min,
upper_bound_interval.max)
return ClosedInterval.build_closed_intervals_from_list(domain_bounds)
def _build_gaussian_process_test_data(self, test_environment):
"""Build up a Gaussian Process randomly by repeatedly drawing from and then adding to the prior.
:param test_environment: parameters describing how to construct a GP prior
:type test_environment: GaussianProcessTestEnvironmentInput
:return: gaussian process environments that can be used to run tests
:rtype: GaussianProcessTestEnvironment
"""
covariance = gp_utils.fill_random_covariance_hyperparameters(
test_environment.hyperparameter_interval,
test_environment.num_hyperparameters,
covariance_type=test_environment.covariance_class,
)
domain_bounds = gp_utils.fill_random_domain_bounds(
test_environment.lower_bound_interval,
test_environment.upper_bound_interval,
test_environment.dim,
)
domain = test_environment.spatial_domain_class(ClosedInterval.build_closed_intervals_from_list(domain_bounds))
points_sampled = domain.generate_uniform_random_points_in_domain(test_environment.num_sampled)
gaussian_process = gp_utils.build_random_gaussian_process(
points_sampled,
covariance,
noise_variance=test_environment.noise_variance,
gaussian_process_type=test_environment.gaussian_process_class,
)
return GaussianProcessTestEnvironment(domain, gaussian_process)
def _build_gaussian_process_test_data(self, test_environment):
"""Build up a Gaussian Process randomly by repeatedly drawing from and then adding to the prior.
:param test_environment: parameters describing how to construct a GP prior
:type test_environment: GaussianProcessTestEnvironmentInput
:return: gaussian process environments that can be used to run tests
:rtype: GaussianProcessTestEnvironment
"""
covariance = gp_utils.fill_random_covariance_hyperparameters(
test_environment.hyperparameter_interval,
test_environment.num_hyperparameters,
covariance_type=test_environment.covariance_class,
)
domain_bounds = gp_utils.fill_random_domain_bounds(
test_environment.lower_bound_interval,
test_environment.upper_bound_interval,
test_environment.dim,
)
domain = test_environment.spatial_domain_class(
ClosedInterval.build_closed_intervals_from_list(domain_bounds))
points_sampled = domain.generate_uniform_random_points_in_domain(
test_environment.num_sampled)
gaussian_process = gp_utils.build_random_gaussian_process(
points_sampled,
covariance,
noise_variance=test_environment.noise_variance,
gaussian_process_type=test_environment.gaussian_process_class,
)
return GaussianProcessTestEnvironment(domain, gaussian_process)
def test_grid_generation(self):
"""Test that ``generate_grid_points`` generates a uniform grid.
Test makes assumptions about the ordering of the output that may be invalidated by
changes to numpy.meshgrid.
"""
domain_bounds = ClosedInterval.build_closed_intervals_from_list([[0.0, 1.0], [-2.0, 3.0], [2.71, 3.14]])
points_per_dimension = [7, 11, 8]
# Test that all points are present
grid = generate_grid_points(points_per_dimension, domain_bounds)
per_axis_grid = [numpy.linspace(bounds.min, bounds.max, points_per_dimension[i])
for i, bounds in enumerate(domain_bounds)]
# Loop ordering assumes the output is ordered a certain way.
for i, y_coord in enumerate(per_axis_grid[1]):
for j, x_coord in enumerate(per_axis_grid[0]):
for k, z_coord in enumerate(per_axis_grid[2]):
truth = numpy.array([x_coord, y_coord, z_coord])
index = i * per_axis_grid[2].size * per_axis_grid[0].size + j * per_axis_grid[2].size + k
test = grid[index, ...]
self.assert_vector_within_relative(test, truth, 0.0)
# Also test that scalar points_per_dimension works
points_per_dimension = [5, 5, 5]
grid_truth = generate_grid_points(points_per_dimension, domain_bounds)
points_per_dimension = 5
grid_test = generate_grid_points(points_per_dimension, domain_bounds)
self.assert_vector_within_relative(grid_test, grid_truth, 0.0)
def test_hyperparameter_gradient_pings(self):
"""Ping test (compare analytic result to finite difference) the gradient wrt hyperparameters."""
h = 2.0e-3
tolerance = 4.0e-5
num_tests = 10
dim = 3
num_hyperparameters = dim + 1
hyperparameter_interval = ClosedInterval(3.0, 5.0)
domain = TensorProductDomain(
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0],
[-1.0, 1.0],
[-1.0, 1.0]]))
points1 = domain.generate_uniform_random_points_in_domain(num_tests)
points2 = domain.generate_uniform_random_points_in_domain(num_tests)
for i in xrange(num_tests):
point_one = points1[i, ...]
point_two = points2[i, ...]
covariance = gp_utils.fill_random_covariance_hyperparameters(
hyperparameter_interval,
num_hyperparameters,
covariance_type=self.CovarianceClass,
)
analytic_grad = covariance.hyperparameter_grad_covariance(
point_one, point_two)
for k in xrange(covariance.num_hyperparameters):
hyperparameters_old = covariance.hyperparameters
# hyperparamter + h
hyperparameters_p = numpy.copy(hyperparameters_old)
hyperparameters_p[k] += h
covariance.hyperparameters = hyperparameters_p
cov_p = covariance.covariance(point_one, point_two)
covariance.hyperparameters = hyperparameters_old
# hyperparamter - h
hyperparameters_m = numpy.copy(hyperparameters_old)
hyperparameters_m[k] -= h
covariance.hyperparameters = hyperparameters_m
cov_m = covariance.covariance(point_one, point_two)
covariance.hyperparameters = hyperparameters_old
# calculate finite diff
fd_grad = (cov_p - cov_m) / (2.0 * h)
self.assert_scalar_within_relative(fd_grad, analytic_grad[k],
tolerance)
def base_setup(cls):
"""Set up parameters for test cases."""
domain_bounds_to_test = [
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0]]),
ClosedInterval.build_closed_intervals_from_list([[-10.0, 10.0]]),
ClosedInterval.build_closed_intervals_from_list([[-500.0, -490.0]]),
ClosedInterval.build_closed_intervals_from_list([[6000.0, 6000.001]]),
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0], [-1.0, 1.0]]),
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0], [-1.0, 1.0], [-1.0, 1.0]]),
ClosedInterval.build_closed_intervals_from_list([[-7000.0, 10000.0], [-8000.0, -7999.0], [10000.06, 10000.0601]]),
]
cls.domains_to_test = [TensorProductDomain(domain_bounds) for domain_bounds in domain_bounds_to_test]
cls.num_points_to_test = (1, 2, 5, 10, 20)
def base_setup(cls):
"""Set up parameters for test cases."""
domain_bounds_to_test = [
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0]]),
ClosedInterval.build_closed_intervals_from_list([[-10.0, 10.0]]),
ClosedInterval.build_closed_intervals_from_list([[-500.0, -490.0]]),
ClosedInterval.build_closed_intervals_from_list([[6000.0, 6000.001]]),
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0], [-1.0, 1.0]]),
ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0], [-1.0, 1.0], [-1.0, 1.0]]),
ClosedInterval.build_closed_intervals_from_list([[-7000.0, 10000.0], [-8000.0, -7999.0], [10000.06, 10000.0601]]),
]
cls.domains_to_test = [TensorProductDomain(domain_bounds) for domain_bounds in domain_bounds_to_test]
cls.num_points_to_test = (1, 2, 5, 10, 20)
def test_hyperparameter_gradient_pings(self):
"""Ping test (compare analytic result to finite difference) the gradient wrt hyperparameters."""
h = 2.0e-3
tolerance = 4.0e-5
num_tests = 10
dim = 3
num_hyperparameters = dim + 1
hyperparameter_interval = ClosedInterval(3.0, 5.0)
domain = TensorProductDomain(ClosedInterval.build_closed_intervals_from_list([[-1.0, 1.0], [-1.0, 1.0], [-1.0, 1.0]]))
points1 = domain.generate_uniform_random_points_in_domain(num_tests)
points2 = domain.generate_uniform_random_points_in_domain(num_tests)
for i in xrange(num_tests):
point_one = points1[i, ...]
point_two = points2[i, ...]
covariance = gp_utils.fill_random_covariance_hyperparameters(
hyperparameter_interval,
num_hyperparameters,
covariance_type=self.CovarianceClass,
)
analytic_grad = covariance.hyperparameter_grad_covariance(point_one, point_two)
for k in xrange(covariance.num_hyperparameters):
hyperparameters_old = covariance.hyperparameters
# hyperparamter + h
hyperparameters_p = numpy.copy(hyperparameters_old)
hyperparameters_p[k] += h
covariance.hyperparameters = hyperparameters_p
cov_p = covariance.covariance(point_one, point_two)
covariance.hyperparameters = hyperparameters_old
# hyperparamter - h
hyperparameters_m = numpy.copy(hyperparameters_old)
hyperparameters_m[k] -= h
covariance.hyperparameters = hyperparameters_m
cov_m = covariance.covariance(point_one, point_two)
covariance.hyperparameters = hyperparameters_old
# calculate finite diff
fd_grad = (cov_p - cov_m) / (2.0 * h)
self.assert_scalar_within_relative(fd_grad, analytic_grad[k], tolerance)
def fill_random_domain_bounds(lower_bound_interval, upper_bound_interval, dim):
r"""Generate a random list of dim ``[min_i, max_i]`` pairs.
The data is organized such that:
``min_i \in [lower_bound_interval.min, lower_bound_interval.max]``
``max_i \in [upper_bound_interval.min, upper_bound_interval.max]``
This is mainly useful for testing or when "random" data is needed so that we get more varied cases than the unit hypercube.
:param lower_bound_interval: an uniform range, ``[min, max]``, from which to draw the domain lower bounds, ``min_i``
:type lower_bound_interval: ClosedInterval
:param upper_bound_interval: an uniform range, ``[min, max]``, from which to draw the domain upper bounds, ``max_i``
:type upper_bound_interval: ClosedInterval
:param dim: the spatial dimension of a point (i.e., number of independent params in experiment)
:type dim: int > 0
:return: ClosedInterval objects with their min, max members initialized as described
:rtype: list of ClosedInterval
"""
domain_bounds = numpy.empty((dim, 2))
domain_bounds[..., 0] = numpy.random.uniform(lower_bound_interval.min, lower_bound_interval.max)
domain_bounds[..., 1] = numpy.random.uniform(upper_bound_interval.min, upper_bound_interval.max)
return ClosedInterval.build_closed_intervals_from_list(domain_bounds)
def multistart_hyperparameter_optimization(
hyperparameter_optimizer,
num_multistarts,
randomness=None,
max_num_threads=DEFAULT_MAX_NUM_THREADS,
status=None,
):
r"""Select the hyperparameters that maximize the specified log likelihood measure of model fit (over the historical data) within the specified domain.
.. Note:: The following comments are copied from
:func:`moe.optimal_learning.python.cpp_wrappers.log_likelihood.multistart_hyperparameter_optimization`.
See :class:`moe.optimal_learning.python.python_version.log_likelihood.GaussianProcessLogMarginalLikelihood` and
:class:`moe.optimal_learning.python.python_version.log_likelihood.GaussianProcessLeaveOneOutLogLikelihood`
for an overview of some example log likelihood-like measures.
Optimizers are: null ('dumb' search), gradient descent, newton
Newton is the suggested optimizer, which is not presently available in Python (use the C++ interface). In Python,
gradient descent is suggested.
TODO(GH-57): Implement hessians and Newton's method.
'dumb' search means this will just evaluate the objective log likelihood measure at num_multistarts 'points'
(hyperparameters) in the domain, uniformly sampled using latin hypercube sampling.
See gpp_python_common.cpp for C++ enum declarations laying out the options for objective and optimizer types.
Currently, during optimization, we recommend that the coordinates of the initial guesses not differ from the
coordinates of the optima by more than about 1 order of magnitude. This is a very (VERY!) rough guideline for
sizing the domain and gd_parameters.num_multistarts; i.e., be wary of sets of initial guesses that cover the space too sparsely.
Solution is guaranteed to lie within the region specified by "domain"; note that this may not be a
true optima (i.e., the gradient may be substantially nonzero).
.. WARNING:: this function fails if NO improvement can be found! In that case,
the output will always be the first randomly chosen point. status will report failure.
TODO(GH-56): Allow callers to pass in a source of randomness.
:param hyperparameter_optimizer: object that optimizes (e.g., gradient descent, newton) the desired log_likelihood
measure over a domain (wrt the hyperparameters of covariance)
:type hyperparameter_optimizer: interfaces.optimization_interfaces.OptimizerInterface subclass
:param num_multistarts: number of times to multistart ``hyperparameter_optimizer``
:type num_multistarts: int > 0
:param randomness: random source used to generate multistart points (UNUSED)
:type randomness: (UNUSED)
:param max_num_threads: maximum number of threads to use, >= 1 (UNUSED)
:type max_num_threads: int > 0
:param status: (output) status messages (e.g., reporting on optimizer success, etc.)
:type status: dict
:return: hyperparameters that maximize the specified log likelihood measure within the specified domain
:rtype: array of float64 with shape (log_likelihood_evaluator.num_hyperparameters)
"""
# Producing the random starts in log10 space improves robustness by clustering some extra points near 0
domain_bounds_log10 = numpy.log10(
hyperparameter_optimizer.domain._domain_bounds)
domain_log10 = TensorProductDomain(
ClosedInterval.build_closed_intervals_from_list(domain_bounds_log10))
random_starts = domain_log10.generate_uniform_random_points_in_domain(
num_points=num_multistarts)
random_starts = numpy.power(10.0, random_starts)
best_hyperparameters, _ = multistart_optimize(
hyperparameter_optimizer, starting_points=random_starts)
# TODO(GH-59): Have GD actually indicate whether updates were found, e.g., in an IOContainer-like structure.
found_flag = True
if status is not None:
status["gradient_descent_found_update"] = found_flag
return best_hyperparameters
def multistart_hyperparameter_optimization(
hyperparameter_optimizer,
num_multistarts,
randomness=None,
max_num_threads=DEFAULT_MAX_NUM_THREADS,
status=None,
):
r"""Select the hyperparameters that maximize the specified log likelihood measure of model fit (over the historical data) within the specified domain.
.. Note:: The following comments are copied from
:func:`moe.optimal_learning.python.cpp_wrappers.log_likelihood.multistart_hyperparameter_optimization`.
See :class:`moe.optimal_learning.python.python_version.log_likelihood.GaussianProcessLogMarginalLikelihood` and
:class:`moe.optimal_learning.python.python_version.log_likelihood.GaussianProcessLeaveOneOutLogLikelihood`
for an overview of some example log likelihood-like measures.
Optimizers are: null ('dumb' search), gradient descent, newton
Newton is the suggested optimizer, which is not presently available in Python (use the C++ interface). In Python,
gradient descent is suggested.
TODO(GH-57): Implement hessians and Newton's method.
'dumb' search means this will just evaluate the objective log likelihood measure at num_multistarts 'points'
(hyperparameters) in the domain, uniformly sampled using latin hypercube sampling.
See gpp_python_common.cpp for C++ enum declarations laying out the options for objective and optimizer types.
Currently, during optimization, we recommend that the coordinates of the initial guesses not differ from the
coordinates of the optima by more than about 1 order of magnitude. This is a very (VERY!) rough guideline for
sizing the domain and gd_parameters.num_multistarts; i.e., be wary of sets of initial guesses that cover the space too sparsely.
Solution is guaranteed to lie within the region specified by "domain"; note that this may not be a
true optima (i.e., the gradient may be substantially nonzero).
.. WARNING:: this function fails if NO improvement can be found! In that case,
the output will always be the first randomly chosen point. status will report failure.
TODO(GH-56): Allow callers to pass in a source of randomness.
:param hyperparameter_optimizer: object that optimizes (e.g., gradient descent, newton) the desired log_likelihood
measure over a domain (wrt the hyperparameters of covariance)
:type hyperparameter_optimizer: interfaces.optimization_interfaces.OptimizerInterface subclass
:param num_multistarts: number of times to multistart ``hyperparameter_optimizer``
:type num_multistarts: int > 0
:param randomness: random source used to generate multistart points (UNUSED)
:type randomness: (UNUSED)
:param max_num_threads: maximum number of threads to use, >= 1 (UNUSED)
:type max_num_threads: int > 0
:param status: status messages (e.g., reporting on optimizer success, etc.)
:type status: dict
:return: hyperparameters that maximize the specified log likelihood measure within the specified domain
:rtype: array of float64 with shape (log_likelihood_evaluator.num_hyperparameters)
"""
# Producing the random starts in log10 space improves robustness by clustering some extra points near 0
domain_bounds_log10 = numpy.log10(hyperparameter_optimizer.domain._domain_bounds)
domain_log10 = TensorProductDomain(ClosedInterval.build_closed_intervals_from_list(domain_bounds_log10))
random_starts = domain_log10.generate_uniform_random_points_in_domain(num_points=num_multistarts)
random_starts = numpy.power(10.0, random_starts)
best_hyperparameters, _ = multistart_optimize(hyperparameter_optimizer, starting_points=random_starts)
# TODO(GH-59): Have GD actually indicate whether updates were found, e.g., in an IOContainer-like structure.
found_flag = True
if status is not None:
status["gradient_descent_found_update"] = found_flag
return best_hyperparameters
