def compute_log_likelihood(self, hyps):
r"""Compute the objective_type measure at the specified hyperparameters.
:return: value of log_likelihood evaluated at hyperparameters (``LL(y | X, \theta)``)
:rtype: float64
"""
# Bound the hyperparameter space to keep things sane. Note all
# hyperparameters live on a log scale
if numpy.any((-20 > hyps) + (hyps > 20)):
return -numpy.inf
if not self.noisy:
hyps[(self.dim+1):] = numpy.log((1+self._num_derivatives)*[1.e-8])
posterior = 1
if self.prior is not None:
posterior = self.prior.lnprob(hyps)
hyps = numpy.exp(hyps)
cov_hyps = hyps[:(self.dim+1)]
noise = hyps[(self.dim+1):]
if posterior == -numpy.inf:
return -numpy.inf
else:
return posterior + C_GP.compute_log_likelihood(
cpp_utils.cppify(self._points_sampled),
cpp_utils.cppify(self._points_sampled_value),
self.dim,
self._num_sampled,
self.objective_type,
cpp_utils.cppify_hyperparameters(cov_hyps),
cpp_utils.cppify(self._derivatives), self._num_derivatives,
cpp_utils.cppify(noise),
)
def restarted_hyperparameter_optimization(
log_likelihood_optimizer,
status=None,
):
# status must be an initialized dict for the call to C++.
if status is None:
status = {}
# C++ expects the domain in log10 space and in list form
domain_bounds_log10 = numpy.log10(
log_likelihood_optimizer.domain._domain_bounds)
hyperparameters_opt = C_GP.restarted_hyperparameter_optimization(
log_likelihood_optimizer.optimizer_parameters,
cpp_utils.cppify(domain_bounds_log10),
cpp_utils.cppify(
log_likelihood_optimizer.objective_function._points_sampled),
cpp_utils.cppify(
log_likelihood_optimizer.objective_function._points_sampled_value),
log_likelihood_optimizer.objective_function.dim,
log_likelihood_optimizer.objective_function._num_sampled,
cpp_utils.cppify_hyperparameters(
log_likelihood_optimizer.objective_function.cov_hyperparameters),
cpp_utils.cppify(
log_likelihood_optimizer.objective_function.noise_variance),
cpp_utils.cppify(
log_likelihood_optimizer.objective_function.derivatives),
log_likelihood_optimizer.objective_function.num_derivatives,
status,
)
return numpy.array(hyperparameters_opt)
def evaluate_log_likelihood_at_hyperparameter_list(
log_likelihood_evaluator,
hyperparameters_to_evaluate,
max_num_threads=DEFAULT_MAX_NUM_THREADS,
status=None,
):
"""Compute the specified log likelihood measure at each input set of hyperparameters.
Generally Newton or gradient descent is preferred but when they fail to converge this may be the only "robust" option.
This function is also useful for plotting or debugging purposes (just to get a bunch of log likelihood values).
Calls into evaluate_log_likelihood_at_hyperparameter_list() in cpp/GPP_python_model_selection.cpp.
:param log_likelihood_evaluator: object specifying which log likelihood measure to evaluate
:type log_likelihood_evaluator: cpp_wrappers.log_likelihood.LogLikelihood
:param hyperparameters_to_evaluate: the hyperparameters at which to compute the specified log likelihood
:type hyperparameters_to_evaluate: array of float64 with shape (num_to_eval, log_likelihood_evaluator.num_hyperparameters)
:param max_num_threads: maximum number of threads to use, >= 1
:type max_num_threads: int > 0
:param status: (output) status messages from C++ (e.g., reporting on optimizer success, etc.)
:type status: dict
:return: log likelihood value at each specified set of hyperparameters
:rtype: array of float64 with shape (hyperparameters_to_evaluate.shape[0])
"""
# status must be an initialized dict for the call to C++.
if status is None:
status = {}
# We could just call log_likelihood_evaluator.compute_log_likelihood() in a loop, but instead we do
# the looping in C++ where it can be multithreaded.
log_likelihood_list = C_GP.evaluate_log_likelihood_at_hyperparameter_list(
cpp_utils.cppify(hyperparameters_to_evaluate),
cpp_utils.cppify(log_likelihood_evaluator._points_sampled),
cpp_utils.cppify(log_likelihood_evaluator._points_sampled_value),
log_likelihood_evaluator.dim,
log_likelihood_evaluator._num_sampled,
log_likelihood_evaluator.objective_type,
cpp_utils.cppify_hyperparameters(
log_likelihood_evaluator.cov_hyperparameters),
cpp_utils.cppify(log_likelihood_evaluator.noise_variance),
cpp_utils.cppify(log_likelihood_evaluator.derivatives),
log_likelihood_evaluator.num_derivatives,
hyperparameters_to_evaluate.shape[0],
max_num_threads,
status,
)
return numpy.array(log_likelihood_list)
def compute_log_likelihood(self):
r"""Compute the objective_type measure at the specified hyperparameters.
:return: value of log_likelihood evaluated at hyperparameters (``LL(y | X, \theta)``)
:rtype: float64
"""
return C_GP.compute_log_likelihood(
cpp_utils.cppify(self._points_sampled),
cpp_utils.cppify(self._points_sampled_value),
self.dim,
self._num_sampled,
self.objective_type,
cpp_utils.cppify_hyperparameters(self.hyperparameters),
cpp_utils.cppify(self._points_sampled_noise_variance),
)
def compute_log_likelihood(self):
r"""Compute the objective_type measure at the specified hyperparameters.
:return: value of log_likelihood evaluated at hyperparameters (``LL(y | X, \theta)``)
:rtype: float64
"""
return C_GP.compute_log_likelihood(
cpp_utils.cppify(self._points_sampled),
cpp_utils.cppify(self._points_sampled_value),
self.dim,
self._num_sampled,
self.objective_type,
cpp_utils.cppify_hyperparameters(self.hyperparameters),
cpp_utils.cppify(self._points_sampled_noise_variance),
)
def compute_grad_log_likelihood(self):
r"""Compute the gradient (wrt hyperparameters) of the objective_type measure at the specified hyperparameters.
:return: grad_log_likelihood: i-th entry is ``\pderiv{LL(y | X, \theta)}{\theta_i}``
:rtype: array of float64 with shape (num_hyperparameters)
"""
grad_log_marginal = C_GP.compute_hyperparameter_grad_log_likelihood(
cpp_utils.cppify(self._points_sampled),
cpp_utils.cppify(self._points_sampled_value),
self.dim,
self._num_sampled,
self.objective_type,
cpp_utils.cppify_hyperparameters(self.hyperparameters),
cpp_utils.cppify(self._points_sampled_noise_variance),
)
return numpy.array(grad_log_marginal)
def compute_grad_log_likelihood(self):
r"""Compute the gradient (wrt hyperparameters) of the objective_type measure at the specified hyperparameters.
:return: grad_log_likelihood: i-th entry is ``\pderiv{LL(y | X, \theta)}{\theta_i}``
:rtype: array of float64 with shape (num_hyperparameters)
"""
grad_log_marginal = C_GP.compute_hyperparameter_grad_log_likelihood(
cpp_utils.cppify(self._points_sampled),
cpp_utils.cppify(self._points_sampled_value),
self.dim,
self._num_sampled,
self.objective_type,
cpp_utils.cppify_hyperparameters(self.hyperparameters),
cpp_utils.cppify(self._points_sampled_noise_variance),
)
return numpy.array(grad_log_marginal)
def evaluate_log_likelihood_at_hyperparameter_list(
log_likelihood_evaluator,
hyperparameters_to_evaluate,
max_num_threads=DEFAULT_MAX_NUM_THREADS,
status=None,
):
"""Compute the specified log likelihood measure at each input set of hyperparameters.
Generally Newton or gradient descent is preferred but when they fail to converge this may be the only "robust" option.
This function is also useful for plotting or debugging purposes (just to get a bunch of log likelihood values).
Calls into evaluate_log_likelihood_at_hyperparameter_list() in cpp/GPP_python_model_selection.cpp.
:param log_likelihood_evaluator: object specifying which log likelihood measure to evaluate
:type log_likelihood_evaluator: cpp_wrappers.log_likelihood.LogLikelihood
:param hyperparameters_to_evaluate: the hyperparameters at which to compute the specified log likelihood
:type hyperparameters_to_evaluate: array of float64 with shape (num_to_eval, log_likelihood_evaluator.num_hyperparameters)
:param max_num_threads: maximum number of threads to use, >= 1
:type max_num_threads: int > 0
:param status: (output) status messages from C++ (e.g., reporting on optimizer success, etc.)
:type status: dict
:return: log likelihood value at each specified set of hyperparameters
:rtype: array of float64 with shape (hyperparameters_to_evaluate.shape[0])
"""
# status must be an initialized dict for the call to C++.
if status is None:
status = {}
# We could just call log_likelihood_evaluator.compute_log_likelihood() in a loop, but instead we do
# the looping in C++ where it can be multithreaded.
log_likelihood_list = C_GP.evaluate_log_likelihood_at_hyperparameter_list(
cpp_utils.cppify(hyperparameters_to_evaluate),
cpp_utils.cppify(log_likelihood_evaluator._points_sampled),
cpp_utils.cppify(log_likelihood_evaluator._points_sampled_value),
log_likelihood_evaluator.dim,
log_likelihood_evaluator._num_sampled,
log_likelihood_evaluator.objective_type,
cpp_utils.cppify_hyperparameters(log_likelihood_evaluator.hyperparameters),
cpp_utils.cppify(log_likelihood_evaluator._points_sampled_noise_variance),
hyperparameters_to_evaluate.shape[0],
max_num_threads,
status,
)
return numpy.array(log_likelihood_list)
def __init__(self, covariance_function, historical_data):
"""Construct a GaussianProcess object that knows how to call C++ for evaluation of member functions.
:param covariance_function: covariance object encoding assumptions about the GP's behavior on our data
:type covariance_function: :class:`moe.optimal_learning.python.interfaces.covariance_interface.CovarianceInterface` subclass
(e.g., from :mod:`moe.optimal_learning.python.cpp_wrappers.covariance`).
:param historical_data: object specifying the already-sampled points, the objective value at those points, and the noise variance associated with each observation
:type historical_data: :class:`moe.optimal_learning.python.data_containers.HistoricalData` object
"""
self._covariance = copy.deepcopy(covariance_function)
self._historical_data = copy.deepcopy(historical_data)
# C++ will maintain its own copy of the contents of hyperparameters and historical_data
self._gaussian_process = C_GP.GaussianProcess(
cpp_utils.cppify_hyperparameters(self._covariance.hyperparameters),
cpp_utils.cppify(historical_data.points_sampled),
cpp_utils.cppify(historical_data.points_sampled_value),
cpp_utils.cppify(historical_data.points_sampled_noise_variance),
self._historical_data.dim,
self._historical_data.num_sampled,
)
def __init__(self, covariance_function, historical_data):
"""Construct a GaussianProcess object that knows how to call C++ for evaluation of member functions.
:param covariance_function: covariance object encoding assumptions about the GP's behavior on our data
:type covariance_function: :class:`moe.optimal_learning.python.interfaces.covariance_interface.CovarianceInterface` subclass
(e.g., from :mod:`moe.optimal_learning.python.cpp_wrappers.covariance`).
:param historical_data: object specifying the already-sampled points, the objective value at those points, and the noise variance associated with each observation
:type historical_data: :class:`moe.optimal_learning.python.data_containers.HistoricalData` object
"""
self._covariance = copy.deepcopy(covariance_function)
self._historical_data = copy.deepcopy(historical_data)
# C++ will maintain its own copy of the contents of hyperparameters and historical_data
self._gaussian_process = C_GP.GaussianProcess(
cpp_utils.cppify_hyperparameters(self._covariance.hyperparameters),
cpp_utils.cppify(historical_data.points_sampled),
cpp_utils.cppify(historical_data.points_sampled_value),
cpp_utils.cppify(historical_data.points_sampled_noise_variance),
self._historical_data.dim,
self._historical_data.num_sampled,
)
def multistart_hyperparameter_optimization(
log_likelihood_optimizer,
num_multistarts,
randomness=None,
max_num_threads=DEFAULT_MAX_NUM_THREADS,
status=None,
):
r"""Select the hyperparameters that maximize the specified log likelihood measure of model fit (over the historical data) within the specified domain.
.. Note:: The following comments are copied to
:mod:`moe.optimal_learning.python.python_version.log_likelihood.multistart_hyperparameter_optimization`.
See :class:`moe.optimal_learning.python.cpp_wrappers.log_likelihood.GaussianProcessLogMarginalLikelihood` and
:class:`moe.optimal_learning.python.cpp_wrappers.log_likelihood.GaussianProcessLeaveOneOutLogLikelihood`
for an overview of some example log likelihood-like measures.
Optimizers are: null ('dumb' search), gradient descent, newton
Newton is the suggested optimizer.
'dumb' search means this will just evaluate the objective log likelihood measure at num_multistarts 'points'
(hyperparameters) in the domain, uniformly sampled using latin hypercube sampling.
The hyperparameter_optimizer_parameters input specifies the desired optimization technique as well as parameters controlling
its behavior (see :mod:`moe.optimal_learning.python.cpp_wrappers.optimization`).
See gpp_python_common.cpp for C++ enum declarations laying out the options for objective and optimizer types.
Currently, during optimization, we recommend that the coordinates of the initial guesses not differ from the
coordinates of the optima by more than about 1 order of magnitude. This is a very (VERY!) rough guideline for
sizing the domain and gd_parameters.num_multistarts; i.e., be wary of sets of initial guesses that cover the space too sparsely.
Note that the domain here must be specified in LOG-10 SPACE!
Solution is guaranteed to lie within the region specified by "domain"; note that this may not be a
true optima (i.e., the gradient may be substantially nonzero).
.. WARNING:: this function fails if NO improvement can be found! In that case,
the output will always be the first randomly chosen point. status will report failure.
:param ei_optimizer: object that optimizes (e.g., gradient descent, newton) log likelihood over a domain
:type ei_optimizer: cpp_wrappers.optimization.*Optimizer object
:param num_multistarts: number of times to multistart ``ei_optimizer`` (UNUSED, data is in log_likelihood_optimizer.optimizer_parameters)
:type num_multistarts: int > 0
:param randomness: RNGs used by C++ to generate initial guesses
:type randomness: RandomnessSourceContainer (C++ object; e.g., from C_GP.RandomnessSourceContainer())
:param max_num_threads: maximum number of threads to use, >= 1
:type max_num_threads: int > 0
:param status: (output) status messages from C++ (e.g., reporting on optimizer success, etc.)
:type status: dict
:return: hyperparameters that maximize the specified log likelihood measure within the specified domain
:rtype: array of float64 with shape (log_likelihood_optimizer.objective_function.num_hyperparameters)
"""
# Create enough randomness sources if none are specified.
if randomness is None:
randomness = C_GP.RandomnessSourceContainer(max_num_threads)
# Set seed based on less repeatable factors (e.g,. time)
randomness.SetRandomizedUniformGeneratorSeed(0)
randomness.SetRandomizedNormalRNGSeed(0)
# status must be an initialized dict for the call to C++.
if status is None:
status = {}
# C++ expects the domain in log10 space and in list form
domain_bounds_log10 = numpy.log10(log_likelihood_optimizer.domain._domain_bounds)
hyperparameters_opt = C_GP.multistart_hyperparameter_optimization(
log_likelihood_optimizer.optimizer_parameters,
cpp_utils.cppify(domain_bounds_log10),
cpp_utils.cppify(log_likelihood_optimizer.objective_function._points_sampled),
cpp_utils.cppify(log_likelihood_optimizer.objective_function._points_sampled_value),
log_likelihood_optimizer.objective_function.dim,
log_likelihood_optimizer.objective_function._num_sampled,
cpp_utils.cppify_hyperparameters(log_likelihood_optimizer.objective_function.hyperparameters),
cpp_utils.cppify(log_likelihood_optimizer.objective_function._points_sampled_noise_variance),
max_num_threads,
randomness,
status,
)
return numpy.array(hyperparameters_opt)
def multistart_hyperparameter_optimization(
log_likelihood_optimizer,
num_multistarts,
randomness=None,
max_num_threads=DEFAULT_MAX_NUM_THREADS,
status=None,
):
r"""Select the hyperparameters that maximize the specified log likelihood measure of model fit (over the historical data) within the specified domain.
.. Note:: The following comments are copied to
:mod:`moe.optimal_learning.python.python_version.log_likelihood.multistart_hyperparameter_optimization`.
See :class:`moe.optimal_learning.python.cpp_wrappers.log_likelihood.GaussianProcessLogMarginalLikelihood` and
:class:`moe.optimal_learning.python.cpp_wrappers.log_likelihood.GaussianProcessLeaveOneOutLogLikelihood`
for an overview of some example log likelihood-like measures.
Optimizers are: null ('dumb' search), gradient descent, newton
Newton is the suggested optimizer.
'dumb' search means this will just evaluate the objective log likelihood measure at num_multistarts 'points'
(hyperparameters) in the domain, uniformly sampled using latin hypercube sampling.
The hyperparameter_optimizer_parameters input specifies the desired optimization technique as well as parameters controlling
its behavior (see :mod:`moe.optimal_learning.python.cpp_wrappers.optimization`).
See gpp_python_common.cpp for C++ enum declarations laying out the options for objective and optimizer types.
Currently, during optimization, we recommend that the coordinates of the initial guesses not differ from the
coordinates of the optima by more than about 1 order of magnitude. This is a very (VERY!) rough guideline for
sizing the domain and gd_parameters.num_multistarts; i.e., be wary of sets of initial guesses that cover the space too sparsely.
Note that the domain here must be specified in LOG-10 SPACE!
Solution is guaranteed to lie within the region specified by "domain"; note that this may not be a
true optima (i.e., the gradient may be substantially nonzero).
.. WARNING:: this function fails if NO improvement can be found! In that case,
the output will always be the first randomly chosen point. status will report failure.
:param log_likelihood_optimizer: object that optimizes (e.g., gradient descent, newton) log likelihood over a domain
:type log_likelihood_optimizer: cpp_wrappers.optimization.*Optimizer object
:param num_multistarts: number of times to multistart ``log_likelihood_optimizer`` (UNUSED, data is in log_likelihood_optimizer.optimizer_parameters)
:type num_multistarts: int > 0
:param randomness: RNGs used by C++ to generate initial guesses
:type randomness: RandomnessSourceContainer (C++ object; e.g., from C_GP.RandomnessSourceContainer())
:param max_num_threads: maximum number of threads to use, >= 1
:type max_num_threads: int > 0
:param status: (output) status messages from C++ (e.g., reporting on optimizer success, etc.)
:type status: dict
:return: hyperparameters that maximize the specified log likelihood measure within the specified domain
:rtype: array of float64 with shape (log_likelihood_optimizer.objective_function.num_hyperparameters)
"""
# Create enough randomness sources if none are specified.
if randomness is None:
randomness = C_GP.RandomnessSourceContainer(max_num_threads)
# Set seed based on less repeatable factors (e.g,. time)
randomness.SetRandomizedUniformGeneratorSeed(0)
randomness.SetRandomizedNormalRNGSeed(0)
# status must be an initialized dict for the call to C++.
if status is None:
status = {}
# C++ expects the domain in log10 space and in list form
domain_bounds_log10 = numpy.log10(log_likelihood_optimizer.domain._domain_bounds)
hyperparameters_opt = C_GP.multistart_hyperparameter_optimization(
log_likelihood_optimizer.optimizer_parameters,
cpp_utils.cppify(domain_bounds_log10),
cpp_utils.cppify(log_likelihood_optimizer.objective_function._points_sampled),
cpp_utils.cppify(log_likelihood_optimizer.objective_function._points_sampled_value),
log_likelihood_optimizer.objective_function.dim,
log_likelihood_optimizer.objective_function._num_sampled,
cpp_utils.cppify_hyperparameters(log_likelihood_optimizer.objective_function.hyperparameters),
cpp_utils.cppify(log_likelihood_optimizer.objective_function._points_sampled_noise_variance),
max_num_threads,
randomness,
status,
)
return numpy.array(hyperparameters_opt)
