def test_invalid_points_sampled_input(self):
"""Test that duplicate points_sampled (via GP_NEXT_POINTS_EPI_ENDPOINT) generate expected Response with error message."""
endpoint = GP_NEXT_POINTS_EPI_ENDPOINT
dict_payload = copy.deepcopy(
GpNextPointsPrettyView._pretty_default_request)
# Invalidate historical info: 0.0 noise and add a duplicate point
for sample_point in dict_payload['gp_historical_info'][
'points_sampled']:
sample_point['value_var'] = 0.0
dict_payload['gp_historical_info']['points_sampled'].append(
dict_payload['gp_historical_info']['points_sampled'][0])
result = self.testapp.post(endpoint,
json.dumps(dict_payload),
expect_errors=True)
# Get the exception that arises from processing invalid hyperparameters
request_schema = GpNextPointsRequest()
params = request_schema.deserialize(dict_payload)
try:
_make_gp_from_params(params)
except Exception as request_exception:
pass
T.assert_equal(result.body,
general_error(request_exception, result.request).body)
def gp_mean_var_response_dict(self, compute_mean=False, compute_var=False, var_diag=False):
"""Produce a response dict (to be serialized) for gp_mean, gp_var, gp_mean_var, and _diag POST requests.
:param compute_mean: whether to compute the GP mean
:type compute_mean: bool
:param compute_var: whether to compute the GP variance or covariance
:type compute_var: bool
:param var_diag: whether to compute the full GP covariance or just the variance terms
:type var_diag: bool
:return: dict with 'endpoint' and optionally 'mean' and 'var' keys depending on inputs
:rtype: dict
"""
params = self.get_params_from_request()
points_to_evaluate = numpy.array(params.get('points_to_evaluate'))
gaussian_process = _make_gp_from_params(params)
response_dict = {}
response_dict['endpoint'] = self._route_name
with timing_context(MEAN_VAR_COMPUTATION_TIMING_LABEL):
if compute_mean:
response_dict['mean'] = gaussian_process.compute_mean_of_points(points_to_evaluate).tolist()
if compute_var:
if var_diag:
response_dict['var'] = numpy.diag(
gaussian_process.compute_variance_of_points(points_to_evaluate)
).tolist()
else:
response_dict['var'] = gaussian_process.compute_variance_of_points(points_to_evaluate).tolist()
return response_dict
def gp_hyper_opt_view(self):
"""Endpoint for gp_hyper_opt POST requests.
.. http:post:: /gp/hyper_opt
Calculates the optimal hyperparameters for a gaussian process, given historical data.
:input: :class:`moe.views.schemas.rest.gp_hyper_opt.GpHyperOptRequest`
:output: :class:`moe.views.schemas.rest.gp_hyper_opt.GpHyperOptResponse`
:status 200: returns a response
:status 500: server error
"""
params = self.get_params_from_request()
max_num_threads = params.get('max_num_threads')
hyperparameter_domain = _make_domain_from_params(params, domain_info_key='hyperparameter_domain_info')
gaussian_process = _make_gp_from_params(params)
covariance_of_process, historical_data = gaussian_process.get_core_data_copy()
optimizer_class, optimizer_parameters, num_random_samples = _make_optimizer_parameters_from_params(params)
log_likelihood_type = params.get('log_likelihood_info')
log_likelihood_eval = LOG_LIKELIHOOD_TYPES_TO_LOG_LIKELIHOOD_METHODS[log_likelihood_type].log_likelihood_class(
covariance_of_process,
historical_data,
)
log_likelihood_optimizer = optimizer_class(
hyperparameter_domain,
log_likelihood_eval,
optimizer_parameters,
num_random_samples=num_random_samples,
)
hyperopt_status = {}
with timing_context(MODEL_SELECTION_TIMING_LABEL):
optimized_hyperparameters = multistart_hyperparameter_optimization(
log_likelihood_optimizer,
optimizer_parameters.num_multistarts,
max_num_threads=max_num_threads,
status=hyperopt_status,
)
covariance_of_process.hyperparameters = optimized_hyperparameters
log_likelihood_eval.current_point = optimized_hyperparameters
return self.form_response({
'endpoint': self._route_name,
'covariance_info': covariance_of_process.get_json_serializable_info(),
'status': {
'log_likelihood': log_likelihood_eval.compute_log_likelihood(),
'grad_log_likelihood': log_likelihood_eval.compute_grad_log_likelihood().tolist(),
'optimizer_success': hyperopt_status,
},
})
def test_invalid_points_sampled_input(self):
"""Test that duplicate points_sampled (via GP_NEXT_POINTS_EPI_ENDPOINT) generate expected Response with error message."""
endpoint = GP_NEXT_POINTS_EPI_ENDPOINT
dict_payload = copy.deepcopy(GpNextPointsPrettyView._pretty_default_request)
# Invalidate historical info: 0.0 noise and add a duplicate point
for sample_point in dict_payload['gp_historical_info']['points_sampled']:
sample_point['value_var'] = 0.0
dict_payload['gp_historical_info']['points_sampled'].append(dict_payload['gp_historical_info']['points_sampled'][0])
result = self.testapp.post(endpoint, json.dumps(dict_payload), expect_errors=True)
# Get the exception that arises from processing invalid hyperparameters
request_schema = GpNextPointsRequest()
params = request_schema.deserialize(dict_payload)
with pytest.raises(Exception) as request_exception:
_make_gp_from_params(params)
assert result.body == general_error(request_exception.value, result.request).body
def get_lie_value(self, params):
"""Return the lie value associated with the lie_method, unless lie_value is explicitly given."""
if params.get('lie_value') is not None:
return params.get('lie_value')
gaussian_process = _make_gp_from_params(params)
points_sampled_values = gaussian_process._historical_data._points_sampled_value.tolist()
if params.get('lie_method') == CONSTANT_LIAR_MIN:
return numpy.amin(points_sampled_values)
elif params.get('lie_method') == CONSTANT_LIAR_MAX:
return numpy.amax(points_sampled_values)
elif params.get('lie_method') == CONSTANT_LIAR_MEAN:
return numpy.mean(points_sampled_values)
else:
raise(NotImplementedError, '{0} is not implemented'.format(params.get('lie_method')))
def get_lie_value(self, params):
"""Return the lie value associated with the lie_method, unless lie_value is explicitly given."""
if params.get("lie_value") is not None:
return params.get("lie_value")
gaussian_process = _make_gp_from_params(params)
points_sampled_values = gaussian_process._historical_data._points_sampled_value.tolist()
if params.get("lie_method") == CONSTANT_LIAR_MIN:
return numpy.amin(points_sampled_values)
elif params.get("lie_method") == CONSTANT_LIAR_MAX:
return numpy.amax(points_sampled_values)
elif params.get("lie_method") == CONSTANT_LIAR_MEAN:
return numpy.mean(points_sampled_values)
else:
raise (NotImplementedError, "{0} is not implemented".format(params.get("lie_method")))
def gp_mean_var_response_dict(self,
compute_mean=False,
compute_var=False,
var_diag=False):
"""Produce a response dict (to be serialized) for gp_mean, gp_var, gp_mean_var, and _diag POST requests.
:param compute_mean: whether to compute the GP mean
:type compute_mean: bool
:param compute_var: whether to compute the GP variance or covariance
:type compute_var: bool
:param var_diag: whether to compute the full GP covariance or just the variance terms
:type var_diag: bool
:return: dict with 'endpoint' and optionally 'mean' and 'var' keys depending on inputs
:rtype: dict
"""
params = self.get_params_from_request()
points_to_evaluate = numpy.array(params.get('points_to_evaluate'))
gaussian_process = _make_gp_from_params(params)
response_dict = {}
response_dict['endpoint'] = self._route_name
with timing_context(MEAN_VAR_COMPUTATION_TIMING_LABEL):
if compute_mean:
response_dict[
'mean'] = gaussian_process.compute_mean_of_points(
points_to_evaluate).tolist()
if compute_var:
if var_diag:
response_dict['var'] = numpy.diag(
gaussian_process.compute_variance_of_points(
points_to_evaluate)).tolist()
else:
response_dict[
'var'] = gaussian_process.compute_variance_of_points(
points_to_evaluate).tolist()
return response_dict
def gp_ei_view(self):
"""Endpoint for gp_ei POST requests.
.. http:post:: /gp/ei
Calculates the Expected Improvement (EI) of a set of points, given historical data.
:input: :class:`moe.views.schemas.GpEiRequest`
:output: :class:`moe.views.schemas.GpEiResponse`
:status 201: returns a response
:status 500: server error
"""
params = self.get_params_from_request()
# TODO(GH-99): Change REST interface to give points_to_evaluate with shape
# (num_to_evaluate, num_to_sample, dim)
# Here we assume the shape is (num_to_evaluate, dim) so we insert an axis, making num_to_sample = 1.
points_to_evaluate = numpy.array(params.get('points_to_evaluate'))[:, numpy.newaxis, :]
points_being_sampled = numpy.array(params.get('points_being_sampled'))
num_mc_iterations = params.get('mc_iterations')
max_num_threads = params.get('max_num_threads')
gaussian_process = _make_gp_from_params(params)
expected_improvement_evaluator = ExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
with timing_context(EI_COMPUTATION_TIMING_LABEL):
expected_improvement = expected_improvement_evaluator.evaluate_at_point_list(
points_to_evaluate,
max_num_threads=max_num_threads,
)
return self.form_response({
'endpoint': self._route_name,
'expected_improvement': expected_improvement.tolist(),
})
def gp_ei_view(self):
"""Endpoint for gp_ei POST requests.
.. http:post:: /gp/ei
Calculates the Expected Improvement (EI) of a set of points, given historical data.
:input: :class:`moe.views.schemas.rest.GpEiRequest`
:output: :class:`moe.views.schemas.rest.GpEiResponse`
:status 200: returns a response
:status 500: server error
"""
params = self.get_params_from_request()
# TODO(GH-99): Change REST interface to give points_to_evaluate with shape
# (num_to_evaluate, num_to_sample, dim)
# Here we assume the shape is (num_to_evaluate, dim) so we insert an axis, making num_to_sample = 1.
points_to_evaluate = numpy.array(params.get('points_to_evaluate'))[:, numpy.newaxis, :]
points_being_sampled = numpy.array(params.get('points_being_sampled'))
num_mc_iterations = params.get('mc_iterations')
max_num_threads = params.get('max_num_threads')
gaussian_process = _make_gp_from_params(params)
expected_improvement_evaluator = ExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
with timing_context(EI_COMPUTATION_TIMING_LABEL):
expected_improvement = expected_improvement_evaluator.evaluate_at_point_list(
points_to_evaluate,
max_num_threads=max_num_threads,
)
return self.form_response({
'endpoint': self._route_name,
'expected_improvement': expected_improvement.tolist(),
})
def compute_next_points_to_sample_response(self, params, optimizer_method_name, route_name, *args, **kwargs):
"""Compute the next points to sample (and their expected improvement) using optimizer_method_name from params in the request.
.. Warning:: Attempting to find ``num_to_sample`` optimal points with
``num_sampled < num_to_sample`` historical points sampled can cause matrix issues under
some conditions. Try requesting ``num_to_sample < num_sampled`` points for better
performance. To bootstrap more points try sampling at random, or from a grid.
:param request_params: the deserialized REST request, containing ei_optimizer_parameters and gp_historical_info
:type request_params: a deserialized self.request_schema object as a dict
:param optimizer_method_name: the optimization method to use
:type optimizer_method_name: string in :const:`moe.views.constant.NEXT_POINTS_OPTIMIZER_METHOD_NAMES`
:param route_name: name of the route being called
:type route_name: string in :const:`moe.views.constant.ALL_REST_ROUTES_ROUTE_NAME_TO_ENDPOINT`
:param ``*args``: extra args to be passed to optimization method
:param ``**kwargs``: extra kwargs to be passed to optimization method
"""
points_being_sampled = numpy.array(params.get('points_being_sampled'))
num_to_sample = params.get('num_to_sample')
num_mc_iterations = params.get('mc_iterations')
max_num_threads = params.get('max_num_threads')
gaussian_process = _make_gp_from_params(params)
ei_opt_status = {}
# TODO(GH-89): Make the optimal_learning library handle this case 'organically' with
# reasonable default behavior and remove hacks like this one.
if gaussian_process.num_sampled == 0:
# If there is no initial data we bootstrap with random points
py_domain = _make_domain_from_params(params, python_version=True)
next_points = py_domain.generate_uniform_random_points_in_domain(num_to_sample)
ei_opt_status['found_update'] = True
expected_improvement_evaluator = PythonExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
else:
# Calculate the next best points to sample given the historical data
optimizer_class, optimizer_parameters, num_random_samples = _make_optimizer_parameters_from_params(params)
if optimizer_class == python_optimization.LBFGSBOptimizer:
domain = RepeatedDomain(num_to_sample, _make_domain_from_params(params, python_version=True))
expected_improvement_evaluator = PythonExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
mvndst_parameters=_make_mvndst_parameters_from_params(params)
)
opt_method = getattr(moe.optimal_learning.python.python_version.expected_improvement, optimizer_method_name)
else:
domain = _make_domain_from_params(params, python_version=False)
expected_improvement_evaluator = ExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
opt_method = getattr(moe.optimal_learning.python.cpp_wrappers.expected_improvement, optimizer_method_name)
expected_improvement_optimizer = optimizer_class(
domain,
expected_improvement_evaluator,
optimizer_parameters,
num_random_samples=num_random_samples,
)
with timing_context(EPI_OPTIMIZATION_TIMING_LABEL):
next_points = opt_method(
expected_improvement_optimizer,
params.get('optimizer_info')['num_multistarts'], # optimizer_parameters.num_multistarts,
num_to_sample,
max_num_threads=max_num_threads,
status=ei_opt_status,
*args,
**kwargs
)
# TODO(GH-285): Use analytic q-EI here
# TODO(GH-314): Need to resolve poential issue with NaNs before using q-EI here
# It may be sufficient to check found_update == False in ei_opt_status
# and then use q-EI, else set EI = 0.
expected_improvement_evaluator.current_point = next_points
# The C++ may fail to compute EI with some ``next_points`` inputs (e.g.,
# ``points_to_sample`` and ``points_begin_sampled`` are too close
# together or too close to ``points_sampled``). We catch the exception when this happens
# and attempt a more numerically robust option.
try:
expected_improvement = expected_improvement_evaluator.compute_expected_improvement()
except Exception as exception:
self.log.info('EI computation failed, probably b/c GP-variance matrix is singular. Error: {0:s}'.format(exception))
# ``_compute_expected_improvement_monte_carlo`` in
# :class:`moe.optimal_learning.python.python_version.expected_improvement.ExpectedImprovement`
# has a more reliable (but very expensive) way to deal with singular variance matrices.
python_ei_eval = PythonExpectedImprovement(
expected_improvement_evaluator._gaussian_process,
points_to_sample=next_points,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
expected_improvement = python_ei_eval.compute_expected_improvement(force_monte_carlo=True)
return self.form_response({
'endpoint': route_name,
'points_to_sample': next_points.tolist(),
'status': {
'expected_improvement': expected_improvement,
'optimizer_success': ei_opt_status,
},
})
def compute_next_points_to_sample_response(self, params,
optimizer_method_name,
route_name, *args, **kwargs):
"""Compute the next points to sample (and their expected improvement) using optimizer_method_name from params in the request.
.. Warning:: Attempting to find ``num_to_sample`` optimal points with
``num_sampled < num_to_sample`` historical points sampled can cause matrix issues under
some conditions. Try requesting ``num_to_sample < num_sampled`` points for better
performance. To bootstrap more points try sampling at random, or from a grid.
:param request_params: the deserialized REST request, containing ei_optimizer_parameters and gp_historical_info
:type request_params: a deserialized self.request_schema object as a dict
:param optimizer_method_name: the optimization method to use
:type optimizer_method_name: string in :const:`moe.views.constant.NEXT_POINTS_OPTIMIZER_METHOD_NAMES`
:param route_name: name of the route being called
:type route_name: string in :const:`moe.views.constant.ALL_REST_ROUTES_ROUTE_NAME_TO_ENDPOINT`
:param ``*args``: extra args to be passed to optimization method
:param ``**kwargs``: extra kwargs to be passed to optimization method
"""
points_being_sampled = numpy.array(params.get('points_being_sampled'))
num_to_sample = params.get('num_to_sample')
num_mc_iterations = params.get('mc_iterations')
max_num_threads = params.get('max_num_threads')
gaussian_process = _make_gp_from_params(params)
ei_opt_status = {}
# TODO(GH-89): Make the optimal_learning library handle this case 'organically' with
# reasonable default behavior and remove hacks like this one.
if gaussian_process.num_sampled == 0:
# If there is no initial data we bootstrap with random points
py_domain = _make_domain_from_params(params, python_version=True)
next_points = py_domain.generate_uniform_random_points_in_domain(
num_to_sample)
ei_opt_status['found_update'] = True
expected_improvement_evaluator = PythonExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
else:
# Calculate the next best points to sample given the historical data
optimizer_class, optimizer_parameters, num_random_samples = _make_optimizer_parameters_from_params(
params)
if optimizer_class == python_optimization.LBFGSBOptimizer:
domain = RepeatedDomain(
num_to_sample,
_make_domain_from_params(params, python_version=True))
expected_improvement_evaluator = PythonExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
mvndst_parameters=_make_mvndst_parameters_from_params(
params))
opt_method = getattr(
moe.optimal_learning.python.python_version.
expected_improvement, optimizer_method_name)
else:
domain = _make_domain_from_params(params, python_version=False)
expected_improvement_evaluator = ExpectedImprovement(
gaussian_process,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
opt_method = getattr(
moe.optimal_learning.python.cpp_wrappers.
expected_improvement, optimizer_method_name)
expected_improvement_optimizer = optimizer_class(
domain,
expected_improvement_evaluator,
optimizer_parameters,
num_random_samples=num_random_samples,
)
with timing_context(EPI_OPTIMIZATION_TIMING_LABEL):
next_points = opt_method(
expected_improvement_optimizer,
params.get('optimizer_info')
['num_multistarts'], # optimizer_parameters.num_multistarts,
num_to_sample,
max_num_threads=max_num_threads,
status=ei_opt_status,
*args,
**kwargs)
# TODO(GH-285): Use analytic q-EI here
# TODO(GH-314): Need to resolve poential issue with NaNs before using q-EI here
# It may be sufficient to check found_update == False in ei_opt_status
# and then use q-EI, else set EI = 0.
expected_improvement_evaluator.current_point = next_points
# The C++ may fail to compute EI with some ``next_points`` inputs (e.g.,
# ``points_to_sample`` and ``points_begin_sampled`` are too close
# together or too close to ``points_sampled``). We catch the exception when this happens
# and attempt a more numerically robust option.
try:
expected_improvement = expected_improvement_evaluator.compute_expected_improvement(
)
except Exception as exception:
self.log.info(
'EI computation failed, probably b/c GP-variance matrix is singular. Error: {0:s}'
.format(exception))
# ``_compute_expected_improvement_monte_carlo`` in
# :class:`moe.optimal_learning.python.python_version.expected_improvement.ExpectedImprovement`
# has a more reliable (but very expensive) way to deal with singular variance matrices.
python_ei_eval = PythonExpectedImprovement(
expected_improvement_evaluator._gaussian_process,
points_to_sample=next_points,
points_being_sampled=points_being_sampled,
num_mc_iterations=num_mc_iterations,
)
expected_improvement = python_ei_eval.compute_expected_improvement(
force_monte_carlo=True)
return self.form_response({
'endpoint': route_name,
'points_to_sample': next_points.tolist(),
'status': {
'expected_improvement': expected_improvement,
'optimizer_success': ei_opt_status,
},
})
