def _check_hyps_convergence(list_hyps,
hyps,
str_cov,
is_fixed_noise,
ratio_threshold=0.05):
assert isinstance(list_hyps, list)
assert isinstance(hyps, dict)
assert isinstance(str_cov, str)
assert isinstance(is_fixed_noise, bool)
assert isinstance(ratio_threshold, float)
is_converged = False
if len(list_hyps) > 0:
hyps_converted = utils_covariance.convert_hyps(
str_cov, hyps, is_fixed_noise=is_fixed_noise)
target_hyps_converted = utils_covariance.convert_hyps(
str_cov, list_hyps[-1], is_fixed_noise=is_fixed_noise)
cur_norm = np.linalg.norm(hyps_converted - target_hyps_converted,
ord=2)
threshold = np.linalg.norm(target_hyps_converted) * ratio_threshold
if np.linalg.norm(hyps_converted - target_hyps_converted,
ord=2) < threshold:
is_converged = True
return is_converged
def test_neg_log_pseudo_l_loocv():
dim_X = 3
str_cov = 'se'
X = np.reshape(np.arange(0, 9), (3, dim_X))
Y = np.expand_dims(np.arange(3, 10, 3), axis=1)
dict_hyps = utils_covariance.get_hyps(str_cov, dim_X)
arr_hyps = utils_covariance.convert_hyps(str_cov,
dict_hyps,
fix_noise=constants.FIX_GP_NOISE)
prior_mu_X = np.zeros((3, 1))
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(np.arange(0, 3), Y, arr_hyps,
str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(X, np.arange(0, 3), arr_hyps,
str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(X, Y, dict_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, 1, prior_mu_X)
with pytest.raises(ValueError) as error:
package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, 'abc',
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, str_cov,
np.arange(0, 3))
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(
np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(
X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(
X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1))
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=1)
with pytest.raises(AssertionError) as error:
package_target.neg_log_pseudo_l_loocv(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
debug=1)
neg_log_pseudo_l_ = package_target.neg_log_pseudo_l_loocv(
X, Y, arr_hyps, str_cov, prior_mu_X)
print(neg_log_pseudo_l_)
truth_log_pseudo_l_ = 21.916822991658695
assert np.abs(neg_log_pseudo_l_ - truth_log_pseudo_l_) < TEST_EPSILON
def _check_hyps_convergence(list_hyps,
hyps,
str_cov,
is_fixed_noise,
ratio_threshold=0.05):
"""
It checks convergence of hyperparameters for Gaussian process regression.
:param list_hyps: list of historical hyperparameters for Gaussian process regression.
:type list_hyps: list
:param hyps: dictionary of hyperparameters for acquisition function.
:type hyps: dict.
:param str_cov: the name of covariance function.
:type str_cov: str.
:param is_fixed_noise: flag for fixing a noise.
:type is_fixed_noise: bool.
:param ratio_threshold: ratio of threshold for checking convergence.
:type ratio_threshold: float, optional
:returns: flag for checking convergence. If converged, it is True.
:rtype: bool.
:raises: AssertionError
"""
assert isinstance(list_hyps, list)
assert isinstance(hyps, dict)
assert isinstance(str_cov, str)
assert isinstance(is_fixed_noise, bool)
assert isinstance(ratio_threshold, float)
is_converged = False
if len(list_hyps) > 0:
hyps_converted = utils_covariance.convert_hyps(
str_cov, hyps, is_fixed_noise=is_fixed_noise)
target_hyps_converted = utils_covariance.convert_hyps(
str_cov, list_hyps[-1], is_fixed_noise=is_fixed_noise)
cur_norm = np.linalg.norm(hyps_converted - target_hyps_converted,
ord=2)
threshold = np.linalg.norm(target_hyps_converted) * ratio_threshold
if np.linalg.norm(hyps_converted - target_hyps_converted,
ord=2) < threshold:
is_converged = True
return is_converged
def test_validate_hyps_arr():
num_dim = 2
str_cov = 'matern32'
cur_hyps = utils_covariance.get_hyps(str_cov, num_dim)
cur_hyps = utils_covariance.convert_hyps(str_cov,
cur_hyps,
is_fixed_noise=False)
with pytest.raises(AssertionError) as error:
_, is_valid = utils_covariance.validate_hyps_arr(123, str_cov, num_dim)
with pytest.raises(AssertionError) as error:
_, is_valid = utils_covariance.validate_hyps_arr(
cur_hyps, 'abc', num_dim)
with pytest.raises(AssertionError) as error:
_, is_valid = utils_covariance.validate_hyps_arr(
cur_hyps, str_cov, 'abc')
def test_log_ml():
dim_X = 3
str_cov = 'se'
X = np.reshape(np.arange(0, 9), (3, dim_X))
Y = np.expand_dims(np.arange(3, 10, 3), axis=1)
dict_hyps = utils_covariance.get_hyps(str_cov, dim_X)
arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, is_fixed_noise=constants.IS_FIXED_GP_NOISE)
prior_mu_X = np.zeros((3, 1))
with pytest.raises(AssertionError) as error:
gp.log_ml(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, dict_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, arr_hyps, 1, prior_mu_X)
with pytest.raises(ValueError) as error:
gp.log_ml(X, Y, arr_hyps, 'abc', prior_mu_X)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3))
with pytest.raises(AssertionError) as error:
gp.log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1))
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_cholesky=1)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=1)
with pytest.raises(AssertionError) as error:
gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1)
log_ml = gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X)
print(log_ml)
truth_log_ml = -65.74995266591506
assert np.abs(log_ml - truth_log_ml) < TEST_EPSILON
log_ml = gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_cholesky=False)
print(log_ml)
truth_log_ml = -65.74995266566506
assert np.abs(log_ml - truth_log_ml) < TEST_EPSILON
def test_validate_hyps_arr():
num_dim = 2
str_cov = 'matern32'
cur_hyps = package_target.get_hyps(str_cov, num_dim)
cur_hyps = package_target.convert_hyps(str_cov, cur_hyps, fix_noise=False)
with pytest.raises(AssertionError) as error:
_, is_valid = package_target.validate_hyps_arr(123, str_cov, num_dim)
with pytest.raises(AssertionError) as error:
_, is_valid = package_target.validate_hyps_arr(cur_hyps, 'abc',
num_dim)
with pytest.raises(AssertionError) as error:
_, is_valid = package_target.validate_hyps_arr(cur_hyps, str_cov,
'abc')
with pytest.raises(AssertionError) as error:
_, is_valid = package_target.validate_hyps_arr(cur_hyps,
str_cov,
num_dim,
use_gp='abc')
def test_convert_hyps():
cur_hyps = {
'noise': 0.1,
'signal': 1.0,
'lengthscales': np.array([1.0, 1.0])
}
with pytest.raises(AssertionError) as error:
utils_covariance.convert_hyps(1.2, 2.1)
with pytest.raises(AssertionError) as error:
utils_covariance.convert_hyps(1.2, dict())
with pytest.raises(AssertionError) as error:
utils_covariance.convert_hyps('se', 2.1)
with pytest.raises(AssertionError) as error:
utils_covariance.convert_hyps('abc', 2.1)
with pytest.raises(AssertionError) as error:
utils_covariance.convert_hyps('abc', cur_hyps)
with pytest.raises(AssertionError) as error:
utils_covariance.convert_hyps('se', dict(), is_fixed_noise=1)
converted_hyps = utils_covariance.convert_hyps('se', cur_hyps)
assert converted_hyps[0] == cur_hyps['noise']
assert converted_hyps[1] == cur_hyps['signal']
assert (converted_hyps[2:] == cur_hyps['lengthscales']).all()
def optimize(
self,
X_train: np.ndarray,
Y_train: np.ndarray,
str_sampling_method: str = constants.STR_SAMPLING_METHOD_AO,
num_samples: int = constants.NUM_SAMPLES_AO,
str_mlm_method: str = constants.STR_MLM_METHOD,
) -> constants.TYPING_TUPLE_ARRAY_DICT:
"""
It computes acquired example, candidates of acquired examples,
acquisition function values for the candidates, covariance matrix,
inverse matrix of the covariance matrix, hyperparameters optimized,
and execution times.
:param X_train: inputs. Shape: (n, d) or (n, m, d).
:type X_train: numpy.ndarray
:param Y_train: outputs. Shape: (n, 1).
:type Y_train: numpy.ndarray
:param str_sampling_method: the name of sampling method for
acquisition function optimization.
:type str_sampling_method: str., optional
:param num_samples: the number of samples.
:type num_samples: int., optional
:param str_mlm_method: the name of marginal likelihood maximization
method for Gaussian process regression.
:type str_mlm_method: str., optional
:returns: acquired example and dictionary of information. Shape: ((d, ), dict.).
:rtype: (numpy.ndarray, dict.)
:raises: AssertionError
"""
assert isinstance(X_train, np.ndarray)
assert isinstance(Y_train, np.ndarray)
assert isinstance(str_sampling_method, str)
assert isinstance(num_samples, int)
assert isinstance(str_mlm_method, str)
assert len(X_train.shape) == 2
assert len(Y_train.shape) == 2
assert Y_train.shape[1] == 1
assert X_train.shape[0] == Y_train.shape[0]
assert X_train.shape[1] == self.num_dim
assert num_samples > 0
assert str_sampling_method in constants.ALLOWED_SAMPLING_METHOD
assert str_mlm_method in constants.ALLOWED_MLM_METHOD
time_start = time.time()
Y_train_orig = Y_train
if self.normalize_Y and str_mlm_method != 'converged':
if self.debug:
self.logger.debug('Responses are normalized.')
Y_train = utils_bo.normalize_min_max(Y_train)
time_start_surrogate = time.time()
if str_mlm_method == 'regular':
cov_X_X, inv_cov_X_X, hyps = gp_kernel.get_optimized_kernel(
X_train,
Y_train,
self.prior_mu,
self.str_cov,
str_optimizer_method=self.str_optimizer_method_gp,
str_modelselection_method=self.str_modelselection_method,
use_ard=self.use_ard,
debug=self.debug)
elif str_mlm_method == 'combined':
from bayeso.gp import gp_likelihood
from bayeso.utils import utils_gp
from bayeso.utils import utils_covariance
prior_mu_train = utils_gp.get_prior_mu(self.prior_mu, X_train)
neg_log_ml_best = np.inf
cov_X_X_best = None
inv_cov_X_X_best = None
hyps_best = None
for cur_str_optimizer_method in ['BFGS', 'Nelder-Mead']:
cov_X_X, inv_cov_X_X, hyps = gp_kernel.get_optimized_kernel(
X_train,
Y_train,
self.prior_mu,
self.str_cov,
str_optimizer_method=cur_str_optimizer_method,
str_modelselection_method=self.str_modelselection_method,
use_ard=self.use_ard,
debug=self.debug)
cur_neg_log_ml_ = gp_likelihood.neg_log_ml(
X_train,
Y_train,
utils_covariance.convert_hyps(
self.str_cov, hyps, fix_noise=constants.FIX_GP_NOISE),
self.str_cov,
prior_mu_train,
use_ard=self.use_ard,
fix_noise=constants.FIX_GP_NOISE,
use_gradient=False,
debug=self.debug)
if cur_neg_log_ml_ < neg_log_ml_best:
neg_log_ml_best = cur_neg_log_ml_
cov_X_X_best = cov_X_X
inv_cov_X_X_best = inv_cov_X_X
hyps_best = hyps
cov_X_X = cov_X_X_best
inv_cov_X_X = inv_cov_X_X_best
hyps = hyps_best
elif str_mlm_method == 'converged':
fix_noise = constants.FIX_GP_NOISE
if self.is_optimize_hyps:
cov_X_X, inv_cov_X_X, hyps = gp_kernel.get_optimized_kernel(
X_train,
Y_train,
self.prior_mu,
self.str_cov,
str_optimizer_method=self.str_optimizer_method_gp,
str_modelselection_method=self.str_modelselection_method,
use_ard=self.use_ard,
debug=self.debug)
self.is_optimize_hyps = not utils_bo.check_hyps_convergence(
self.historical_hyps, hyps, self.str_cov, fix_noise)
else: # pragma: no cover
if self.debug:
self.logger.debug('hyps converged.')
hyps = self.historical_hyps[-1]
cov_X_X, inv_cov_X_X, _ = covariance.get_kernel_inverse(
X_train,
hyps,
self.str_cov,
fix_noise=fix_noise,
debug=self.debug)
else: # pragma: no cover
raise ValueError('optimize: missing condition for str_mlm_method.')
self.historical_hyps.append(hyps)
time_end_surrogate = time.time()
time_start_acq = time.time()
fun_negative_acquisition = lambda X_test: -1.0 * self.compute_acquisitions(
X_test, X_train, Y_train, cov_X_X, inv_cov_X_X, hyps)
next_point, next_points = self._optimize(
fun_negative_acquisition,
str_sampling_method=str_sampling_method,
num_samples=num_samples)
time_end_acq = time.time()
acquisitions = fun_negative_acquisition(next_points)
time_end = time.time()
dict_info = {
'next_points': next_points,
'acquisitions': acquisitions,
'Y_original': Y_train_orig,
'Y_normalized': Y_train,
'cov_X_X': cov_X_X,
'inv_cov_X_X': inv_cov_X_X,
'hyps': hyps,
'time_surrogate': time_end_surrogate - time_start_surrogate,
'time_acq': time_end_acq - time_start_acq,
'time_overall': time_end - time_start,
}
if self.debug:
self.logger.debug('overall time consumed to acquire: %.4f sec.',
time_end - time_start)
return next_point, dict_info
def test_neg_log_ml():
dim_X = 3
str_cov = 'se'
X = np.reshape(np.arange(0, 9), (3, dim_X))
Y = np.expand_dims(np.arange(3, 10, 3), axis=1)
is_fixed_noise = False
dict_hyps = utils_covariance.get_hyps(str_cov, dim_X)
arr_hyps = utils_covariance.convert_hyps(str_cov,
dict_hyps,
is_fixed_noise=is_fixed_noise)
prior_mu_X = np.zeros((3, 1))
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, Y, dict_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, Y, arr_hyps, 1, prior_mu_X)
with pytest.raises(ValueError) as error:
gp_scipy.neg_log_ml(X, Y, arr_hyps, 'abc', prior_mu_X)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3))
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y,
arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, np.expand_dims(np.arange(0, 4), axis=1),
arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov,
np.expand_dims(np.arange(0, 4), axis=1))
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_cholesky=1)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
is_fixed_noise=1)
with pytest.raises(AssertionError) as error:
gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1)
neg_log_ml_ = gp_scipy.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
is_fixed_noise=is_fixed_noise,
is_gradient=False,
is_cholesky=True)
print(neg_log_ml_)
truth_log_ml_ = 21.916650988532854
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_ = gp_scipy.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
is_fixed_noise=is_fixed_noise,
is_gradient=False,
is_cholesky=False)
print(neg_log_ml_)
truth_log_ml_ = 21.91665090519953
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_ = gp_scipy.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
is_fixed_noise=is_fixed_noise,
is_gradient=True,
is_cholesky=False)
print(neg_log_ml_)
truth_log_ml_ = 21.91665090519953
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_, neg_grad_log_ml_ = gp_scipy.neg_log_ml(
X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
is_fixed_noise=is_fixed_noise,
is_gradient=True,
is_cholesky=True)
print(neg_log_ml_)
print(neg_grad_log_ml_)
truth_log_ml_ = 21.916650988532854
truth_grad_log_ml_ = np.array([
-4.09907399e-01,
-4.09912156e+01,
-8.88182458e-04,
-8.88182458e-04,
-8.88182458e-04,
])
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
assert np.all(np.abs(neg_grad_log_ml_ - truth_grad_log_ml_) < TEST_EPSILON)
def get_optimized_kernel(
X_train: np.ndarray,
Y_train: np.ndarray,
prior_mu: constants.TYPING_UNION_CALLABLE_NONE,
str_cov: str,
str_optimizer_method: str = constants.STR_OPTIMIZER_METHOD_GP,
str_modelselection_method: str = constants.STR_MODELSELECTION_METHOD,
use_ard: bool = constants.USE_ARD,
fix_noise: bool = constants.FIX_GP_NOISE,
debug: bool = False) -> constants.TYPING_TUPLE_TWO_ARRAYS_DICT:
"""
This function computes the kernel matrix optimized by optimization
method specified, its inverse matrix, and the optimized hyperparameters.
:param X_train: inputs. Shape: (n, d) or (n, m, d).
:type X_train: numpy.ndarray
:param Y_train: outputs. Shape: (n, 1).
:type Y_train: numpy.ndarray
:param prior_mu: prior mean function or None.
:type prior_mu: callable or NoneType
:param str_cov: the name of covariance function.
:type str_cov: str.
:param str_optimizer_method: the name of optimization method.
:type str_optimizer_method: str., optional
:param str_modelselection_method: the name of model selection method.
:type str_modelselection_method: str., optional
:param use_ard: flag for using automatic relevance determination.
:type use_ard: bool., optional
:param fix_noise: flag for fixing a noise.
:type fix_noise: bool., optional
:param debug: flag for printing log messages.
:type debug: bool., optional
:returns: a tuple of kernel matrix over `X_train`, kernel matrix
inverse, and dictionary of hyperparameters.
:rtype: tuple of (numpy.ndarray, numpy.ndarray, dict.)
:raises: AssertionError, ValueError
"""
# TODO: check to input same fix_noise to convert_hyps and restore_hyps
utils_gp.validate_common_args(X_train, Y_train, str_cov, prior_mu, debug)
assert isinstance(str_optimizer_method, str)
assert isinstance(str_modelselection_method, str)
assert isinstance(use_ard, bool)
assert isinstance(fix_noise, bool)
utils_covariance.check_str_cov('get_optimized_kernel', str_cov,
X_train.shape)
assert str_optimizer_method in constants.ALLOWED_OPTIMIZER_METHOD_GP
assert str_modelselection_method in constants.ALLOWED_MODELSELECTION_METHOD
use_gradient = bool(str_optimizer_method != 'Nelder-Mead')
# TODO: Now, use_gradient is fixed as False.
# use_gradient = False
time_start = time.time()
if debug:
logger.debug('str_optimizer_method: %s', str_optimizer_method)
logger.debug('str_modelselection_method: %s',
str_modelselection_method)
logger.debug('use_gradient: %s', use_gradient)
prior_mu_train = utils_gp.get_prior_mu(prior_mu, X_train)
if str_cov in constants.ALLOWED_COV_BASE:
num_dim = X_train.shape[1]
elif str_cov in constants.ALLOWED_COV_SET:
num_dim = X_train.shape[2]
use_gradient = False
if str_modelselection_method == 'ml':
neg_log_ml_ = lambda hyps: gp_likelihood.neg_log_ml(
X_train,
Y_train,
hyps,
str_cov,
prior_mu_train,
use_ard=use_ard,
fix_noise=fix_noise,
use_gradient=use_gradient,
debug=debug)
elif str_modelselection_method == 'loocv':
# TODO: add use_ard.
neg_log_ml_ = lambda hyps: gp_likelihood.neg_log_pseudo_l_loocv(
X_train,
Y_train,
hyps,
str_cov,
prior_mu_train,
fix_noise=fix_noise,
debug=debug)
use_gradient = False
else: # pragma: no cover
raise ValueError(
'get_optimized_kernel: missing conditions for str_modelselection_method.'
)
hyps_converted = utils_covariance.convert_hyps(str_cov,
utils_covariance.get_hyps(
str_cov,
num_dim,
use_ard=use_ard),
fix_noise=fix_noise)
if str_optimizer_method in ['BFGS', 'SLSQP']:
result_optimized = scipy.optimize.minimize(neg_log_ml_,
hyps_converted,
method=str_optimizer_method,
jac=use_gradient,
options={'disp': False})
if debug:
logger.debug('negative log marginal likelihood: %.6f',
result_optimized.fun)
logger.debug('scipy message: %s', result_optimized.message)
result_optimized = result_optimized.x
elif str_optimizer_method in ['L-BFGS-B', 'SLSQP-Bounded']:
if str_optimizer_method == 'SLSQP-Bounded':
str_optimizer_method = 'SLSQP'
bounds = utils_covariance.get_range_hyps(str_cov,
num_dim,
use_ard=use_ard,
fix_noise=fix_noise)
result_optimized = scipy.optimize.minimize(neg_log_ml_,
hyps_converted,
method=str_optimizer_method,
bounds=bounds,
jac=use_gradient,
options={'disp': False})
if debug:
logger.debug('negative log marginal likelihood: %.6f',
result_optimized.fun)
logger.debug('scipy message: %s', result_optimized.message)
result_optimized = result_optimized.x
elif str_optimizer_method in ['Nelder-Mead']:
result_optimized = scipy.optimize.minimize(neg_log_ml_,
hyps_converted,
method=str_optimizer_method,
options={'disp': False})
if debug:
logger.debug('negative log marginal likelihood: %.6f',
result_optimized.fun)
logger.debug('scipy message: %s', result_optimized.message)
result_optimized = result_optimized.x
else: # pragma: no cover
raise ValueError(
'get_optimized_kernel: missing conditions for str_optimizer_method'
)
hyps = utils_covariance.restore_hyps(str_cov,
result_optimized,
use_ard=use_ard,
fix_noise=fix_noise)
hyps = utils_covariance.validate_hyps_dict(hyps, str_cov, num_dim)
cov_X_X, inv_cov_X_X, _ = covariance.get_kernel_inverse(
X_train, hyps, str_cov, fix_noise=fix_noise, debug=debug)
time_end = time.time()
if debug:
logger.debug('hyps optimized: %s', utils_logger.get_str_hyps(hyps))
logger.debug('time consumed to construct gpr: %.4f sec.',
time_end - time_start)
return cov_X_X, inv_cov_X_X, hyps
def get_optimized_kernel(
X_train,
Y_train,
prior_mu,
str_cov,
str_optimizer_method=constants.STR_OPTIMIZER_METHOD_GP,
str_modelselection_method=constants.STR_MODELSELECTION_METHOD,
is_fixed_noise=constants.IS_FIXED_GP_NOISE,
debug=False):
# TODO: check to input same is_fixed_noise to convert_hyps and restore_hyps
assert isinstance(X_train, np.ndarray)
assert isinstance(Y_train, np.ndarray)
assert callable(prior_mu) or prior_mu is None
assert isinstance(str_cov, str)
assert isinstance(str_optimizer_method, str)
assert isinstance(str_modelselection_method, str)
assert isinstance(is_fixed_noise, bool)
assert isinstance(debug, bool)
assert len(Y_train.shape) == 2
assert X_train.shape[0] == Y_train.shape[0]
_check_str_cov('get_optimized_kernel', str_cov, X_train.shape)
assert str_optimizer_method in constants.ALLOWED_OPTIMIZER_METHOD_GP
assert str_modelselection_method in constants.ALLOWED_MODELSELECTION_METHOD
time_start = time.time()
if debug:
print('[DEBUG] get_optimized_kernel in gp.py: str_optimizer_method {}'.
format(str_optimizer_method))
print(
'[DEBUG] get_optimized_kernel in gp.py: str_modelselection_method {}'
.format(str_modelselection_method))
prior_mu_train = get_prior_mu(prior_mu, X_train)
if str_cov in constants.ALLOWED_GP_COV_BASE:
num_dim = X_train.shape[1]
elif str_cov in constants.ALLOWED_GP_COV_SET:
num_dim = X_train.shape[2]
if str_modelselection_method == 'ml':
neg_log_ml = lambda hyps: -1.0 * log_ml(X_train,
Y_train,
hyps,
str_cov,
prior_mu_train,
is_fixed_noise=is_fixed_noise,
debug=debug)
elif str_modelselection_method == 'loocv':
neg_log_ml = lambda hyps: -1.0 * log_pseudo_l_loocv(X_train,
Y_train,
hyps,
str_cov,
prior_mu_train,
is_fixed_noise=
is_fixed_noise,
debug=debug)
else: # pragma: no cover
raise ValueError(
'get_optimized_kernel: missing conditions for str_modelselection_method.'
)
hyps_converted = utils_covariance.convert_hyps(
str_cov,
utils_covariance.get_hyps(str_cov, num_dim),
is_fixed_noise=is_fixed_noise,
)
if str_optimizer_method == 'BFGS':
result_optimized = scipy.optimize.minimize(neg_log_ml,
hyps_converted,
method=str_optimizer_method)
result_optimized = result_optimized.x
elif str_optimizer_method == 'L-BFGS-B':
bounds = utils_covariance.get_range_hyps(str_cov,
num_dim,
is_fixed_noise=is_fixed_noise)
result_optimized = scipy.optimize.minimize(neg_log_ml,
hyps_converted,
method=str_optimizer_method,
bounds=bounds)
result_optimized = result_optimized.x
# TODO: Fill this conditions
elif str_optimizer_method == 'DIRECT': # pragma: no cover
raise NotImplementedError(
'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.'
)
elif str_optimizer_method == 'CMA-ES': # pragma: no cover
raise NotImplementedError(
'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.'
)
# INFO: It is allowed, but a condition is missed.
else: # pragma: no cover
raise ValueError(
'get_optimized_kernel: missing conditions for str_optimizer_method'
)
hyps = utils_covariance.restore_hyps(str_cov,
result_optimized,
is_fixed_noise=is_fixed_noise)
hyps, _ = utils_covariance.validate_hyps_dict(hyps, str_cov, num_dim)
cov_X_X, inv_cov_X_X = get_kernel_inverse(X_train,
hyps,
str_cov,
debug=debug)
time_end = time.time()
if debug:
print('[DEBUG] get_optimized_kernel in gp.py: optimized hyps for gpr',
hyps)
print('[DEBUG] get_optimized_kernel in gp.py: time consumed',
time_end - time_start, 'sec.')
return cov_X_X, inv_cov_X_X, hyps
def test_neg_log_ml():
dim_X = 3
str_cov = 'se'
X = np.reshape(np.arange(0, 9), (3, dim_X))
Y = np.expand_dims(np.arange(3, 10, 3), axis=1)
fix_noise = False
use_gp = False
dict_hyps = utils_covariance.get_hyps(str_cov, dim_X, use_gp=use_gp)
arr_hyps = utils_covariance.convert_hyps(str_cov,
dict_hyps,
fix_noise=fix_noise,
use_gp=use_gp)
prior_mu_X = np.zeros((3, 1))
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(np.arange(0, 3), Y, arr_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, np.arange(0, 3), arr_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, dict_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, 1, prior_mu_X)
with pytest.raises(ValueError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, 'abc', prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3))
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y,
arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, np.expand_dims(np.arange(0, 4), axis=1),
arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, str_cov,
np.expand_dims(np.arange(0, 4), axis=1))
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=1)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1)
neg_log_ml_ = package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=fix_noise,
use_gradient=False)
print(neg_log_ml_)
truth_log_ml_ = 5.634155417555853
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_, neg_grad_log_ml_ = package_target.neg_log_ml(
X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=fix_noise,
use_gradient=True)
print(neg_log_ml_)
print(neg_grad_log_ml_)
print(neg_grad_log_ml_[2])
print(neg_grad_log_ml_[3])
print(neg_grad_log_ml_[4])
truth_log_ml_ = 5.634155417555853
truth_grad_log_ml_ = np.array([
-1.60446383e-02,
1.75087448e-01,
-1.60448396e+00,
-1.836237221888097e-05,
-1.836237221888097e-05,
-1.836237221888097e-05,
])
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
assert np.all(np.abs(neg_grad_log_ml_ - truth_grad_log_ml_) < TEST_EPSILON)
dict_hyps = utils_covariance.get_hyps(str_cov, dim_X, use_gp=use_gp)
arr_hyps = utils_covariance.convert_hyps(str_cov,
dict_hyps,
fix_noise=True,
use_gp=use_gp)
neg_log_ml_, neg_grad_log_ml_ = package_target.neg_log_ml(
X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=True, use_gradient=True)
print(neg_log_ml_)
print(neg_grad_log_ml_)
def get_optimized_kernel(
X_train,
Y_train,
prior_mu,
str_cov,
str_optimizer_method=constants.STR_OPTIMIZER_METHOD_GP,
str_modelselection_method=constants.STR_MODELSELECTION_METHOD,
is_fixed_noise=constants.IS_FIXED_GP_NOISE,
debug=False):
"""
This function computes the kernel matrix optimized by optimization method specified, its inverse matrix, and the optimized hyperparameters.
:param X_train: inputs. Shape: (n, d) or (n, m, d).
:type X_train: numpy.ndarray
:param Y_train: outputs. Shape: (n, 1).
:type Y_train: numpy.ndarray
:param prior_mu: prior mean function or None.
:type prior_mu: function or NoneType
:param str_cov: the name of covariance function.
:type str_cov: str.
:param str_optimizer_method: the name of optimization method.
:type str_optimizer_method: str., optional
:param str_modelselection_method: the name of model selection method.
:type str_modelselection_method: str., optional
:param is_fixed_noise: flag for fixing a noise.
:type is_fixed_noise: bool., optional
:param debug: flag for printing log messages.
:type debug: bool., optional
:returns: a tuple of kernel matrix over `X_train`, kernel matrix inverse, and dictionary of hyperparameters.
:rtype: tuple of (numpy.ndarray, numpy.ndarray, dict.)
:raises: AssertionError, ValueError
"""
# TODO: check to input same is_fixed_noise to convert_hyps and restore_hyps
assert isinstance(X_train, np.ndarray)
assert isinstance(Y_train, np.ndarray)
assert callable(prior_mu) or prior_mu is None
assert isinstance(str_cov, str)
assert isinstance(str_optimizer_method, str)
assert isinstance(str_modelselection_method, str)
assert isinstance(is_fixed_noise, bool)
assert isinstance(debug, bool)
assert len(Y_train.shape) == 2
assert X_train.shape[0] == Y_train.shape[0]
_check_str_cov('get_optimized_kernel', str_cov, X_train.shape)
assert str_optimizer_method in constants.ALLOWED_OPTIMIZER_METHOD_GP
assert str_modelselection_method in constants.ALLOWED_MODELSELECTION_METHOD
# TODO: fix this.
is_gradient = True
time_start = time.time()
if debug:
print('[DEBUG] get_optimized_kernel in gp.py: str_optimizer_method {}'.
format(str_optimizer_method))
print(
'[DEBUG] get_optimized_kernel in gp.py: str_modelselection_method {}'
.format(str_modelselection_method))
prior_mu_train = get_prior_mu(prior_mu, X_train)
if str_cov in constants.ALLOWED_GP_COV_BASE:
num_dim = X_train.shape[1]
elif str_cov in constants.ALLOWED_GP_COV_SET:
num_dim = X_train.shape[2]
is_gradient = False
if str_modelselection_method == 'ml':
neg_log_ml_ = lambda hyps: neg_log_ml(X_train,
Y_train,
hyps,
str_cov,
prior_mu_train,
is_fixed_noise=is_fixed_noise,
is_gradient=is_gradient,
debug=debug)
elif str_modelselection_method == 'loocv':
neg_log_ml_ = lambda hyps: neg_log_pseudo_l_loocv(X_train,
Y_train,
hyps,
str_cov,
prior_mu_train,
is_fixed_noise=
is_fixed_noise,
debug=debug)
is_gradient = False
else: # pragma: no cover
raise ValueError(
'get_optimized_kernel: missing conditions for str_modelselection_method.'
)
hyps_converted = utils_covariance.convert_hyps(
str_cov,
utils_covariance.get_hyps(str_cov, num_dim),
is_fixed_noise=is_fixed_noise,
)
if str_optimizer_method == 'BFGS':
result_optimized = scipy.optimize.minimize(neg_log_ml_,
hyps_converted,
method=str_optimizer_method,
jac=is_gradient,
options={'disp': False})
result_optimized = result_optimized.x
elif str_optimizer_method == 'L-BFGS-B':
bounds = utils_covariance.get_range_hyps(str_cov,
num_dim,
is_fixed_noise=is_fixed_noise)
result_optimized = scipy.optimize.minimize(neg_log_ml_,
hyps_converted,
method=str_optimizer_method,
bounds=bounds,
jac=is_gradient,
options={'disp': False})
result_optimized = result_optimized.x
# TODO: Fill this conditions
elif str_optimizer_method == 'DIRECT': # pragma: no cover
raise NotImplementedError(
'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.'
)
elif str_optimizer_method == 'CMA-ES': # pragma: no cover
raise NotImplementedError(
'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.'
)
else: # pragma: no cover
raise ValueError(
'get_optimized_kernel: missing conditions for str_optimizer_method'
)
hyps = utils_covariance.restore_hyps(str_cov,
result_optimized,
is_fixed_noise=is_fixed_noise)
hyps, _ = utils_covariance.validate_hyps_dict(hyps, str_cov, num_dim)
cov_X_X, inv_cov_X_X, grad_cov_X_X = get_kernel_inverse(
X_train, hyps, str_cov, is_fixed_noise=is_fixed_noise, debug=debug)
time_end = time.time()
if debug:
print('[DEBUG] get_optimized_kernel in gp.py: optimized hyps for gpr',
hyps)
print('[DEBUG] get_optimized_kernel in gp.py: time consumed',
time_end - time_start, 'sec.')
return cov_X_X, inv_cov_X_X, hyps
def test_convert_hyps():
cur_hyps = {
'noise': 0.1,
'signal': 1.0,
'lengthscales': np.array([2.0, 2.0])
}
with pytest.raises(AssertionError) as error:
package_target.convert_hyps(1.2, 2.1)
with pytest.raises(AssertionError) as error:
package_target.convert_hyps(1.2, dict())
with pytest.raises(AssertionError) as error:
package_target.convert_hyps('se', 2.1)
with pytest.raises(AssertionError) as error:
package_target.convert_hyps('abc', 2.1)
with pytest.raises(AssertionError) as error:
package_target.convert_hyps('abc', cur_hyps)
with pytest.raises(AssertionError) as error:
package_target.convert_hyps('se', dict(), fix_noise=1)
with pytest.raises(AssertionError) as error:
package_target.convert_hyps('se', cur_hyps, use_gp=1)
with pytest.raises(AssertionError) as error:
package_target.convert_hyps('se', cur_hyps, use_gp='abc')
converted_hyps = package_target.convert_hyps('se',
cur_hyps,
fix_noise=False)
assert len(converted_hyps.shape) == 1
assert converted_hyps.shape[0] == 4
assert converted_hyps[0] == cur_hyps['noise']
assert converted_hyps[1] == cur_hyps['signal']
assert (converted_hyps[2:] == cur_hyps['lengthscales']).all()
converted_hyps = package_target.convert_hyps('se',
cur_hyps,
fix_noise=True)
assert len(converted_hyps.shape) == 1
assert converted_hyps.shape[0] == 3
assert converted_hyps[0] == cur_hyps['signal']
assert (converted_hyps[1:] == cur_hyps['lengthscales']).all()
cur_hyps = {
'noise': 0.1,
'signal': 1.0,
'lengthscales': np.array([2.0, 2.0]),
'dof': 100.0
}
converted_hyps = package_target.convert_hyps('se',
cur_hyps,
fix_noise=False,
use_gp=False)
assert len(converted_hyps.shape) == 1
assert converted_hyps.shape[0] == 5
assert converted_hyps[0] == cur_hyps['noise']
assert converted_hyps[1] == cur_hyps['dof']
assert converted_hyps[2] == cur_hyps['signal']
assert (converted_hyps[3:] == cur_hyps['lengthscales']).all()
converted_hyps = package_target.convert_hyps('se',
cur_hyps,
fix_noise=True,
use_gp=False)
assert len(converted_hyps.shape) == 1
assert converted_hyps.shape[0] == 4
assert converted_hyps[0] == cur_hyps['dof']
assert converted_hyps[1] == cur_hyps['signal']
assert (converted_hyps[2:] == cur_hyps['lengthscales']).all()
def test_neg_log_ml():
dim_X = 3
str_cov = 'se'
X = np.reshape(np.arange(0, 9), (3, dim_X))
Y = np.expand_dims(np.arange(3, 10, 3), axis=1)
fix_noise = False
dict_hyps = utils_covariance.get_hyps(str_cov, dim_X)
arr_hyps = utils_covariance.convert_hyps(str_cov,
dict_hyps,
fix_noise=fix_noise)
prior_mu_X = np.zeros((3, 1))
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(np.arange(0, 3), Y, arr_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, np.arange(0, 3), arr_hyps, str_cov,
prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, dict_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, 1, prior_mu_X)
with pytest.raises(ValueError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, 'abc', prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3))
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y,
arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, np.expand_dims(np.arange(0, 4), axis=1),
arr_hyps, str_cov, prior_mu_X)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, str_cov,
np.expand_dims(np.arange(0, 4), axis=1))
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
use_cholesky=1)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=1)
with pytest.raises(AssertionError) as error:
package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1)
neg_log_ml_ = package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=fix_noise,
use_gradient=False,
use_cholesky=True)
print(neg_log_ml_)
truth_log_ml_ = 21.916650988532854
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_ = package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=fix_noise,
use_gradient=False,
use_cholesky=False)
print(neg_log_ml_)
truth_log_ml_ = 21.91665090519953
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_ = package_target.neg_log_ml(X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=fix_noise,
use_gradient=True,
use_cholesky=False)
print(neg_log_ml_)
truth_log_ml_ = 21.91665090519953
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
neg_log_ml_, neg_grad_log_ml_ = package_target.neg_log_ml(
X,
Y,
arr_hyps,
str_cov,
prior_mu_X,
fix_noise=fix_noise,
use_gradient=True,
use_cholesky=True)
print(neg_log_ml_)
print(neg_grad_log_ml_)
print(neg_grad_log_ml_[2])
print(neg_grad_log_ml_[3])
print(neg_grad_log_ml_[4])
truth_log_ml_ = 21.916650988532854
truth_grad_log_ml_ = np.array([
-4.09907399e-01,
-4.09912156e+01,
-0.00029606081917620415,
-0.00029606081917620415,
-0.00029606081917620415,
])
assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON
assert np.all(np.abs(neg_grad_log_ml_ - truth_grad_log_ml_) < TEST_EPSILON)