def _check_hyps_convergence(list_hyps, hyps, str_cov, is_fixed_noise, ratio_threshold=0.05): assert isinstance(list_hyps, list) assert isinstance(hyps, dict) assert isinstance(str_cov, str) assert isinstance(is_fixed_noise, bool) assert isinstance(ratio_threshold, float) is_converged = False if len(list_hyps) > 0: hyps_converted = utils_covariance.convert_hyps( str_cov, hyps, is_fixed_noise=is_fixed_noise) target_hyps_converted = utils_covariance.convert_hyps( str_cov, list_hyps[-1], is_fixed_noise=is_fixed_noise) cur_norm = np.linalg.norm(hyps_converted - target_hyps_converted, ord=2) threshold = np.linalg.norm(target_hyps_converted) * ratio_threshold if np.linalg.norm(hyps_converted - target_hyps_converted, ord=2) < threshold: is_converged = True return is_converged
def test_neg_log_pseudo_l_loocv(): dim_X = 3 str_cov = 'se' X = np.reshape(np.arange(0, 9), (3, dim_X)) Y = np.expand_dims(np.arange(3, 10, 3), axis=1) dict_hyps = utils_covariance.get_hyps(str_cov, dim_X) arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, fix_noise=constants.FIX_GP_NOISE) prior_mu_X = np.zeros((3, 1)) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(X, Y, dict_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, 1, prior_mu_X) with pytest.raises(ValueError) as error: package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, 'abc', prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, str_cov, np.arange(0, 3)) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv( np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv( X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv( X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1)) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=1) with pytest.raises(AssertionError) as error: package_target.neg_log_pseudo_l_loocv(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1) neg_log_pseudo_l_ = package_target.neg_log_pseudo_l_loocv( X, Y, arr_hyps, str_cov, prior_mu_X) print(neg_log_pseudo_l_) truth_log_pseudo_l_ = 21.916822991658695 assert np.abs(neg_log_pseudo_l_ - truth_log_pseudo_l_) < TEST_EPSILON
def _check_hyps_convergence(list_hyps, hyps, str_cov, is_fixed_noise, ratio_threshold=0.05): """ It checks convergence of hyperparameters for Gaussian process regression. :param list_hyps: list of historical hyperparameters for Gaussian process regression. :type list_hyps: list :param hyps: dictionary of hyperparameters for acquisition function. :type hyps: dict. :param str_cov: the name of covariance function. :type str_cov: str. :param is_fixed_noise: flag for fixing a noise. :type is_fixed_noise: bool. :param ratio_threshold: ratio of threshold for checking convergence. :type ratio_threshold: float, optional :returns: flag for checking convergence. If converged, it is True. :rtype: bool. :raises: AssertionError """ assert isinstance(list_hyps, list) assert isinstance(hyps, dict) assert isinstance(str_cov, str) assert isinstance(is_fixed_noise, bool) assert isinstance(ratio_threshold, float) is_converged = False if len(list_hyps) > 0: hyps_converted = utils_covariance.convert_hyps( str_cov, hyps, is_fixed_noise=is_fixed_noise) target_hyps_converted = utils_covariance.convert_hyps( str_cov, list_hyps[-1], is_fixed_noise=is_fixed_noise) cur_norm = np.linalg.norm(hyps_converted - target_hyps_converted, ord=2) threshold = np.linalg.norm(target_hyps_converted) * ratio_threshold if np.linalg.norm(hyps_converted - target_hyps_converted, ord=2) < threshold: is_converged = True return is_converged
def test_validate_hyps_arr(): num_dim = 2 str_cov = 'matern32' cur_hyps = utils_covariance.get_hyps(str_cov, num_dim) cur_hyps = utils_covariance.convert_hyps(str_cov, cur_hyps, is_fixed_noise=False) with pytest.raises(AssertionError) as error: _, is_valid = utils_covariance.validate_hyps_arr(123, str_cov, num_dim) with pytest.raises(AssertionError) as error: _, is_valid = utils_covariance.validate_hyps_arr( cur_hyps, 'abc', num_dim) with pytest.raises(AssertionError) as error: _, is_valid = utils_covariance.validate_hyps_arr( cur_hyps, str_cov, 'abc')
def test_log_ml(): dim_X = 3 str_cov = 'se' X = np.reshape(np.arange(0, 9), (3, dim_X)) Y = np.expand_dims(np.arange(3, 10, 3), axis=1) dict_hyps = utils_covariance.get_hyps(str_cov, dim_X) arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, is_fixed_noise=constants.IS_FIXED_GP_NOISE) prior_mu_X = np.zeros((3, 1)) with pytest.raises(AssertionError) as error: gp.log_ml(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp.log_ml(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, dict_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, arr_hyps, 1, prior_mu_X) with pytest.raises(ValueError) as error: gp.log_ml(X, Y, arr_hyps, 'abc', prior_mu_X) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3)) with pytest.raises(AssertionError) as error: gp.log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp.log_ml(X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1)) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_cholesky=1) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=1) with pytest.raises(AssertionError) as error: gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1) log_ml = gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X) print(log_ml) truth_log_ml = -65.74995266591506 assert np.abs(log_ml - truth_log_ml) < TEST_EPSILON log_ml = gp.log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_cholesky=False) print(log_ml) truth_log_ml = -65.74995266566506 assert np.abs(log_ml - truth_log_ml) < TEST_EPSILON
def test_validate_hyps_arr(): num_dim = 2 str_cov = 'matern32' cur_hyps = package_target.get_hyps(str_cov, num_dim) cur_hyps = package_target.convert_hyps(str_cov, cur_hyps, fix_noise=False) with pytest.raises(AssertionError) as error: _, is_valid = package_target.validate_hyps_arr(123, str_cov, num_dim) with pytest.raises(AssertionError) as error: _, is_valid = package_target.validate_hyps_arr(cur_hyps, 'abc', num_dim) with pytest.raises(AssertionError) as error: _, is_valid = package_target.validate_hyps_arr(cur_hyps, str_cov, 'abc') with pytest.raises(AssertionError) as error: _, is_valid = package_target.validate_hyps_arr(cur_hyps, str_cov, num_dim, use_gp='abc')
def test_convert_hyps(): cur_hyps = { 'noise': 0.1, 'signal': 1.0, 'lengthscales': np.array([1.0, 1.0]) } with pytest.raises(AssertionError) as error: utils_covariance.convert_hyps(1.2, 2.1) with pytest.raises(AssertionError) as error: utils_covariance.convert_hyps(1.2, dict()) with pytest.raises(AssertionError) as error: utils_covariance.convert_hyps('se', 2.1) with pytest.raises(AssertionError) as error: utils_covariance.convert_hyps('abc', 2.1) with pytest.raises(AssertionError) as error: utils_covariance.convert_hyps('abc', cur_hyps) with pytest.raises(AssertionError) as error: utils_covariance.convert_hyps('se', dict(), is_fixed_noise=1) converted_hyps = utils_covariance.convert_hyps('se', cur_hyps) assert converted_hyps[0] == cur_hyps['noise'] assert converted_hyps[1] == cur_hyps['signal'] assert (converted_hyps[2:] == cur_hyps['lengthscales']).all()
def optimize( self, X_train: np.ndarray, Y_train: np.ndarray, str_sampling_method: str = constants.STR_SAMPLING_METHOD_AO, num_samples: int = constants.NUM_SAMPLES_AO, str_mlm_method: str = constants.STR_MLM_METHOD, ) -> constants.TYPING_TUPLE_ARRAY_DICT: """ It computes acquired example, candidates of acquired examples, acquisition function values for the candidates, covariance matrix, inverse matrix of the covariance matrix, hyperparameters optimized, and execution times. :param X_train: inputs. Shape: (n, d) or (n, m, d). :type X_train: numpy.ndarray :param Y_train: outputs. Shape: (n, 1). :type Y_train: numpy.ndarray :param str_sampling_method: the name of sampling method for acquisition function optimization. :type str_sampling_method: str., optional :param num_samples: the number of samples. :type num_samples: int., optional :param str_mlm_method: the name of marginal likelihood maximization method for Gaussian process regression. :type str_mlm_method: str., optional :returns: acquired example and dictionary of information. Shape: ((d, ), dict.). :rtype: (numpy.ndarray, dict.) :raises: AssertionError """ assert isinstance(X_train, np.ndarray) assert isinstance(Y_train, np.ndarray) assert isinstance(str_sampling_method, str) assert isinstance(num_samples, int) assert isinstance(str_mlm_method, str) assert len(X_train.shape) == 2 assert len(Y_train.shape) == 2 assert Y_train.shape[1] == 1 assert X_train.shape[0] == Y_train.shape[0] assert X_train.shape[1] == self.num_dim assert num_samples > 0 assert str_sampling_method in constants.ALLOWED_SAMPLING_METHOD assert str_mlm_method in constants.ALLOWED_MLM_METHOD time_start = time.time() Y_train_orig = Y_train if self.normalize_Y and str_mlm_method != 'converged': if self.debug: self.logger.debug('Responses are normalized.') Y_train = utils_bo.normalize_min_max(Y_train) time_start_surrogate = time.time() if str_mlm_method == 'regular': cov_X_X, inv_cov_X_X, hyps = gp_kernel.get_optimized_kernel( X_train, Y_train, self.prior_mu, self.str_cov, str_optimizer_method=self.str_optimizer_method_gp, str_modelselection_method=self.str_modelselection_method, use_ard=self.use_ard, debug=self.debug) elif str_mlm_method == 'combined': from bayeso.gp import gp_likelihood from bayeso.utils import utils_gp from bayeso.utils import utils_covariance prior_mu_train = utils_gp.get_prior_mu(self.prior_mu, X_train) neg_log_ml_best = np.inf cov_X_X_best = None inv_cov_X_X_best = None hyps_best = None for cur_str_optimizer_method in ['BFGS', 'Nelder-Mead']: cov_X_X, inv_cov_X_X, hyps = gp_kernel.get_optimized_kernel( X_train, Y_train, self.prior_mu, self.str_cov, str_optimizer_method=cur_str_optimizer_method, str_modelselection_method=self.str_modelselection_method, use_ard=self.use_ard, debug=self.debug) cur_neg_log_ml_ = gp_likelihood.neg_log_ml( X_train, Y_train, utils_covariance.convert_hyps( self.str_cov, hyps, fix_noise=constants.FIX_GP_NOISE), self.str_cov, prior_mu_train, use_ard=self.use_ard, fix_noise=constants.FIX_GP_NOISE, use_gradient=False, debug=self.debug) if cur_neg_log_ml_ < neg_log_ml_best: neg_log_ml_best = cur_neg_log_ml_ cov_X_X_best = cov_X_X inv_cov_X_X_best = inv_cov_X_X hyps_best = hyps cov_X_X = cov_X_X_best inv_cov_X_X = inv_cov_X_X_best hyps = hyps_best elif str_mlm_method == 'converged': fix_noise = constants.FIX_GP_NOISE if self.is_optimize_hyps: cov_X_X, inv_cov_X_X, hyps = gp_kernel.get_optimized_kernel( X_train, Y_train, self.prior_mu, self.str_cov, str_optimizer_method=self.str_optimizer_method_gp, str_modelselection_method=self.str_modelselection_method, use_ard=self.use_ard, debug=self.debug) self.is_optimize_hyps = not utils_bo.check_hyps_convergence( self.historical_hyps, hyps, self.str_cov, fix_noise) else: # pragma: no cover if self.debug: self.logger.debug('hyps converged.') hyps = self.historical_hyps[-1] cov_X_X, inv_cov_X_X, _ = covariance.get_kernel_inverse( X_train, hyps, self.str_cov, fix_noise=fix_noise, debug=self.debug) else: # pragma: no cover raise ValueError('optimize: missing condition for str_mlm_method.') self.historical_hyps.append(hyps) time_end_surrogate = time.time() time_start_acq = time.time() fun_negative_acquisition = lambda X_test: -1.0 * self.compute_acquisitions( X_test, X_train, Y_train, cov_X_X, inv_cov_X_X, hyps) next_point, next_points = self._optimize( fun_negative_acquisition, str_sampling_method=str_sampling_method, num_samples=num_samples) time_end_acq = time.time() acquisitions = fun_negative_acquisition(next_points) time_end = time.time() dict_info = { 'next_points': next_points, 'acquisitions': acquisitions, 'Y_original': Y_train_orig, 'Y_normalized': Y_train, 'cov_X_X': cov_X_X, 'inv_cov_X_X': inv_cov_X_X, 'hyps': hyps, 'time_surrogate': time_end_surrogate - time_start_surrogate, 'time_acq': time_end_acq - time_start_acq, 'time_overall': time_end - time_start, } if self.debug: self.logger.debug('overall time consumed to acquire: %.4f sec.', time_end - time_start) return next_point, dict_info
def test_neg_log_ml(): dim_X = 3 str_cov = 'se' X = np.reshape(np.arange(0, 9), (3, dim_X)) Y = np.expand_dims(np.arange(3, 10, 3), axis=1) is_fixed_noise = False dict_hyps = utils_covariance.get_hyps(str_cov, dim_X) arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, is_fixed_noise=is_fixed_noise) prior_mu_X = np.zeros((3, 1)) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, dict_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, 1, prior_mu_X) with pytest.raises(ValueError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, 'abc', prior_mu_X) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3)) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1)) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_cholesky=1) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=1) with pytest.raises(AssertionError) as error: gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1) neg_log_ml_ = gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=is_fixed_noise, is_gradient=False, is_cholesky=True) print(neg_log_ml_) truth_log_ml_ = 21.916650988532854 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_ = gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=is_fixed_noise, is_gradient=False, is_cholesky=False) print(neg_log_ml_) truth_log_ml_ = 21.91665090519953 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_ = gp_scipy.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=is_fixed_noise, is_gradient=True, is_cholesky=False) print(neg_log_ml_) truth_log_ml_ = 21.91665090519953 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_, neg_grad_log_ml_ = gp_scipy.neg_log_ml( X, Y, arr_hyps, str_cov, prior_mu_X, is_fixed_noise=is_fixed_noise, is_gradient=True, is_cholesky=True) print(neg_log_ml_) print(neg_grad_log_ml_) truth_log_ml_ = 21.916650988532854 truth_grad_log_ml_ = np.array([ -4.09907399e-01, -4.09912156e+01, -8.88182458e-04, -8.88182458e-04, -8.88182458e-04, ]) assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON assert np.all(np.abs(neg_grad_log_ml_ - truth_grad_log_ml_) < TEST_EPSILON)
def get_optimized_kernel( X_train: np.ndarray, Y_train: np.ndarray, prior_mu: constants.TYPING_UNION_CALLABLE_NONE, str_cov: str, str_optimizer_method: str = constants.STR_OPTIMIZER_METHOD_GP, str_modelselection_method: str = constants.STR_MODELSELECTION_METHOD, use_ard: bool = constants.USE_ARD, fix_noise: bool = constants.FIX_GP_NOISE, debug: bool = False) -> constants.TYPING_TUPLE_TWO_ARRAYS_DICT: """ This function computes the kernel matrix optimized by optimization method specified, its inverse matrix, and the optimized hyperparameters. :param X_train: inputs. Shape: (n, d) or (n, m, d). :type X_train: numpy.ndarray :param Y_train: outputs. Shape: (n, 1). :type Y_train: numpy.ndarray :param prior_mu: prior mean function or None. :type prior_mu: callable or NoneType :param str_cov: the name of covariance function. :type str_cov: str. :param str_optimizer_method: the name of optimization method. :type str_optimizer_method: str., optional :param str_modelselection_method: the name of model selection method. :type str_modelselection_method: str., optional :param use_ard: flag for using automatic relevance determination. :type use_ard: bool., optional :param fix_noise: flag for fixing a noise. :type fix_noise: bool., optional :param debug: flag for printing log messages. :type debug: bool., optional :returns: a tuple of kernel matrix over `X_train`, kernel matrix inverse, and dictionary of hyperparameters. :rtype: tuple of (numpy.ndarray, numpy.ndarray, dict.) :raises: AssertionError, ValueError """ # TODO: check to input same fix_noise to convert_hyps and restore_hyps utils_gp.validate_common_args(X_train, Y_train, str_cov, prior_mu, debug) assert isinstance(str_optimizer_method, str) assert isinstance(str_modelselection_method, str) assert isinstance(use_ard, bool) assert isinstance(fix_noise, bool) utils_covariance.check_str_cov('get_optimized_kernel', str_cov, X_train.shape) assert str_optimizer_method in constants.ALLOWED_OPTIMIZER_METHOD_GP assert str_modelselection_method in constants.ALLOWED_MODELSELECTION_METHOD use_gradient = bool(str_optimizer_method != 'Nelder-Mead') # TODO: Now, use_gradient is fixed as False. # use_gradient = False time_start = time.time() if debug: logger.debug('str_optimizer_method: %s', str_optimizer_method) logger.debug('str_modelselection_method: %s', str_modelselection_method) logger.debug('use_gradient: %s', use_gradient) prior_mu_train = utils_gp.get_prior_mu(prior_mu, X_train) if str_cov in constants.ALLOWED_COV_BASE: num_dim = X_train.shape[1] elif str_cov in constants.ALLOWED_COV_SET: num_dim = X_train.shape[2] use_gradient = False if str_modelselection_method == 'ml': neg_log_ml_ = lambda hyps: gp_likelihood.neg_log_ml( X_train, Y_train, hyps, str_cov, prior_mu_train, use_ard=use_ard, fix_noise=fix_noise, use_gradient=use_gradient, debug=debug) elif str_modelselection_method == 'loocv': # TODO: add use_ard. neg_log_ml_ = lambda hyps: gp_likelihood.neg_log_pseudo_l_loocv( X_train, Y_train, hyps, str_cov, prior_mu_train, fix_noise=fix_noise, debug=debug) use_gradient = False else: # pragma: no cover raise ValueError( 'get_optimized_kernel: missing conditions for str_modelselection_method.' ) hyps_converted = utils_covariance.convert_hyps(str_cov, utils_covariance.get_hyps( str_cov, num_dim, use_ard=use_ard), fix_noise=fix_noise) if str_optimizer_method in ['BFGS', 'SLSQP']: result_optimized = scipy.optimize.minimize(neg_log_ml_, hyps_converted, method=str_optimizer_method, jac=use_gradient, options={'disp': False}) if debug: logger.debug('negative log marginal likelihood: %.6f', result_optimized.fun) logger.debug('scipy message: %s', result_optimized.message) result_optimized = result_optimized.x elif str_optimizer_method in ['L-BFGS-B', 'SLSQP-Bounded']: if str_optimizer_method == 'SLSQP-Bounded': str_optimizer_method = 'SLSQP' bounds = utils_covariance.get_range_hyps(str_cov, num_dim, use_ard=use_ard, fix_noise=fix_noise) result_optimized = scipy.optimize.minimize(neg_log_ml_, hyps_converted, method=str_optimizer_method, bounds=bounds, jac=use_gradient, options={'disp': False}) if debug: logger.debug('negative log marginal likelihood: %.6f', result_optimized.fun) logger.debug('scipy message: %s', result_optimized.message) result_optimized = result_optimized.x elif str_optimizer_method in ['Nelder-Mead']: result_optimized = scipy.optimize.minimize(neg_log_ml_, hyps_converted, method=str_optimizer_method, options={'disp': False}) if debug: logger.debug('negative log marginal likelihood: %.6f', result_optimized.fun) logger.debug('scipy message: %s', result_optimized.message) result_optimized = result_optimized.x else: # pragma: no cover raise ValueError( 'get_optimized_kernel: missing conditions for str_optimizer_method' ) hyps = utils_covariance.restore_hyps(str_cov, result_optimized, use_ard=use_ard, fix_noise=fix_noise) hyps = utils_covariance.validate_hyps_dict(hyps, str_cov, num_dim) cov_X_X, inv_cov_X_X, _ = covariance.get_kernel_inverse( X_train, hyps, str_cov, fix_noise=fix_noise, debug=debug) time_end = time.time() if debug: logger.debug('hyps optimized: %s', utils_logger.get_str_hyps(hyps)) logger.debug('time consumed to construct gpr: %.4f sec.', time_end - time_start) return cov_X_X, inv_cov_X_X, hyps
def get_optimized_kernel( X_train, Y_train, prior_mu, str_cov, str_optimizer_method=constants.STR_OPTIMIZER_METHOD_GP, str_modelselection_method=constants.STR_MODELSELECTION_METHOD, is_fixed_noise=constants.IS_FIXED_GP_NOISE, debug=False): # TODO: check to input same is_fixed_noise to convert_hyps and restore_hyps assert isinstance(X_train, np.ndarray) assert isinstance(Y_train, np.ndarray) assert callable(prior_mu) or prior_mu is None assert isinstance(str_cov, str) assert isinstance(str_optimizer_method, str) assert isinstance(str_modelselection_method, str) assert isinstance(is_fixed_noise, bool) assert isinstance(debug, bool) assert len(Y_train.shape) == 2 assert X_train.shape[0] == Y_train.shape[0] _check_str_cov('get_optimized_kernel', str_cov, X_train.shape) assert str_optimizer_method in constants.ALLOWED_OPTIMIZER_METHOD_GP assert str_modelselection_method in constants.ALLOWED_MODELSELECTION_METHOD time_start = time.time() if debug: print('[DEBUG] get_optimized_kernel in gp.py: str_optimizer_method {}'. format(str_optimizer_method)) print( '[DEBUG] get_optimized_kernel in gp.py: str_modelselection_method {}' .format(str_modelselection_method)) prior_mu_train = get_prior_mu(prior_mu, X_train) if str_cov in constants.ALLOWED_GP_COV_BASE: num_dim = X_train.shape[1] elif str_cov in constants.ALLOWED_GP_COV_SET: num_dim = X_train.shape[2] if str_modelselection_method == 'ml': neg_log_ml = lambda hyps: -1.0 * log_ml(X_train, Y_train, hyps, str_cov, prior_mu_train, is_fixed_noise=is_fixed_noise, debug=debug) elif str_modelselection_method == 'loocv': neg_log_ml = lambda hyps: -1.0 * log_pseudo_l_loocv(X_train, Y_train, hyps, str_cov, prior_mu_train, is_fixed_noise= is_fixed_noise, debug=debug) else: # pragma: no cover raise ValueError( 'get_optimized_kernel: missing conditions for str_modelselection_method.' ) hyps_converted = utils_covariance.convert_hyps( str_cov, utils_covariance.get_hyps(str_cov, num_dim), is_fixed_noise=is_fixed_noise, ) if str_optimizer_method == 'BFGS': result_optimized = scipy.optimize.minimize(neg_log_ml, hyps_converted, method=str_optimizer_method) result_optimized = result_optimized.x elif str_optimizer_method == 'L-BFGS-B': bounds = utils_covariance.get_range_hyps(str_cov, num_dim, is_fixed_noise=is_fixed_noise) result_optimized = scipy.optimize.minimize(neg_log_ml, hyps_converted, method=str_optimizer_method, bounds=bounds) result_optimized = result_optimized.x # TODO: Fill this conditions elif str_optimizer_method == 'DIRECT': # pragma: no cover raise NotImplementedError( 'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.' ) elif str_optimizer_method == 'CMA-ES': # pragma: no cover raise NotImplementedError( 'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.' ) # INFO: It is allowed, but a condition is missed. else: # pragma: no cover raise ValueError( 'get_optimized_kernel: missing conditions for str_optimizer_method' ) hyps = utils_covariance.restore_hyps(str_cov, result_optimized, is_fixed_noise=is_fixed_noise) hyps, _ = utils_covariance.validate_hyps_dict(hyps, str_cov, num_dim) cov_X_X, inv_cov_X_X = get_kernel_inverse(X_train, hyps, str_cov, debug=debug) time_end = time.time() if debug: print('[DEBUG] get_optimized_kernel in gp.py: optimized hyps for gpr', hyps) print('[DEBUG] get_optimized_kernel in gp.py: time consumed', time_end - time_start, 'sec.') return cov_X_X, inv_cov_X_X, hyps
def test_neg_log_ml(): dim_X = 3 str_cov = 'se' X = np.reshape(np.arange(0, 9), (3, dim_X)) Y = np.expand_dims(np.arange(3, 10, 3), axis=1) fix_noise = False use_gp = False dict_hyps = utils_covariance.get_hyps(str_cov, dim_X, use_gp=use_gp) arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, fix_noise=fix_noise, use_gp=use_gp) prior_mu_X = np.zeros((3, 1)) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, dict_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, 1, prior_mu_X) with pytest.raises(ValueError) as error: package_target.neg_log_ml(X, Y, arr_hyps, 'abc', prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3)) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1)) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=1) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1) neg_log_ml_ = package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=fix_noise, use_gradient=False) print(neg_log_ml_) truth_log_ml_ = 5.634155417555853 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_, neg_grad_log_ml_ = package_target.neg_log_ml( X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=fix_noise, use_gradient=True) print(neg_log_ml_) print(neg_grad_log_ml_) print(neg_grad_log_ml_[2]) print(neg_grad_log_ml_[3]) print(neg_grad_log_ml_[4]) truth_log_ml_ = 5.634155417555853 truth_grad_log_ml_ = np.array([ -1.60446383e-02, 1.75087448e-01, -1.60448396e+00, -1.836237221888097e-05, -1.836237221888097e-05, -1.836237221888097e-05, ]) assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON assert np.all(np.abs(neg_grad_log_ml_ - truth_grad_log_ml_) < TEST_EPSILON) dict_hyps = utils_covariance.get_hyps(str_cov, dim_X, use_gp=use_gp) arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, fix_noise=True, use_gp=use_gp) neg_log_ml_, neg_grad_log_ml_ = package_target.neg_log_ml( X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=True, use_gradient=True) print(neg_log_ml_) print(neg_grad_log_ml_)
def get_optimized_kernel( X_train, Y_train, prior_mu, str_cov, str_optimizer_method=constants.STR_OPTIMIZER_METHOD_GP, str_modelselection_method=constants.STR_MODELSELECTION_METHOD, is_fixed_noise=constants.IS_FIXED_GP_NOISE, debug=False): """ This function computes the kernel matrix optimized by optimization method specified, its inverse matrix, and the optimized hyperparameters. :param X_train: inputs. Shape: (n, d) or (n, m, d). :type X_train: numpy.ndarray :param Y_train: outputs. Shape: (n, 1). :type Y_train: numpy.ndarray :param prior_mu: prior mean function or None. :type prior_mu: function or NoneType :param str_cov: the name of covariance function. :type str_cov: str. :param str_optimizer_method: the name of optimization method. :type str_optimizer_method: str., optional :param str_modelselection_method: the name of model selection method. :type str_modelselection_method: str., optional :param is_fixed_noise: flag for fixing a noise. :type is_fixed_noise: bool., optional :param debug: flag for printing log messages. :type debug: bool., optional :returns: a tuple of kernel matrix over `X_train`, kernel matrix inverse, and dictionary of hyperparameters. :rtype: tuple of (numpy.ndarray, numpy.ndarray, dict.) :raises: AssertionError, ValueError """ # TODO: check to input same is_fixed_noise to convert_hyps and restore_hyps assert isinstance(X_train, np.ndarray) assert isinstance(Y_train, np.ndarray) assert callable(prior_mu) or prior_mu is None assert isinstance(str_cov, str) assert isinstance(str_optimizer_method, str) assert isinstance(str_modelselection_method, str) assert isinstance(is_fixed_noise, bool) assert isinstance(debug, bool) assert len(Y_train.shape) == 2 assert X_train.shape[0] == Y_train.shape[0] _check_str_cov('get_optimized_kernel', str_cov, X_train.shape) assert str_optimizer_method in constants.ALLOWED_OPTIMIZER_METHOD_GP assert str_modelselection_method in constants.ALLOWED_MODELSELECTION_METHOD # TODO: fix this. is_gradient = True time_start = time.time() if debug: print('[DEBUG] get_optimized_kernel in gp.py: str_optimizer_method {}'. format(str_optimizer_method)) print( '[DEBUG] get_optimized_kernel in gp.py: str_modelselection_method {}' .format(str_modelselection_method)) prior_mu_train = get_prior_mu(prior_mu, X_train) if str_cov in constants.ALLOWED_GP_COV_BASE: num_dim = X_train.shape[1] elif str_cov in constants.ALLOWED_GP_COV_SET: num_dim = X_train.shape[2] is_gradient = False if str_modelselection_method == 'ml': neg_log_ml_ = lambda hyps: neg_log_ml(X_train, Y_train, hyps, str_cov, prior_mu_train, is_fixed_noise=is_fixed_noise, is_gradient=is_gradient, debug=debug) elif str_modelselection_method == 'loocv': neg_log_ml_ = lambda hyps: neg_log_pseudo_l_loocv(X_train, Y_train, hyps, str_cov, prior_mu_train, is_fixed_noise= is_fixed_noise, debug=debug) is_gradient = False else: # pragma: no cover raise ValueError( 'get_optimized_kernel: missing conditions for str_modelselection_method.' ) hyps_converted = utils_covariance.convert_hyps( str_cov, utils_covariance.get_hyps(str_cov, num_dim), is_fixed_noise=is_fixed_noise, ) if str_optimizer_method == 'BFGS': result_optimized = scipy.optimize.minimize(neg_log_ml_, hyps_converted, method=str_optimizer_method, jac=is_gradient, options={'disp': False}) result_optimized = result_optimized.x elif str_optimizer_method == 'L-BFGS-B': bounds = utils_covariance.get_range_hyps(str_cov, num_dim, is_fixed_noise=is_fixed_noise) result_optimized = scipy.optimize.minimize(neg_log_ml_, hyps_converted, method=str_optimizer_method, bounds=bounds, jac=is_gradient, options={'disp': False}) result_optimized = result_optimized.x # TODO: Fill this conditions elif str_optimizer_method == 'DIRECT': # pragma: no cover raise NotImplementedError( 'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.' ) elif str_optimizer_method == 'CMA-ES': # pragma: no cover raise NotImplementedError( 'get_optimized_kernel: allowed str_optimizer_method, but it is not implemented.' ) else: # pragma: no cover raise ValueError( 'get_optimized_kernel: missing conditions for str_optimizer_method' ) hyps = utils_covariance.restore_hyps(str_cov, result_optimized, is_fixed_noise=is_fixed_noise) hyps, _ = utils_covariance.validate_hyps_dict(hyps, str_cov, num_dim) cov_X_X, inv_cov_X_X, grad_cov_X_X = get_kernel_inverse( X_train, hyps, str_cov, is_fixed_noise=is_fixed_noise, debug=debug) time_end = time.time() if debug: print('[DEBUG] get_optimized_kernel in gp.py: optimized hyps for gpr', hyps) print('[DEBUG] get_optimized_kernel in gp.py: time consumed', time_end - time_start, 'sec.') return cov_X_X, inv_cov_X_X, hyps
def test_convert_hyps(): cur_hyps = { 'noise': 0.1, 'signal': 1.0, 'lengthscales': np.array([2.0, 2.0]) } with pytest.raises(AssertionError) as error: package_target.convert_hyps(1.2, 2.1) with pytest.raises(AssertionError) as error: package_target.convert_hyps(1.2, dict()) with pytest.raises(AssertionError) as error: package_target.convert_hyps('se', 2.1) with pytest.raises(AssertionError) as error: package_target.convert_hyps('abc', 2.1) with pytest.raises(AssertionError) as error: package_target.convert_hyps('abc', cur_hyps) with pytest.raises(AssertionError) as error: package_target.convert_hyps('se', dict(), fix_noise=1) with pytest.raises(AssertionError) as error: package_target.convert_hyps('se', cur_hyps, use_gp=1) with pytest.raises(AssertionError) as error: package_target.convert_hyps('se', cur_hyps, use_gp='abc') converted_hyps = package_target.convert_hyps('se', cur_hyps, fix_noise=False) assert len(converted_hyps.shape) == 1 assert converted_hyps.shape[0] == 4 assert converted_hyps[0] == cur_hyps['noise'] assert converted_hyps[1] == cur_hyps['signal'] assert (converted_hyps[2:] == cur_hyps['lengthscales']).all() converted_hyps = package_target.convert_hyps('se', cur_hyps, fix_noise=True) assert len(converted_hyps.shape) == 1 assert converted_hyps.shape[0] == 3 assert converted_hyps[0] == cur_hyps['signal'] assert (converted_hyps[1:] == cur_hyps['lengthscales']).all() cur_hyps = { 'noise': 0.1, 'signal': 1.0, 'lengthscales': np.array([2.0, 2.0]), 'dof': 100.0 } converted_hyps = package_target.convert_hyps('se', cur_hyps, fix_noise=False, use_gp=False) assert len(converted_hyps.shape) == 1 assert converted_hyps.shape[0] == 5 assert converted_hyps[0] == cur_hyps['noise'] assert converted_hyps[1] == cur_hyps['dof'] assert converted_hyps[2] == cur_hyps['signal'] assert (converted_hyps[3:] == cur_hyps['lengthscales']).all() converted_hyps = package_target.convert_hyps('se', cur_hyps, fix_noise=True, use_gp=False) assert len(converted_hyps.shape) == 1 assert converted_hyps.shape[0] == 4 assert converted_hyps[0] == cur_hyps['dof'] assert converted_hyps[1] == cur_hyps['signal'] assert (converted_hyps[2:] == cur_hyps['lengthscales']).all()
def test_neg_log_ml(): dim_X = 3 str_cov = 'se' X = np.reshape(np.arange(0, 9), (3, dim_X)) Y = np.expand_dims(np.arange(3, 10, 3), axis=1) fix_noise = False dict_hyps = utils_covariance.get_hyps(str_cov, dim_X) arr_hyps = utils_covariance.convert_hyps(str_cov, dict_hyps, fix_noise=fix_noise) prior_mu_X = np.zeros((3, 1)) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(np.arange(0, 3), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, np.arange(0, 3), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, dict_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, 1, prior_mu_X) with pytest.raises(ValueError) as error: package_target.neg_log_ml(X, Y, arr_hyps, 'abc', prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, np.arange(0, 3)) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(np.reshape(np.arange(0, 12), (4, dim_X)), Y, arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, np.expand_dims(np.arange(0, 4), axis=1), arr_hyps, str_cov, prior_mu_X) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, np.expand_dims(np.arange(0, 4), axis=1)) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, use_cholesky=1) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=1) with pytest.raises(AssertionError) as error: package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, debug=1) neg_log_ml_ = package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=fix_noise, use_gradient=False, use_cholesky=True) print(neg_log_ml_) truth_log_ml_ = 21.916650988532854 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_ = package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=fix_noise, use_gradient=False, use_cholesky=False) print(neg_log_ml_) truth_log_ml_ = 21.91665090519953 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_ = package_target.neg_log_ml(X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=fix_noise, use_gradient=True, use_cholesky=False) print(neg_log_ml_) truth_log_ml_ = 21.91665090519953 assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON neg_log_ml_, neg_grad_log_ml_ = package_target.neg_log_ml( X, Y, arr_hyps, str_cov, prior_mu_X, fix_noise=fix_noise, use_gradient=True, use_cholesky=True) print(neg_log_ml_) print(neg_grad_log_ml_) print(neg_grad_log_ml_[2]) print(neg_grad_log_ml_[3]) print(neg_grad_log_ml_[4]) truth_log_ml_ = 21.916650988532854 truth_grad_log_ml_ = np.array([ -4.09907399e-01, -4.09912156e+01, -0.00029606081917620415, -0.00029606081917620415, -0.00029606081917620415, ]) assert np.abs(neg_log_ml_ - truth_log_ml_) < TEST_EPSILON assert np.all(np.abs(neg_grad_log_ml_ - truth_grad_log_ml_) < TEST_EPSILON)