def param_maximum_likelihood(gpy_gp: GPy.models.GPRegression,
test_model: bool = True,
test_X: np.ndarray = None,
test_Y: np.ndarray = None,
variance_prior: GPy.priors.Prior = None,
lengthscale_prior: GPy.priors.Prior = None,
noise_prior: GPy.priors.Prior = None,
fix_noise_params=None):
"""
Find the maximum likelihood estimates of the hyperparameters. The last element in the vector returned is the
Gaussian noise hyperparameter that we are interested in.
The hyperparameters correspond to
$
\theta^* = argmin(-log(P(Y|\theta)))
$
If prior_mean and prior_var arguments (the parameter prior and covariance, assuming a Gaussian parameter prior
$p(\theta)$ are provided, the optimised hyperparameters corrspond to the MAP-II estimate:
$
\theta^* = argmin(-log(P(Y|/theta) - log(P(\theta))
$
:param gpy_gp: An initialised GPy GPRegression object
:param test_model: toggle whether to display the information of the fitted GPRegression model
:return: The vector of hyperparameters found from maximum likelihood estimate
"""
if test_model and (test_X is None or test_Y is None):
raise ValueError()
if variance_prior is not None:
gpy_gp.kern.variance.set_prior(variance_prior)
if lengthscale_prior is not None:
gpy_gp.kern.lengthscale.set_prior(lengthscale_prior)
if noise_prior is not None:
gpy_gp.Gaussian_noise.variance.set_prior(noise_prior)
if fix_noise_params is not None:
gpy_gp.Gaussian_noise[:] = fix_noise_params
gpy_gp.Gaussian_noise.variance.fix()
start = time.time()
gpy_gp.optimize(messages=True, max_iters=MLE_optimisation_iteration)
gpy_gp.optimize_restarts(num_restarts=MLE_optimisation_restart)
res = [
gpy_gp.kern.variance, gpy_gp.kern.lengthscale,
gpy_gp.Gaussian_noise.variance
]
end = time.time()
rmse, ll = np.nan, np.nan
if test_model:
if noise_prior is not None: model = 'MAP'
else: model = 'MLE'
print("----------------- Testing " + model + " -------------------")
print("Fix noise hyperparameter ?", fix_noise_params)
print("Clock time: ", end - start)
display(gpy_gp)
rmse, ll = test_gp(
gpy_gp,
test_X,
test_Y,
)
return gpy_gp, res, rmse, ll, end - start
def lin_shrinkage(gpy_gp: GPy.models.GPRegression,
test_model: bool = True,
test_X: np.ndarray = None,
test_Y: np.ndarray = None,
c: float = 1e-6,
plot_stuff=False):
"""
Use linear shrinkage method to estimate the Gaussian noise hyperparameter
:param gpy_gp:
:param test_model:
:return:
"""
start = time.time()
train_x = gpy_gp.X
priors = GPy.priors.LogGaussian(mu=0., sigma=2.)
gpy_gp.kern.variance.set_prior(priors)
gpy_gp.kern.lengthscale.set_prior(priors)
i, j = 0, 0
metrics = np.empty((5, 3 + gpy_gp.input_dim))
kern_dim = 1 + gpy_gp.input_dim
gpy_gp.Gaussian_noise.fix()
lml = -np.inf
while j < 1:
i = 0
theta = np.exp(
multivariate_normal(mean=np.zeros((kern_dim, )),
cov=4 * np.eye(kern_dim)).rvs())
while i < 1000:
gpy_gp.optimize(start=theta, max_iters=1)
K = gpy_gp.kern.K(train_x)
eig = np.linalg.eigvals(K).real
eig = np.sort(eig)[::-1]
delta_lambda = np.diff(eig)
try:
n_outlier = delta_lambda[np.abs(delta_lambda) > c].argmax()
except ValueError:
n_outlier = 0
eig_bulk = eig[n_outlier:]
# bm = np.sum(eig_bulk) / eig_bulk.shape[0]
lambda_b = np.median(eig_bulk)
alpha = 1. / (1. + lambda_b)
gpy_gp.Gaussian_noise.variance[:] = (1 - alpha)
gpy_gp.kern.variance[:] *= alpha
theta = gpy_gp.param_array[:-1]
# print(i, lml, lambda_b)
# optimise the rest of the hyperparameters
lml = gpy_gp.log_likelihood()
i += 1
if plot_stuff:
n_eig = eig.shape[0]
plt.subplot(211)
plt.axvline(int(len(eig) / 2))
plt.plot(np.log(eig), marker='.', linestyle='None')
plt.subplot(212)
plt.hist(eig[int(len(eig) / 5):], bins=80)
plt.axvline(lambda_b)
plt.show()
print('LML After Optimisation Restart', gpy_gp.log_likelihood())
print('Eig lambda_b/avg', lambda_b)
metrics[j, :-1] = gpy_gp.param_array
metrics[j, -1] = lml
j += 1
# print(metrics)
best_iter = metrics[:, -1].argmax()
gpy_gp.kern.variance[:] = metrics[best_iter, 0]
gpy_gp.kern.lengthscale[:] = metrics[best_iter, 1:-2]
gpy_gp.Gaussian_noise.variance[:] = metrics[best_iter, -2]
# print(metrics)
final_params = metrics[best_iter, :-1]
end = time.time()
rmse, ll = np.nan, np.nan
if test_model:
print("----------------- Testing Linear Shrinkage -------------------")
display(gpy_gp)
print("Clock time: ", end - start)
rmse, ll = test_gp(
gpy_gp,
test_X,
test_Y,
)
gpy_gp.Gaussian_noise.unfix()
gpy_gp.unset_priors()
return gpy_gp, final_params, rmse, ll, end - start
def param_thikonov(gpy_gp: GPy.models.GPRegression,
test_model: bool = True,
test_X: np.ndarray = None,
test_Y: np.ndarray = None,
c: float = 1e-3,
plot_stuff=False,
save_stuff=True):
"""
Use Thikonov Regularisation method to estimate the Gaussian noise hyperparameter
1. We compute the eigenvalue-eigenvector decomposition of the K matrix
2. Compute the number of outliers in the eigenspectrum
3. Estimate the bulk mean of the eigenvalues - this will be used as the noise variance
:param gpy_gp:
:param test_model:
:return:
"""
start = time.time()
train_x = gpy_gp.X
priors = GPy.priors.LogGaussian(mu=0., sigma=2.)
gpy_gp.kern.variance.set_prior(priors)
gpy_gp.kern.lengthscale.set_prior(priors)
i, j = 0, 0
metrics = np.empty((10, 3 + gpy_gp.input_dim))
kern_dim = 1 + gpy_gp.input_dim
gpy_gp.Gaussian_noise.fix()
lml = -np.inf
C = np.zeros((10, 25))
while j < 10:
i = 0
theta = np.exp(
multivariate_normal(mean=np.zeros((kern_dim, )),
cov=4 * np.eye(kern_dim)).rvs())
while i < 25:
gpy_gp.optimize(start=theta, max_iters=20)
K = gpy_gp.kern.K(train_x)
eig = np.linalg.eigvals(K).real
eig = np.sort(eig)[::-1]
sigma = np.std(K)
delta_lambda = np.empty(eig.shape[0] - 1)
for k in range(1, eig.shape[0]):
delta_lambda[k - 1] = (eig[k] - eig[k - 1]) / eig[0]
try:
n_outlier = delta_lambda[np.abs(delta_lambda) > c].argmax()
except ValueError:
n_outlier = 0
eig_bulk = eig[n_outlier:]
# eig_bulk = eig[eig <= eig_max]
# bm = np.sum(eig_bulk) / eig_bulk.shape[0]
med = np.median(eig_bulk)
# mean = np.median(eig)
gpy_gp.Gaussian_noise.variance[:] = med
theta = gpy_gp.param_array[:-1]
# print(i, lml, med)
# optimise the rest of the hyperparameters
lml = gpy_gp.log_likelihood()
if save_stuff:
C[j, i] = med
if i % 10 == 0:
#np.savetxt('output/7Mar/covariance_matrix_'+str(j)+'_'+str(i)+'.txt', K)
#np.savetxt('output/7Mar/median_bulk_'+str(j)+'_'+str(i)+'.txt', med.reshape(1, 1))
pass
i += 1
if plot_stuff:
plt.subplot(211)
plt.axvline(int(len(eig) / 2))
plt.plot(np.log(eig), marker='.', linestyle='None')
plt.subplot(212)
plt.hist(eig[int(len(eig) / 5):], bins=80)
plt.axvline(med)
plt.show()
print('LML After Optimisation Restart', gpy_gp.log_likelihood(), j)
print('Eig med/avg', med)
metrics[j, :-1] = gpy_gp.param_array
metrics[j, -1] = lml
j = j + 1
# print(metrics)
best_iter = metrics[:, -1].argmax()
gpy_gp.kern.variance[:] = metrics[best_iter, 0]
gpy_gp.kern.lengthscale[:] = metrics[best_iter, 1:-2]
gpy_gp.Gaussian_noise.variance[:] = metrics[best_iter, -2]
# print(metrics)
np.savetxt('output/7Mar/beta_.txt', C)
final_params = metrics[best_iter, :-1]
end = time.time()
rmse, ll = np.nan, np.nan
if test_model:
print("----------------- Testing Thikonov -------------------")
display(gpy_gp)
print("Clock time: ", end - start)
rmse, ll = test_gp(
gpy_gp,
test_X,
test_Y,
)
gpy_gp.Gaussian_noise.unfix()
gpy_gp.unset_priors()
return gpy_gp, final_params, rmse, ll, end - start