def _SLcholesky_autoreg(C, nsteps=None, **kwargs):
"""Simple wrapper around cholesky to incrementally regularize the
matrix until successful computation.
For `nsteps` we boost diagonal 10-fold each time from the
'epsilon' of the respective dtype. If None -- would proceed until
reaching 1.
"""
if nsteps is None:
nsteps = -int(np.floor(np.log10(np.finfo(float).eps)))
result = None
for step in xrange(nsteps):
epsilon_value = (10**step) * np.finfo(C.dtype).eps
epsilon = epsilon_value * np.eye(C.shape[0])
try:
result = SLcholesky(C + epsilon, lower=True)
except SLAError, e:
warning("Cholesky decomposition lead to failure: %s. "
"As requested, performing auto-regularization but "
"for better control you might prefer to regularize "
"yourself by providing lm parameter to GPR" % e)
if step < nsteps - 1:
if __debug__:
debug(
"GPR", "Failed to obtain cholesky on "
"auto-regularization step %d value %g. Got %s."
" Boosting lambda more to reg. C." %
(step, epsilon_value, e))
continue
else:
raise
def _train(self, data):
"""Train the classifier using `data` (`Dataset`).
"""
# local bindings for faster lookup
params = self.params
retrainable = params.retrainable
if retrainable:
newkernel = False
newL = False
_changedData = self._changedData
self._train_fv = train_fv = data.samples
# GRP relies on numerical labels
# yoh: yeah -- GPR now is purely regression so no conversion
# is necessary
train_labels = data.sa[self.get_space()].value
self._train_labels = train_labels
if not retrainable or _changedData['traindata'] \
or _changedData.get('kernel_params', False):
if __debug__:
debug("GPR", "Computing train train kernel matrix")
self.__kernel.compute(train_fv)
self._km_train_train = km_train_train = asarray(self.__kernel)
newkernel = True
if retrainable:
self._km_train_test = None # reset to facilitate recomputation
else:
if __debug__:
debug(
"GPR", "Not recomputing kernel since retrainable and "
"nothing has changed")
km_train_train = self._km_train_train # reuse
if not retrainable or newkernel or _changedData['params']:
if __debug__:
debug("GPR", "Computing L. sigma_noise=%g" \
% params.sigma_noise)
# XXX it seems that we do not need binding to object, but may be
# commented out code would return?
self._C = km_train_train + \
params.sigma_noise ** 2 * \
np.identity(km_train_train.shape[0], 'd')
# The following decomposition could raise
# np.linalg.linalg.LinAlgError because of numerical
# reasons, due to the too rapid decay of 'self._C'
# eigenvalues. In that case we try adding a small constant
# to self._C, e.g. epsilon=1.0e-20. It should be a form of
# Tikhonov regularization. This is equivalent to adding
# little white gaussian noise to data.
#
# XXX EO: how to choose epsilon?
#
# Cholesky decomposition is provided by three different
# NumPy/SciPy routines (fastest first):
# 1) self._LL = scipy.linalg.cho_factor(self._C, lower=True)
# self._L = L = np.tril(self._LL[0])
# 2) self._L = scipy.linalg.cholesky(self._C, lower=True)
# 3) self._L = numpy.linalg.cholesky(self._C)
# Even though 1 is the fastest we choose 2 since 1 does
# not return a clean lower-triangular matrix (see docstring).
# PBS: I just made it so the KernelMatrix is regularized
# all the time. I figured that if ever you were going to
# use regularization, you would want to set it yourself
# and use the same value for all folds of your data.
# YOH: Ideally so, but in real "use cases" some might have no
# clue, also our unittests (actually clfs_examples) might
# fail without any good reason. So lets return a magic with
# an option to forbid any regularization (if lm is None)
try:
# apply regularization
lm, C = params.lm, self._C
if lm is not None:
epsilon = lm * np.eye(C.shape[0])
self._L = SLcholesky(C + epsilon, lower=True)
else:
# do 10 attempts to raise each time by 10
self._L = _SLcholesky_autoreg(C, nsteps=None, lower=True)
self._LL = (self._L, True)
except SLAError:
raise SLAError("Kernel matrix is not positive, definite. "
"Try increasing the lm parameter.")
pass
newL = True
else:
if __debug__:
debug(
"GPR", "Not computing L since kernel, data and params "
"stayed the same")
# XXX we leave _alpha being recomputed, although we could check
# if newL or _changedData['targets']
#
if __debug__:
debug("GPR", "Computing alpha")
# L = self._L # reuse
# self._alpha = NLAsolve(L.transpose(),
# NLAsolve(L, train_labels))
# Faster:
self._alpha = SLcho_solve(self._LL, train_labels)
# compute only if the state is enabled
if self.ca.is_enabled('log_marginal_likelihood'):
self.compute_log_marginal_likelihood()
pass
if retrainable:
# we must assign it only if it is retrainable
self.ca.retrained = not newkernel or not newL
if __debug__:
debug("GPR", "Done training")
pass
"for better control you might prefer to regularize "
"yourself by providing lm parameter to GPR" % e)
if step < nsteps - 1:
if __debug__:
debug(
"GPR", "Failed to obtain cholesky on "
"auto-regularization step %d value %g. Got %s."
" Boosting lambda more to reg. C." %
(step, epsilon_value, e))
continue
else:
raise
if result is None:
# no loop was done for some reason
result = SLcholesky(C, lower=True)
return result
class GPR(Classifier):
"""Gaussian Process Regression (GPR).
"""
predicted_variances = ConditionalAttribute(
enabled=False, doc="Variance per each predicted value")
log_marginal_likelihood = ConditionalAttribute(
enabled=False, doc="Log Marginal Likelihood")