class GroupClusterThreshold_NN3(Learner):
"""Statistical evaluation of group-level average accuracy maps
This algorithm can be used to perform cluster-thresholding of
searchlight-based group analyses. It implements a two-stage procedure that
uses the results of within-subject permutation analyses, estimates a per
feature cluster forming threshold (via bootstrap), and uses the thresholded
bootstrap samples to estimate the distribution of cluster sizes in
group-average accuracy maps under the NULL hypothesis, as described in [1]_.
Note: this class implements a modified version of that algorithm. The
present implementation differs in, at least, four aspects from the
description in that paper.
1) Cluster p-values refer to the probability of observing a particular
cluster size or a larger one (original paper: probability to observe a
larger cluster only). Consequently, probabilities reported by this
implementation will have a tendency to be higher in comparison.
2) Clusters found in the original (unpermuted) accuracy map are always
included in the NULL distribution estimate of cluster sizes. This
provides an explicit lower bound for probabilities, as there will
always be at least one observed cluster for every cluster size found
in the original accuracy map. Consequently, it is impossible to get a
probability of zero for clusters of any size (see [2] for more
information).
3) Bootstrap accuracy maps that contain no clusters are counted in a
dedicated size-zero bin in the NULL distribution of cluster sizes.
This change yields reliable cluster-probabilities even for very low
featurewise threshold probabilities, where (some portion) of the
bootstrap accuracy maps do not contain any clusters.
4) The method for FWE-correction used by the original authors is not
provided. Instead, a range of alternatives implemented by the
statsmodels package are available.
Moreover, this implementation minimizes the required memory demands and
allows for computing large numbers of bootstrap samples without
significant increase in memory demand (CPU time trade-off).
Instances of this class must be trained before than can be used to
threshold accuracy maps. The training dataset must match the following
criteria:
1) For every subject in the group, it must contain multiple accuracy maps
that are the result of a within-subject classification analysis
based on permuted class labels. One map must corresponds to one fixed
permutation for all features in the map, as described in [1]_. The
original authors recommend 100 accuracy maps per subject for a typical
searchlight analysis.
2) It must contain a sample attribute indicating which sample is
associated with which subject, because bootstrapping average accuracy
maps is implemented by randomly drawing one map from each subject.
The name of the attribute can be configured via the ``chunk_attr``
parameter.
After training, an instance can be called with a dataset to perform
threshold and statistical evaluation. Unless a single-sample dataset
is passed, all samples in the input dataset will be averaged prior
thresholding.
Returns
-------
Dataset
This is a shallow copy of the input dataset (after a potential
averaging), hence contains the same data and attributes. In addition it
includes the following attributes:
``fa.featurewise_thresh``
Vector with feature-wise cluster-forming thresholds.
``fa.clusters_featurewise_thresh``
Vector with labels for clusters after thresholding the input data
with the desired feature-wise probability. Each unique non-zero
element corresponds to an individual super-threshold cluster. Cluster
values are sorted by cluster size (number of features). The largest
cluster is always labeled with ``1``.
``fa.clusters_fwe_thresh``
Vector with labels for super-threshold clusters after correction for
multiple comparisons. The attribute is derived from
``fa.clusters_featurewise_thresh`` by removing all clusters that
do not pass the threshold when controlling for the family-wise error
rate.
``a.clusterstats``
Record array with information on all detected clusters. The array is
sorted according to cluster size, starting with the largest cluster
in terms of number of features. The array contains the fields ``size``
(number of features comprising the cluster), ``mean``, ``median``,
min``, ``max``, ``std`` (respective descriptive statistics for all
clusters), and ``prob_raw`` (probability of observing the cluster of a
this size or larger under the NULL hypothesis). If correction for
multiple comparisons is enabled an additional field ``prob_corrected``
(probability after correction) is added.
``a.clusterlocations``
Record array with information on the location of all detected clusters.
The array is sorted according to cluster size (same order as
``a.clusterstats``. The array contains the fields ``max``
(feature coordinate of the maximum score within the cluster, and
``center_of_mass`` (coordinate of the center of mass; weighted by
the feature values within the cluster.
References
----------
.. [1] Johannes Stelzer, Yi Chen and Robert Turner (2013). Statistical
inference and multiple testing correction in classification-based
multi-voxel pattern analysis (MVPA): Random permutations and cluster
size control. NeuroImage, 65, 69--82.
.. [2] Smyth, G. K., & Phipson, B. (2010). Permutation P-values Should
Never Be Zero: Calculating Exact P-values When Permutations Are
Randomly Drawn. Statistical Applications in Genetics and Molecular
Biology, 9, 1--12.
"""
n_bootstrap = Parameter(
100000, constraints=EnsureInt() & EnsureRange(min=1),
doc="""Number of bootstrap samples to be generated from the training
dataset. For each sample, an average map will be computed from a
set of randomly drawn samples (one from each chunk). Bootstrap
samples will be used to estimate a featurewise NULL distribution of
accuracy values for initial thresholding, and to estimate the NULL
distribution of cluster sizes under the NULL hypothesis. A larger
number of bootstrap samples reduces the lower bound of
probabilities, which may be beneficial for multiple comparison
correction.""")
feature_thresh_prob = Parameter(
0.001, constraints=EnsureFloat() & EnsureRange(min=0.0, max=1.0),
doc="""Feature-wise probability threshold. The value corresponding
to this probability in the NULL distribution of accuracies will
be used as threshold for cluster forming. Given that the NULL
distribution is estimated per feature, the actual threshold value
will vary across features yielding a threshold vector. The number
of bootstrap samples need to be adequate for a desired probability.
A ``ValueError`` is raised otherwise.""")
chunk_attr = Parameter(
'chunks',
doc="""Name of the attribute indicating the individual chunks from
which a single sample each is drawn for averaging into a bootstrap
sample.""")
fwe_rate = Parameter(
0.05, constraints=EnsureFloat() & EnsureRange(min=0.0, max=1.0),
doc="""Family-wise error rate for multiple comparison correction
of cluster size probabilities.""")
multicomp_correction = Parameter(
'fdr_bh', constraints=EnsureChoice('bonferroni', 'sidak', 'holm-sidak',
'holm', 'simes-hochberg', 'hommel',
'fdr_bh', 'fdr_by', None),
doc="""Strategy for multiple comparison correction of cluster
probabilities. All methods supported by statsmodels' ``multitest``
are available. In addition, ``None`` can be specified to disable
correction.""")
n_blocks = Parameter(
1, constraints=EnsureInt() & EnsureRange(min=1),
doc="""Number of segments used to compute the feature-wise NULL
distributions. This parameter determines the peak memory demand.
In case of a single segment a matrix of size
(n_bootstrap x nfeatures) will be allocated. Increasing the number
of segments reduces the peak memory demand by that roughly factor.
""")
n_proc = Parameter(
1, constraints=EnsureInt() & EnsureRange(min=1),
doc="""Number of parallel processes to use for computation.
Requires `joblib` external module.""")
def __init__(self, **kwargs):
# force disable auto-train: would make no sense
Learner.__init__(self, auto_train=False, **kwargs)
if 1. / (self.params.n_bootstrap + 1) > self.params.feature_thresh_prob:
raise ValueError('number of bootstrap samples is insufficient for'
' the desired threshold probability')
self.untrain()
def _untrain(self):
self._thrmap = None
self._null_cluster_sizes = None
@due.dcite(
Doi("10.1016/j.neuroimage.2012.09.063"),
description="Statistical assessment of (searchlight) MVPA results",
tags=['implementation'])
def _train(self, ds):
# shortcuts
chunk_attr = self.params.chunk_attr
#
# Step 0: bootstrap maps by drawing one for each chunk and average them
# (do N iterations)
# this could take a lot of memory, hence instead of computing the maps
# we compute the source maps they can be computed from and then (re)build
# the matrix of bootstrapped maps either row-wise or column-wise (as
# needed) to save memory by a factor of (close to) `n_bootstrap`
# which samples belong to which chunk
chunk_samples = dict([(c, np.where(ds.sa[chunk_attr].value == c)[0])
for c in ds.sa[chunk_attr].unique])
# pre-built the bootstrap combinations
bcombos = [[random.sample(v, 1)[0] for v in chunk_samples.values()]
for i in xrange(self.params.n_bootstrap)]
bcombos = np.array(bcombos, dtype=int)
#
# Step 1: find the per-feature threshold that corresponds to some p
# in the NULL
segwidth = ds.nfeatures / self.params.n_blocks
# speed things up by operating on an array not a dataset
ds_samples = ds.samples
if __debug__:
debug('GCTHR',
'Compute per-feature thresholds in %i blocks of %i features'
% (self.params.n_blocks, segwidth))
# Execution can be done in parallel as the estimation is independent
# across features
def featuresegment_producer(ncols):
for segstart in xrange(0, ds.nfeatures, ncols):
# one average map for every stored bcombo
# this also slices the input data into feature subsets
# for the compute blocks
yield [np.mean(
# get a view to a subset of the features
# -- should be somewhat efficient as feature axis is
# sliced
ds_samples[sidx, segstart:segstart + ncols],
axis=0)
for sidx in bcombos]
if self.params.n_proc == 1:
# Serial execution
thrmap = np.hstack( # merge across compute blocks
[get_thresholding_map(d, self.params.feature_thresh_prob)
# compute a partial threshold map for as many features
# as fit into a compute block
for d in featuresegment_producer(segwidth)])
else:
# Parallel execution
verbose_level_parallel = 50 \
if (__debug__ and 'GCTHR' in debug.active) else 0
# local import as only parallel execution needs this
from joblib import Parallel, delayed
# same code as above, just in parallel with joblib's Parallel
thrmap = np.hstack(
Parallel(n_jobs=self.params.n_proc,
pre_dispatch=self.params.n_proc,
verbose=verbose_level_parallel)(
delayed(get_thresholding_map)
(d, self.params.feature_thresh_prob)
for d in featuresegment_producer(segwidth)))
# store for later thresholding of input data
self._thrmap = thrmap
#
# Step 2: threshold all NULL maps and build distribution of NULL cluster
# sizes
#
cluster_sizes = Counter()
# recompute the bootstrap average maps to threshold them and determine
# cluster sizes
dsa = dict(mapper=ds.a.mapper) if 'mapper' in ds.a else {}
if __debug__:
debug('GCTHR', 'Estimating NULL distribution of cluster sizes')
# this step can be computed in parallel chunks to speeds things up
if self.params.n_proc == 1:
# Serial execution
for sidx in bcombos:
avgmap = np.mean(ds_samples[sidx], axis=0)[None]
# apply threshold
clustermap = avgmap > thrmap
# wrap into a throw-away dataset to get the reverse mapping right
bds = Dataset(clustermap, a=dsa)
# this function reverse-maps every sample one-by-one, hence no need
# to collect chunks of bootstrapped maps
cluster_sizes = get_cluster_sizes(bds, cluster_sizes)
else:
# Parallel execution
# same code as above, just restructured for joblib's Parallel
for jobres in Parallel(n_jobs=self.params.n_proc,
pre_dispatch=self.params.n_proc,
verbose=verbose_level_parallel)(
delayed(get_cluster_sizes)
(Dataset(np.mean(ds_samples[sidx],
axis=0)[None] > thrmap,
a=dsa))
for sidx in bcombos):
# aggregate
cluster_sizes += jobres
# store cluster size histogram for later p-value evaluation
# use a sparse matrix for easy consumption (max dim is the number of
# features, i.e. biggest possible cluster)
scl = dok_matrix((1, ds.nfeatures + 1), dtype=int)
for s in cluster_sizes:
scl[0, s] = cluster_sizes[s]
self._null_cluster_sizes = scl
def _call(self, ds):
if len(ds) > 1:
# average all samples into one, assuming we got something like one
# sample per subject as input
avgr = mean_sample()
ds = avgr(ds)
# threshold input; at this point we only have one sample left
thrd = ds.samples[0] > self._thrmap
# mapper default
mapper = IdentityMapper()
# overwrite if possible
if hasattr(ds, 'a') and 'mapper' in ds.a:
mapper = ds.a.mapper
# reverse-map input
othrd = _verified_reverse1(mapper, thrd)
# TODO: what is your purpose in life osamp? ;-)
osamp = _verified_reverse1(mapper, ds.samples[0])
# prep output dataset
outds = ds.copy(deep=False)
outds.fa['featurewise_thresh'] = self._thrmap
# determine clusters
labels, num = measurements.label(othrd,structure=np.ones([3,3,3]))
area = measurements.sum(othrd,
labels,
index=np.arange(1, num + 1)).astype(int)
com = measurements.center_of_mass(
osamp, labels=labels, index=np.arange(1, num + 1))
maxpos = measurements.maximum_position(
osamp, labels=labels, index=np.arange(1, num + 1))
# for the rest we need the labels flattened
labels = mapper.forward1(labels)
# relabel clusters starting with the biggest and increase index with
# decreasing size
ordered_labels = np.zeros(labels.shape, dtype=int)
ordered_area = np.zeros(area.shape, dtype=int)
ordered_com = np.zeros((num, len(osamp.shape)), dtype=float)
ordered_maxpos = np.zeros((num, len(osamp.shape)), dtype=float)
for i, idx in enumerate(np.argsort(area)):
ordered_labels[labels == idx + 1] = num - i
# kinda ugly, but we are looping anyway
ordered_area[i] = area[idx]
ordered_com[i] = com[idx]
ordered_maxpos[i] = maxpos[idx]
labels = ordered_labels
area = ordered_area[::-1]
com = ordered_com[::-1]
maxpos = ordered_maxpos[::-1]
del ordered_labels # this one can be big
# store cluster labels after forward-mapping
outds.fa['clusters_featurewise_thresh'] = labels.copy()
# location info
outds.a['clusterlocations'] = \
np.rec.fromarrays(
[com, maxpos], names=('center_of_mass', 'max'))
# update cluster size histogram with the actual result to get a
# proper lower bound for p-values
# this will make a copy, because the original matrix is int
cluster_probs_raw = _transform_to_pvals(
area, self._null_cluster_sizes.astype('float'))
clusterstats = (
[area, cluster_probs_raw],
['size', 'prob_raw']
)
# evaluate a bunch of stats for all clusters
morestats = {}
for cid in xrange(len(area)):
# keep clusters on outer loop, because selection is more expensive
clvals = ds.samples[0, labels == cid + 1]
for id_, fx in (
('mean', np.mean),
('median', np.median),
('min', np.min),
('max', np.max),
('std', np.std)):
stats = morestats.get(id_, [])
stats.append(fx(clvals))
morestats[id_] = stats
for k, v in morestats.items():
clusterstats[0].append(v)
clusterstats[1].append(k)
if self.params.multicomp_correction is not None:
# do a local import as only this tiny portion needs statsmodels
import statsmodels.stats.multitest as smm
rej, probs_corr = smm.multipletests(
cluster_probs_raw,
alpha=self.params.fwe_rate,
method=self.params.multicomp_correction)[:2]
# store corrected per-cluster probabilities
clusterstats[0].append(probs_corr)
clusterstats[1].append('prob_corrected')
# remove cluster labels that did not pass the FWE threshold
for i, r in enumerate(rej):
if not r:
labels[labels == i + 1] = 0
outds.fa['clusters_fwe_thresh'] = labels
outds.a['clusterstats'] = \
np.rec.fromarrays(clusterstats[0], names=clusterstats[1])
return outds
class GPR(Classifier):
"""Gaussian Process Regression (GPR).
"""
predicted_variances = ConditionalAttribute(
enabled=False, doc="Variance per each predicted value")
log_marginal_likelihood = ConditionalAttribute(
enabled=False, doc="Log Marginal Likelihood")
log_marginal_likelihood_gradient = ConditionalAttribute(
enabled=False, doc="Log Marginal Likelihood Gradient")
__tags__ = ['gpr', 'regression', 'retrainable']
# NOTE XXX Parameters of the classifier. Values available as
# clf.parameter or clf.params.parameter, or as
# clf.params['parameter'] (as the full Parameter object)
#
# __doc__ and __repr__ for class is conviniently adjusted to
# reflect values of those params
# Kernel machines/classifiers should be refactored also to behave
# the same and define kernel parameter appropriately... TODO, but SVMs
# already kinda do it nicely ;-)
sigma_noise = Parameter(
0.001,
constraints=EnsureFloat() & EnsureRange(min=1e-10),
doc="the standard deviation of the gaussian noise.")
# XXX For now I don't introduce kernel parameter since yet to unify
# kernel machines
#kernel = Parameter(None, allowedtype='Kernel',
# doc="Kernel object defining the covariance between instances. "
# "(Defaults to KernelSquaredExponential if None in arguments)")
lm = Parameter(None,
constraints=((EnsureFloat() & EnsureRange(min=0.0))
| EnsureNone()),
doc="""The regularization term lambda.
Increase this when the kernel matrix is not positive definite. If None,
some regularization will be provided upon necessity""")
def __init__(self, kernel=None, **kwargs):
"""Initialize a GPR regression analysis.
Parameters
----------
kernel : Kernel
a kernel object defining the covariance between instances.
(Defaults to SquaredExponentialKernel if None in arguments)
"""
# init base class first
Classifier.__init__(self, **kwargs)
# It does not make sense to calculate a confusion matrix for a GPR
# XXX it does ;) it will be a RegressionStatistics actually ;-)
# So if someone desires -- let him have it
# self.ca.enable('training_stats', False)
# set kernel:
if kernel is None:
kernel = SquaredExponentialKernel()
debug(
"GPR",
"No kernel was provided, falling back to default: %s" % kernel)
self.__kernel = kernel
# append proper clf_internal depending on the kernel
# TODO: add "__tags__" to kernels since the check
# below does not scale
if isinstance(kernel, (GeneralizedLinearKernel, LinearKernel)):
self.__tags__ += ['linear']
else:
self.__tags__ += ['non-linear']
if externals.exists('openopt') \
and not 'has_sensitivity' in self.__tags__:
self.__tags__ += ['has_sensitivity']
# No need to initialize conditional attributes. Unless they got set
# they would raise an exception self.predicted_variances =
# None self.log_marginal_likelihood = None
self._init_internals()
pass
def _init_internals(self):
"""Reset some internal variables to None.
To be used in constructor and untrain()
"""
self._train_fv = None
self._labels = None
self._km_train_train = None
self._train_labels = None
self._alpha = None
self._L = None
self._LL = None
# XXX EO: useful for model selection but not working in general
# self.__kernel.reset()
pass
def __repr__(self):
"""String summary of the object
"""
return super(GPR,
self).__repr__(prefixes=['kernel=%s' % self.__kernel])
def compute_log_marginal_likelihood(self):
"""
Compute log marginal likelihood using self.train_fv and self.targets.
"""
if __debug__:
debug("GPR", "Computing log_marginal_likelihood")
self.ca.log_marginal_likelihood = \
-0.5*Ndot(self._train_labels, self._alpha) - \
Nlog(self._L.diagonal()).sum() - \
self._km_train_train.shape[0] * _halflog2pi
return self.ca.log_marginal_likelihood
def compute_gradient_log_marginal_likelihood(self):
"""Compute gradient of the log marginal likelihood. This
version use a more compact formula provided by Williams and
Rasmussen book.
"""
# XXX EO: check whether the precomputed self.alpha self.Kinv
# are actually the ones corresponding to the hyperparameters
# used to compute this gradient!
# YYY EO: currently this is verified outside gpr.py but it is
# not an efficient solution.
# XXX EO: Do some memoizing since it could happen that some
# hyperparameters are kept constant by user request, so we
# don't need (somtimes) to recompute the corresponding
# gradient again. COULD THIS BE TAKEN INTO ACCOUNT BY THE
# NEW CACHED KERNEL INFRASTRUCTURE?
# self.Kinv = np.linalg.inv(self._C)
# Faster:
Kinv = SLcho_solve(self._LL, np.eye(self._L.shape[0]))
alphalphaT = np.dot(self._alpha[:, None], self._alpha[None, :])
tmp = alphalphaT - Kinv
# Pass tmp to __kernel and let it compute its gradient terms.
# This scales up to huge number of hyperparameters:
grad_LML_hypers = self.__kernel.compute_lml_gradient(
tmp, self._train_fv)
grad_K_sigma_n = 2.0 * self.params.sigma_noise * np.eye(tmp.shape[0])
# Add the term related to sigma_noise:
# grad_LML_sigma_n = 0.5 * np.trace(np.dot(tmp,grad_K_sigma_n))
# Faster formula: tr(AB) = (A*B.T).sum()
grad_LML_sigma_n = 0.5 * (tmp * (grad_K_sigma_n).T).sum()
lml_gradient = np.hstack([grad_LML_sigma_n, grad_LML_hypers])
self.log_marginal_likelihood_gradient = lml_gradient
return lml_gradient
def compute_gradient_log_marginal_likelihood_logscale(self):
"""Compute gradient of the log marginal likelihood when
hyperparameters are in logscale. This version use a more
compact formula provided by Williams and Rasmussen book.
"""
# Kinv = np.linalg.inv(self._C)
# Faster:
Kinv = SLcho_solve(self._LL, np.eye(self._L.shape[0]))
alphalphaT = np.dot(self._alpha[:, None], self._alpha[None, :])
tmp = alphalphaT - Kinv
grad_LML_log_hypers = \
self.__kernel.compute_lml_gradient_logscale(tmp, self._train_fv)
grad_K_log_sigma_n = 2.0 * self.params.sigma_noise**2 * np.eye(
Kinv.shape[0])
# Add the term related to sigma_noise:
# grad_LML_log_sigma_n = 0.5 * np.trace(np.dot(tmp, grad_K_log_sigma_n))
# Faster formula: tr(AB) = (A * B.T).sum()
grad_LML_log_sigma_n = 0.5 * (tmp * (grad_K_log_sigma_n).T).sum()
lml_gradient = np.hstack([grad_LML_log_sigma_n, grad_LML_log_hypers])
self.log_marginal_likelihood_gradient = lml_gradient
return lml_gradient
##REF: Name was automagically refactored
def get_sensitivity_analyzer(self, flavor='auto', **kwargs):
"""Returns a sensitivity analyzer for GPR.
Parameters
----------
flavor : str
What sensitivity to provide. Valid values are
'linear', 'model_select', 'auto'.
In case of 'auto' selects 'linear' for linear kernel
and 'model_select' for the rest. 'linear' corresponds to
GPRLinearWeights and 'model_select' to GRPWeights
"""
# XXX The following two lines does not work since
# self.__kernel is instance of LinearKernel and not
# just LinearKernel. How to fix?
# YYY yoh is not sure what is the problem... LinearKernel is actually
# kernel.LinearKernel so everything shoudl be ok
if flavor == 'auto':
flavor = ('model_select', 'linear')\
[int(isinstance(self.__kernel, GeneralizedLinearKernel)
or
isinstance(self.__kernel, LinearKernel))]
if __debug__:
debug("GPR", "Returning '%s' sensitivity analyzer" % flavor)
# Return proper sensitivity
if flavor == 'linear':
return GPRLinearWeights(self, **kwargs)
elif flavor == 'model_select':
# sanity check
if not ('has_sensitivity' in self.__tags__):
raise ValueError, \
"model_select flavor is not available probably " \
"due to not available 'openopt' module"
return GPRWeights(self, **kwargs)
else:
raise ValueError, "Flavor %s is not recognized" % flavor
def _train(self, data):
"""Train the classifier using `data` (`Dataset`).
"""
# local bindings for faster lookup
params = self.params
retrainable = params.retrainable
if retrainable:
newkernel = False
newL = False
_changedData = self._changedData
self._train_fv = train_fv = data.samples
# GRP relies on numerical labels
# yoh: yeah -- GPR now is purely regression so no conversion
# is necessary
train_labels = data.sa[self.get_space()].value
self._train_labels = train_labels
if not retrainable or _changedData['traindata'] \
or _changedData.get('kernel_params', False):
if __debug__:
debug("GPR", "Computing train train kernel matrix")
self.__kernel.compute(train_fv)
self._km_train_train = km_train_train = asarray(self.__kernel)
newkernel = True
if retrainable:
self._km_train_test = None # reset to facilitate recomputation
else:
if __debug__:
debug(
"GPR", "Not recomputing kernel since retrainable and "
"nothing has changed")
km_train_train = self._km_train_train # reuse
if not retrainable or newkernel or _changedData['params']:
if __debug__:
debug("GPR", "Computing L. sigma_noise=%g" \
% params.sigma_noise)
# XXX it seems that we do not need binding to object, but may be
# commented out code would return?
self._C = km_train_train + \
params.sigma_noise ** 2 * \
np.identity(km_train_train.shape[0], 'd')
# The following decomposition could raise
# np.linalg.linalg.LinAlgError because of numerical
# reasons, due to the too rapid decay of 'self._C'
# eigenvalues. In that case we try adding a small constant
# to self._C, e.g. epsilon=1.0e-20. It should be a form of
# Tikhonov regularization. This is equivalent to adding
# little white gaussian noise to data.
#
# XXX EO: how to choose epsilon?
#
# Cholesky decomposition is provided by three different
# NumPy/SciPy routines (fastest first):
# 1) self._LL = scipy.linalg.cho_factor(self._C, lower=True)
# self._L = L = np.tril(self._LL[0])
# 2) self._L = scipy.linalg.cholesky(self._C, lower=True)
# 3) self._L = numpy.linalg.cholesky(self._C)
# Even though 1 is the fastest we choose 2 since 1 does
# not return a clean lower-triangular matrix (see docstring).
# PBS: I just made it so the KernelMatrix is regularized
# all the time. I figured that if ever you were going to
# use regularization, you would want to set it yourself
# and use the same value for all folds of your data.
# YOH: Ideally so, but in real "use cases" some might have no
# clue, also our unittests (actually clfs_examples) might
# fail without any good reason. So lets return a magic with
# an option to forbid any regularization (if lm is None)
try:
# apply regularization
lm, C = params.lm, self._C
if lm is not None:
epsilon = lm * np.eye(C.shape[0])
self._L = SLcholesky(C + epsilon, lower=True)
else:
# do 10 attempts to raise each time by 10
self._L = _SLcholesky_autoreg(C, nsteps=None, lower=True)
self._LL = (self._L, True)
except SLAError:
raise SLAError("Kernel matrix is not positive, definite. "
"Try increasing the lm parameter.")
pass
newL = True
else:
if __debug__:
debug(
"GPR", "Not computing L since kernel, data and params "
"stayed the same")
# XXX we leave _alpha being recomputed, although we could check
# if newL or _changedData['targets']
#
if __debug__:
debug("GPR", "Computing alpha")
# L = self._L # reuse
# self._alpha = NLAsolve(L.transpose(),
# NLAsolve(L, train_labels))
# Faster:
self._alpha = SLcho_solve(self._LL, train_labels)
# compute only if the state is enabled
if self.ca.is_enabled('log_marginal_likelihood'):
self.compute_log_marginal_likelihood()
pass
if retrainable:
# we must assign it only if it is retrainable
self.ca.retrained = not newkernel or not newL
if __debug__:
debug("GPR", "Done training")
pass
@accepts_dataset_as_samples
def _predict(self, data):
"""
Predict the output for the provided data.
"""
retrainable = self.params.retrainable
ca = self.ca
if not retrainable or self._changedData['testdata'] \
or self._km_train_test is None:
if __debug__:
debug('GPR', "Computing train test kernel matrix")
self.__kernel.compute(self._train_fv, data)
km_train_test = asarray(self.__kernel)
if retrainable:
self._km_train_test = km_train_test
ca.repredicted = False
else:
if __debug__:
debug('GPR', "Not recomputing train test kernel matrix")
km_train_test = self._km_train_test
ca.repredicted = True
predictions = Ndot(km_train_test.transpose(), self._alpha)
if ca.is_enabled('predicted_variances'):
# do computation only if conditional attribute was enabled
if not retrainable or self._km_test_test is None \
or self._changedData['testdata']:
if __debug__:
debug('GPR', "Computing test test kernel matrix")
self.__kernel.compute(data)
km_test_test = asarray(self.__kernel)
if retrainable:
self._km_test_test = km_test_test
else:
if __debug__:
debug('GPR', "Not recomputing test test kernel matrix")
km_test_test = self._km_test_test
if __debug__:
debug("GPR", "Computing predicted variances")
L = self._L
# v = NLAsolve(L, km_train_test)
# Faster:
piv = np.arange(L.shape[0])
v = SL.lu_solve((L.T, piv), km_train_test, trans=1)
# self.predicted_variances = \
# Ndiag(km_test_test - Ndot(v.T, v)) \
# + self.sigma_noise**2
# Faster formula: np.diag(Ndot(v.T, v)) = (v**2).sum(0):
ca.predicted_variances = Ndiag(km_test_test) - (v ** 2).sum(0) \
+ self.params.sigma_noise ** 2
pass
if __debug__:
debug("GPR", "Done predicting")
ca.estimates = predictions
return predictions
##REF: Name was automagically refactored
def _set_retrainable(self, value, force=False):
"""Internal function : need to set _km_test_test
"""
super(GPR, self)._set_retrainable(value, force)
if force or (value and value != self.params.retrainable):
self._km_test_test = None
def _untrain(self):
super(GPR, self)._untrain()
# XXX might need to take special care for retrainable. later
self._init_internals()
def set_hyperparameters(self, hyperparameter):
"""
Set hyperparameters' values.
Note that 'hyperparameter' is a sequence so the order of its
values is important. First value must be sigma_noise, then
other kernel's hyperparameters values follow in the exact
order the kernel expect them to be.
"""
if hyperparameter[0] < self.params['sigma_noise'].min:
raise InvalidHyperparameterError()
self.params.sigma_noise = hyperparameter[0]
if hyperparameter.size > 1:
self.__kernel.set_hyperparameters(hyperparameter[1:])
pass
return
kernel = property(fget=lambda self: self.__kernel)
pass