def _concat_results(sl=None, dataset=None, roi_ids=None, results=None):
"""The simplest implementation for collecting the results --
just put them into a list
This this implementation simply collects them into a list and
uses only sl. for assigning conditional attributes. But
custom implementation might make use of more/less of them.
Implemented as @staticmethod just to emphasize that in
principle it is independent of the actual searchlight instance
"""
# collect results
results = sum(results, [])
if __debug__ and 'SLC' in debug.active:
debug('SLC', '') # just newline
resshape = len(results) and np.asanyarray(results[0]).shape or 'N/A'
debug('SLC', ' hstacking %d results of shape %s'
% (len(results), resshape))
# but be careful: this call also serves as conversion from parallel maps
# to regular lists!
# this uses the Dataset-hstack
result_ds = hstack(results)
if __debug__:
debug('SLC', " hstacked shape %s" % (result_ds.shape,))
if sl.ca.is_enabled('roi_feature_ids'):
sl.ca.roi_feature_ids = [r.a.roi_feature_ids for r in results]
if sl.ca.is_enabled('roi_sizes'):
sl.ca.roi_sizes = [r.a.roi_sizes for r in results]
if sl.ca.is_enabled('roi_center_ids'):
sl.ca.roi_center_ids = [r.a.roi_center_ids for r in results]
if 'mapper' in dataset.a:
# since we know the space we can stick the original mapper into the
# results as well
if roi_ids is None:
result_ds.a['mapper'] = copy.copy(dataset.a.mapper)
else:
# there is an additional selection step that needs to be
# expressed by another mapper
mapper = copy.copy(dataset.a.mapper)
# NNO if the orignal mapper has no append (because it's not a
# chainmapper, for example), we make our own chainmapper.
feat_sel_mapper = StaticFeatureSelection(
roi_ids, dshape=dataset.shape[1:])
if hasattr(mapper, 'append'):
mapper.append(feat_sel_mapper)
else:
mapper = ChainMapper([dataset.a.mapper,
feat_sel_mapper])
result_ds.a['mapper'] = mapper
# store the center ids as a feature attribute
result_ds.fa['center_ids'] = roi_ids
return result_ds
def _call(self, dataset):
"""Perform the ROI search.
"""
# local binding
nproc = self.nproc
if nproc is None and externals.exists('pprocess'):
import pprocess
try:
nproc = pprocess.get_number_of_cores() or 1
except AttributeError:
warning("pprocess version %s has no API to figure out maximal "
"number of cores. Using 1"
% externals.versions['pprocess'])
nproc = 1
# train the queryengine
self._queryengine.train(dataset)
# decide whether to run on all possible center coords or just a provided
# subset
if isinstance(self.__roi_ids, str):
roi_ids = dataset.fa[self.__roi_ids].value.nonzero()[0]
elif self.__roi_ids is not None:
roi_ids = self.__roi_ids
# safeguard against stupidity
if __debug__:
if max(roi_ids) >= dataset.nfeatures:
raise IndexError, \
"Maximal center_id found is %s whenever given " \
"dataset has only %d features" \
% (max(roi_ids), dataset.nfeatures)
else:
roi_ids = np.arange(dataset.nfeatures)
# pass to subclass
results, roi_sizes = self._sl_call(dataset, roi_ids, nproc)
if not roi_sizes is None:
self.ca.roi_sizes = roi_sizes
if 'mapper' in dataset.a:
# since we know the space we can stick the original mapper into the
# results as well
if self.__roi_ids is None:
results.a['mapper'] = copy.copy(dataset.a.mapper)
else:
# there is an additional selection step that needs to be
# expressed by another mapper
mapper = copy.copy(dataset.a.mapper)
mapper.append(StaticFeatureSelection(roi_ids,
dshape=dataset.shape[1:]))
results.a['mapper'] = mapper
# charge state
self.ca.raw_results = results
# return raw results, base-class will take care of transformations
return results
def _call(self, dataset):
"""Perform the ROI search.
"""
# local binding
nproc = self.nproc
if nproc is None and externals.exists('pprocess'):
import pprocess
try:
nproc = pprocess.get_number_of_cores() or 1
except AttributeError:
warning("pprocess version %s has no API to figure out maximal "
"number of cores. Using 1" %
externals.versions['pprocess'])
nproc = 1
# train the queryengine
self._queryengine.train(dataset)
# decide whether to run on all possible center coords or just a provided
# subset
if isinstance(self.__roi_ids, str):
roi_ids = dataset.fa[self.__roi_ids].value.nonzero()[0]
elif self.__roi_ids is not None:
roi_ids = self.__roi_ids
# safeguard against stupidity
if __debug__:
if max(roi_ids) >= dataset.nfeatures:
raise IndexError, \
"Maximal center_id found is %s whenever given " \
"dataset has only %d features" \
% (max(roi_ids), dataset.nfeatures)
else:
roi_ids = np.arange(dataset.nfeatures)
# pass to subclass
results = self._sl_call(dataset, roi_ids, nproc)
if 'mapper' in dataset.a:
# since we know the space we can stick the original mapper into the
# results as well
if self.__roi_ids is None:
results.a['mapper'] = copy.copy(dataset.a.mapper)
else:
# there is an additional selection step that needs to be
# expressed by another mapper
mapper = copy.copy(dataset.a.mapper)
mapper.append(
StaticFeatureSelection(roi_ids, dshape=dataset.shape[1:]))
results.a['mapper'] = mapper
# charge state
self.ca.raw_results = results
# return raw results, base-class will take care of transformations
return results
def test_confusion_based_error(self, l_clf):
train = datasets['uni2medium']
train = train[train.sa.train == 1]
# to check if we fail to classify for 3 labels
test3 = datasets['uni3medium']
test3 = test3[test3.sa.train == 1]
err = ConfusionBasedError(clf=l_clf)
terr = TransferMeasure(l_clf, Splitter('train', attr_values=[1,1]),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'))
self.assertRaises(UnknownStateError, err, None)
"""Shouldn't be able to access the state yet"""
l_clf.train(train)
e, te = err(None), terr(train)
te = np.asscalar(te)
self.assertTrue(abs(e-te) < 1e-10,
msg="ConfusionBasedError (%.2g) should be equal to TransferError "
"(%.2g) on traindataset" % (e, te))
# this will print nasty WARNING but it is ok -- it is just checking code
# NB warnings are not printed while doing whole testing
warning("Don't worry about the following warning.")
if 'multiclass' in l_clf.__tags__:
self.assertFalse(terr(test3) is None)
# try copying the beast
terr_copy = copy(terr)
def _call(self, dataset):
analyzers = []
# create analyzers
for clf in self.clf.clfs:
if self.__analyzer is None:
analyzer = clf.get_sensitivity_analyzer(**(self._slave_kwargs))
if analyzer is None:
raise ValueError, \
"Wasn't able to figure basic analyzer for clf %r" % \
(clf,)
if __debug__:
debug("SA", "Selected analyzer %r for clf %r" % \
(analyzer, clf))
else:
# XXX shallow copy should be enough...
analyzer = copy.copy(self.__analyzer)
# assign corresponding classifier
analyzer.clf = clf
# if clf was trained already - don't train again
if clf.trained:
analyzer._force_train = False
analyzers.append(analyzer)
self.__combined_analyzer.analyzers = analyzers
# XXX not sure if we don't want to call directly ._call(dataset) to avoid
# double application of transformers/combiners, after all we are just
# 'proxying' here to combined_analyzer...
# YOH: decided -- lets call ._call
return self.__combined_analyzer._call(dataset)
def _sl_call(self, dataset, roi_ids, nproc):
"""Classical generic searchlight implementation
"""
assert (self.results_backend in ('native', 'hdf5'))
# compute
if nproc is not None and nproc > 1:
# split all target ROIs centers into `nproc` equally sized blocks
nproc_needed = min(len(roi_ids), nproc)
nblocks = nproc_needed \
if self.nblocks is None else self.nblocks
roi_blocks = np.array_split(roi_ids, nblocks)
# the next block sets up the infrastructure for parallel computing
# this can easily be changed into a ParallelPython loop, if we
# decide to have a PP job server in PyMVPA
import pprocess
p_results = pprocess.Map(limit=nproc_needed)
if __debug__:
debug(
'SLC', "Starting off %s child processes for nblocks=%i" %
(nproc_needed, nblocks))
compute = p_results.manage(pprocess.MakeParallel(self._proc_block))
for iblock, block in enumerate(roi_blocks):
# should we maybe deepcopy the measure to have a unique and
# independent one per process?
seed = mvpa2.get_random_seed()
compute(block,
dataset,
copy.copy(self.__datameasure),
seed=seed,
iblock=iblock)
else:
# otherwise collect the results in an 1-item list
p_results = [
self._proc_block(roi_ids, dataset, self.__datameasure)
]
# Finally collect and possibly process results
# p_results here is either a generator from pprocess.Map or a list.
# In case of a generator it allows to process results as they become
# available
result_ds = self.results_fx(
sl=self,
dataset=dataset,
roi_ids=roi_ids,
results=self.__handle_all_results(p_results))
# Assure having a dataset (for paranoid ones)
if not is_datasetlike(result_ds):
try:
result_a = np.atleast_1d(result_ds)
except ValueError, e:
if 'setting an array element with a sequence' in str(e):
# try forcing object array. Happens with
# test_custom_results_fx_logic on numpy 1.4.1 on Debian
# squeeze
result_a = np.array(result_ds, dtype=object)
else:
raise
result_ds = Dataset(result_a)
def test_ds_shallowcopy():
# lets use some instance of somewhat evolved dataset
ds = normal_feature_dataset()
ds.samples = ds.samples.view(myarray)
# SHALLOW copy the beast
ds_ = copy.copy(ds)
# verify that we have the same data
assert_array_equal(ds.samples, ds_.samples)
assert_array_equal(ds.targets, ds_.targets)
assert_array_equal(ds.chunks, ds_.chunks)
# array subclass survives
ok_(isinstance(ds_.samples, myarray))
# modify and see that we actually DO change the data in both
ds_.samples[0, 0] = 1234
assert_array_equal(ds.samples, ds_.samples)
assert_array_equal(ds.targets, ds_.targets)
assert_array_equal(ds.chunks, ds_.chunks)
ds_.sa.targets[0] = 'ab'
ds_.sa.chunks[0] = 234
assert_array_equal(ds.samples, ds_.samples)
assert_array_equal(ds.targets, ds_.targets)
assert_array_equal(ds.chunks, ds_.chunks)
ok_(ds.sa.targets[0] == 'ab')
ok_(ds.sa.chunks[0] == 234)
def _sl_call(self, dataset, roi_ids, nproc):
"""Classical generic searchlight implementation
"""
assert(self.results_backend in ('native', 'hdf5'))
# compute
if nproc is not None and nproc > 1:
# split all target ROIs centers into `nproc` equally sized blocks
nproc_needed = min(len(roi_ids), nproc)
roi_blocks = np.array_split(roi_ids, nproc_needed)
# the next block sets up the infrastructure for parallel computing
# this can easily be changed into a ParallelPython loop, if we
# decide to have a PP job server in PyMVPA
import pprocess
p_results = pprocess.Map(limit=nproc_needed)
if __debug__:
debug('SLC', "Starting off child processes for nproc=%i"
% nproc_needed)
compute = p_results.manage(
pprocess.MakeParallel(self._proc_block))
for iblock, block in enumerate(roi_blocks):
# should we maybe deepcopy the measure to have a unique and
# independent one per process?
compute(block, dataset, copy.copy(self.__datameasure),
iblock=iblock)
# collect results
results = []
if self.ca.is_enabled('roi_sizes'):
roi_sizes = []
else:
roi_sizes = None
for r, rsizes in p_results:
results += self.__handle_results(r)
if not roi_sizes is None:
roi_sizes += rsizes
else:
# otherwise collect the results in a list
results, roi_sizes = \
self._proc_block(roi_ids, dataset, self.__datameasure)
results = self.__handle_results(results)
if __debug__ and 'SLC' in debug.active:
debug('SLC', '') # just newline
resshape = len(results) and np.asanyarray(results[0]).shape or 'N/A'
debug('SLC', ' hstacking %d results of shape %s'
% (len(results), resshape))
# but be careful: this call also serves as conversion from parallel maps
# to regular lists!
# this uses the Dataset-hstack
result_ds = hstack(results)
if self.ca.is_enabled('roi_feature_ids'):
self.ca.roi_feature_ids = [r.a.roi_feature_ids for r in results]
if __debug__:
debug('SLC', " hstacked shape %s" % (result_ds.shape,))
return result_ds, roi_sizes
def __getitem__(self, key):
# if just one is requested return just one, otherwise return a
# NodeChain again
if isinstance(key, int):
return self._nodes[key]
else:
# operate on shallow copy of self
sliced = copy.copy(self)
sliced._nodes = self._nodes[key]
return sliced
def _sl_call(self, dataset, roi_ids, nproc):
"""Classical generic searchlight implementation
"""
assert(self.results_backend in ('native', 'hdf5'))
# compute
if nproc is not None and nproc > 1:
# split all target ROIs centers into `nproc` equally sized blocks
nproc_needed = min(len(roi_ids), nproc)
nblocks = nproc_needed \
if self.nblocks is None else self.nblocks
roi_blocks = np.array_split(roi_ids, nblocks)
# the next block sets up the infrastructure for parallel computing
# this can easily be changed into a ParallelPython loop, if we
# decide to have a PP job server in PyMVPA
import pprocess
p_results = pprocess.Map(limit=nproc_needed)
if __debug__:
debug('SLC', "Starting off %s child processes for nblocks=%i"
% (nproc_needed, nblocks))
compute = p_results.manage(
pprocess.MakeParallel(self._proc_block))
for iblock, block in enumerate(roi_blocks):
# should we maybe deepcopy the measure to have a unique and
# independent one per process?
seed = mvpa2.get_random_seed()
compute(block, dataset, copy.copy(self.__datameasure),
seed=seed, iblock=iblock)
else:
# otherwise collect the results in an 1-item list
p_results = [
self._proc_block(roi_ids, dataset, self.__datameasure)]
# Finally collect and possibly process results
# p_results here is either a generator from pprocess.Map or a list.
# In case of a generator it allows to process results as they become
# available
result_ds = self.results_fx(sl=self,
dataset=dataset,
roi_ids=roi_ids,
results=self.__handle_all_results(p_results))
# Assure having a dataset (for paranoid ones)
if not is_datasetlike(result_ds):
try:
result_a = np.atleast_1d(result_ds)
except ValueError, e:
if 'setting an array element with a sequence' in str(e):
# try forcing object array. Happens with
# test_custom_results_fx_logic on numpy 1.4.1 on Debian
# squeeze
result_a = np.array(result_ds, dtype=object)
else:
raise
result_ds = Dataset(result_a)
def _precall(self, ds):
# estimate the NULL distribution when functor is given
if not self.__null_dist is None:
if __debug__:
debug("STAT", "Estimating NULL distribution using %s" % self.__null_dist)
# we need a matching measure instance, but we have to disable
# the estimation of the null distribution in that child to prevent
# infinite looping.
measure = copy.copy(self)
measure.__null_dist = None
self.__null_dist.fit(measure, ds)
def _precall(self, ds):
# estimate the NULL distribution when functor is given
if not self.__null_dist is None:
if __debug__:
debug("STAT", "Estimating NULL distribution using %s"
% self.__null_dist)
# we need a matching measure instance, but we have to disable
# the estimation of the null distribution in that child to prevent
# infinite looping.
measure = copy.copy(self)
measure.__null_dist = None
self.__null_dist.fit(measure, ds)
def as_descrete_time(self, dt, storeoffset=False, offsetattr='offset'):
"""Convert `onset` and `duration` information into descrete timepoints.
Parameters
----------
dt : float
Temporal distance between two timepoints in the same unit as `onset`
and `duration`.
storeoffset : bool
If True, the temporal offset between original `onset` and
descretized onset is stored as an additional item.
offsetattr : str
The name of the attribute that is used to store the computed offset
in case the `storeoffset` is enabled.
Returns
-------
A copy of the original `Event` with `onset` and optionally `duration`
replaced by their corresponding descrete timepoint. The new onset will
correspond to the timepoint just before or exactly at the original
onset. The new duration will be the number of timepoints covering the
event from the computed onset timepoint till the timepoint exactly at
the end, or just after the event.
Note again, that the new values are expressed as #timepoint and not
in their original unit!
"""
dt = float(dt)
onset = self['onset']
out = copy(self)
# get the timepoint just prior the onset
out['onset'] = int(np.floor(onset / dt))
if storeoffset:
# compute offset
offset = onset - (out['onset'] * dt)
out[offsetattr] = offset
if out.has_key('duration'):
# how many timepoint cover the event (from computed onset
# to the one timepoint just after the end of the event
out['duration'] = int(np.ceil((onset + out['duration']) / dt) \
- out['onset'])
return out
def _train(self, ds):
"""Proceed and select the features recursively eliminating less
important ones.
Parameters
----------
ds : Dataset
used to compute sensitivity maps and train a classifier
to determine the transfer error
"""
# get the initial split into train and test
dataset, testdataset = self._get_traintest_ds(ds)
if __debug__:
debug('RFEC',
"Initiating RFE with training on %s and testing using %s",
(dataset, testdataset))
errors = []
"""Computed error for each tested features set."""
ca = self.ca
ca.nfeatures = []
"""Number of features at each step. Since it is not used by the
algorithm it is stored directly in the conditional attribute"""
ca.history = np.arange(dataset.nfeatures)
"""Store the last step # when the feature was still present
"""
ca.sensitivities = []
stop = False
"""Flag when RFE should be stopped."""
results = None
"""Will hold the best feature set ever."""
wdataset = dataset
"""Operate on working dataset initially identical."""
wtestdataset = testdataset
"""Same feature selection has to be performs on test dataset as well.
This will hold the current testdataset."""
step = 0
"""Counter how many selection step where done."""
orig_feature_ids = np.arange(dataset.nfeatures)
"""List of feature Ids as per original dataset remaining at any given
step"""
sensitivity = None
"""Contains the latest sensitivity map."""
result_selected_ids = orig_feature_ids
"""Resultant ids of selected features. Since the best is not
necessarily is the last - we better keep this one around. By
default -- all features are there"""
selected_ids = result_selected_ids
isthebest = True
"""By default (e.g. no errors even estimated) every step is the best one
"""
while wdataset.nfeatures > 0:
if __debug__:
debug('RFEC',
"Step %d: nfeatures=%d" % (step, wdataset.nfeatures))
# mark the features which are present at this step
# if it brings anyb mentionable computational burden in the future,
# only mark on removed features at each step
ca.history[orig_feature_ids] = step
# Compute sensitivity map
if self.__update_sensitivity or sensitivity == None:
sensitivity = self._fmeasure(wdataset)
if len(sensitivity) > 1:
raise ValueError(
"RFE cannot handle multiple sensitivities at once. "
"'%s' returned %i sensitivities."
% (self._fmeasure.__class__.__name__,
len(sensitivity)))
if ca.is_enabled("sensitivities"):
ca.sensitivities.append(sensitivity)
if self._pmeasure:
# get error for current feature set (handles optional retraining)
error = np.asscalar(self._evaluate_pmeasure(wdataset, wtestdataset))
# Record the error
errors.append(error)
# Check if it is time to stop and if we got
# the best result
if self._stopping_criterion is not None:
stop = self._stopping_criterion(errors)
if self._bestdetector is not None:
isthebest = self._bestdetector(errors)
else:
error = None
nfeatures = wdataset.nfeatures
if ca.is_enabled("nfeatures"):
ca.nfeatures.append(wdataset.nfeatures)
# store result
if isthebest:
result_selected_ids = orig_feature_ids
if __debug__:
debug('RFEC',
"Step %d: nfeatures=%d error=%s best/stop=%d/%d " %
(step, nfeatures, error, isthebest, stop))
# stop if it is time to finish
if nfeatures == 1 or nfeatures <= self.nfeatures_min or stop:
break
# Select features to preserve
selected_ids = self._fselector(sensitivity)
if __debug__:
debug('RFEC_',
"Sensitivity: %s, nfeatures_selected=%d, selected_ids: %s" %
(sensitivity, len(selected_ids), selected_ids))
# Create a dataset only with selected features
wdataset = wdataset[:, selected_ids]
# select corresponding sensitivity values if they are not
# recomputed
if not self.__update_sensitivity:
if len(sensitivity.shape) >= 2:
assert(sensitivity.shape[0] == 1) # there must be only 1 sample
sensitivity = sensitivity[:, selected_ids]
else:
sensitivity = sensitivity[selected_ids]
# need to update the test dataset as well
# XXX why should it ever become None?
# yoh: because we can have __transfer_error computed
# using wdataset. See xia-generalization estimate
# in lightsvm. Or for god's sake leave-one-out
# on a wdataset
# TODO: document these cases in this class
if testdataset is not None:
wtestdataset = wtestdataset[:, selected_ids]
step += 1
# WARNING: THIS MUST BE THE LAST THING TO DO ON selected_ids
selected_ids.sort()
if self.ca.is_enabled("history") \
or self.ca.is_enabled('selected_ids'):
orig_feature_ids = orig_feature_ids[selected_ids]
# we already have the initial sensitivities, so even for a shared
# classifier we can cleanup here
if self._pmeasure:
self._pmeasure.untrain()
# charge conditional attributes
self.ca.errors = errors
self.ca.selected_ids = result_selected_ids
if __debug__:
debug('RFEC',
"Selected %d features: %s",
(len(result_selected_ids), result_selected_ids))
# announce desired features to the underlying slice mapper
# do copy to survive later selections
self._safe_assign_slicearg(copy(result_selected_ids))
# call super to set _Xshape etc
super(RFE, self)._train(dataset)
def _train(self, ds):
# local binding
fmeasure = self._fmeasure
fselector = self._fselector
scriterion = self._stopping_criterion
bestdetector = self._bestdetector
# init
# Computed error for each tested features set.
errors = []
# feature candidate are all features in the pattern object
candidates = range(ds.nfeatures)
# initially empty list of selected features
selected = []
# results in here please
results = None
# as long as there are candidates left
# the loop will most likely get broken earlier if the stopping
# criterion is reached
while len(candidates):
# measures for all candidates
measures = []
# for all possible candidates
for i, candidate in enumerate(candidates):
if __debug__:
debug('IFSC', "Tested %i" % i, cr=True)
# take the new candidate and all already selected features
# select a new temporay feature subset from the dataset
# slice the full dataset, because for the initial iteration
# steps this will be much mure effecient than splitting the
# full ds into train and test at first
fslm = StaticFeatureSelection(selected + [candidate])
fslm.train(ds)
candidate_ds = fslm(ds)
# activate the dataset splitter
dsgen = self._splitter.generate(candidate_ds)
# and derived the dataset part that is used for computing the selection
# criterion
trainds = dsgen.next()
# compute data measure on the training part of this feature set
measures.append(fmeasure(trainds))
# relies on ds.item() to work properly
measures = [np.asscalar(m) for m in measures]
# Select promissing feature candidates (staging)
# IDs are only applicable to the current set of feature candidates
tmp_staging_ids = fselector(measures)
# translate into real candidate ids
staging_ids = [candidates[i] for i in tmp_staging_ids]
# mark them as selected and remove from candidates
selected += staging_ids
for i in staging_ids:
candidates.remove(i)
# actually run the performance measure to estimate "quality" of
# selection
fslm = StaticFeatureSelection(selected)
fslm.train(ds)
selectedds = fslm(ds)
# split into train and test part
trainds, testds = self._get_traintest_ds(selectedds)
# evaluate and store
error = self._evaluate_pmeasure(trainds, testds)
errors.append(np.asscalar(error))
# intermediate cleanup, so the datasets do not hand around while
# the next candidate evaluation is computed
del trainds
del testds
# Check if it is time to stop and if we got
# the best result
stop = scriterion(errors)
isthebest = bestdetector(errors)
if __debug__:
debug('IFSC',
"nselected %i; error: %.4f " \
"best/stop=%d/%d\n" \
% (len(selected), errors[-1], isthebest, stop),
cr=True, lf=True)
if isthebest:
# announce desired features to the underlying slice mapper
# do copy to survive later selections
self._safe_assign_slicearg(copy(selected))
# leave the loop when the criterion is reached
if stop:
break
# charge state
self.ca.errors = errors
def __call__(self, datasets):
"""Estimate mappers for each dataset using searchlight-based
hyperalignment.
Parameters
----------
datasets : list or tuple of datasets
Returns
-------
A list of trained StaticProjectionMappers of the same length as datasets
"""
# Perform some checks first before modifying internal state
params = self.params
ndatasets = len(datasets)
if len(datasets) <= 1:
raise ValueError("SearchlightHyperalignment needs > 1 dataset to "
"operate on. Got: %d" % self.ndatasets)
if params.ref_ds in params.exclude_from_model:
raise ValueError("Requested reference dataset %i is also "
"in the exclude list." % params.ref_ds)
if params.ref_ds >= ndatasets:
raise ValueError("Requested reference dataset %i is out of "
"bounds. We have only %i datasets provided"
% (params.ref_ds, self.ndatasets))
# The rest of the checks are just warnings
self.ndatasets = ndatasets
_shpaldebug("SearchlightHyperalignment %s for %i datasets"
% (self, self.ndatasets))
if params.ref_ds != params.hyperalignment.params.ref_ds:
warning('Supplied ref_ds & hyperalignment instance ref_ds:%d differ.'
% params.hyperalignment.params.ref_ds)
warning('Using default hyperalignment instance with ref_ds: %d' % params.ref_ds)
params.hyperalignment = Hyperalignment(ref_ds=params.ref_ds)
if len(params.exclude_from_model) > 0:
warning("These datasets will not participate in building common "
"model: %s" % params.exclude_from_model)
if __debug__:
# verify that datasets were zscored prior the alignment since it is
# assumed/required preprocessing step
for ids, ds in enumerate(datasets):
for f, fname, tval in ((np.mean, 'means', 0),
(np.std, 'stds', 1)):
vals = f(ds, axis=0)
vals_comp = np.abs(vals - tval) > 1e-5
if np.any(vals_comp):
warning('%d %s are too different (max diff=%g) from %d in '
'dataset %d to come from a zscored dataset. '
'Please zscore datasets first for correct operation '
'(unless if was intentional)'
% (np.sum(vals_comp), fname,
np.max(np.abs(vals)), tval, ids))
# Setting up SearchlightHyperalignment
# we need to know which original features where comprising the
# individual SL ROIs
_shpaldebug('Initializing FeatureSelectionHyperalignment.')
hmeasure = FeatureSelectionHyperalignment(
featsel=params.featsel,
hyperalignment=params.hyperalignment,
full_matrix=params.combine_neighbormappers,
use_same_features=params.use_same_features,
exclude_from_model=params.exclude_from_model,
dtype=params.dtype)
# Performing SL processing manually
_shpaldebug("Setting up for searchlights")
if params.nproc is None and externals.exists('pprocess'):
import pprocess
try:
params.nproc = pprocess.get_number_of_cores() or 1
except AttributeError:
warning("pprocess version %s has no API to figure out maximal "
"number of cores. Using 1"
% externals.versions['pprocess'])
params.nproc = 1
# XXX I think this class should already accept a single dataset only.
# It should have a ``space`` setting that names a sample attribute that
# can be used to identify individual/original datasets.
# Taking a single dataset as argument would be cleaner, because the
# algorithm relies on the assumption that there is a coarse feature
# alignment, i.e. the SL ROIs cover roughly the same area
queryengines = self._get_trained_queryengines(
datasets, params.queryengine, params.radius, params.ref_ds)
# For surface nodes to voxels queryengines, roi_seed hardly makes sense
if isinstance(queryengines[params.ref_ds], SurfaceVerticesQueryEngine):
self.force_roi_seed = False
if not self.params.combine_neighbormappers:
raise NotImplementedError("Mapping from voxels to surface nodes is not "
"implmented yet. Try setting combine_neighbormappers to True.")
self.nfeatures = datasets[params.ref_ds].nfeatures
_shpaldebug("Performing Hyperalignment in searchlights")
# Setting up centers for running SL Hyperalignment
if params.sparse_radius is None:
roi_ids = self._get_verified_ids(queryengines) \
if params.mask_node_ids is None \
else params.mask_node_ids
else:
if params.queryengine is not None:
raise NotImplementedError(
"using sparse_radius whenever custom queryengine is "
"provided is not yet supported.")
_shpaldebug("Setting up sparse neighborhood")
from mvpa2.misc.neighborhood import scatter_neighborhoods
if params.mask_node_ids is None:
scoords, sidx = scatter_neighborhoods(
Sphere(params.sparse_radius),
datasets[params.ref_ds].fa.voxel_indices,
deterministic=True)
roi_ids = sidx
else:
scoords, sidx = scatter_neighborhoods(
Sphere(params.sparse_radius),
datasets[params.ref_ds].fa.voxel_indices[params.mask_node_ids],
deterministic=True)
roi_ids = [params.mask_node_ids[sid] for sid in sidx]
# Initialize projections
_shpaldebug('Initializing projection matrices')
self.projections = [
csc_matrix((self.nfeatures, self.nfeatures), dtype=params.dtype)
for isub in range(self.ndatasets)]
# compute
if params.nproc is not None and params.nproc > 1:
# split all target ROIs centers into `nproc` equally sized blocks
nproc_needed = min(len(roi_ids), params.nproc)
params.nblocks = nproc_needed \
if params.nblocks is None else params.nblocks
params.nblocks = min(len(roi_ids), params.nblocks)
node_blocks = np.array_split(roi_ids, params.nblocks)
# the next block sets up the infrastructure for parallel computing
# this can easily be changed into a ParallelPython loop, if we
# decide to have a PP job server in PyMVPA
import pprocess
p_results = pprocess.Map(limit=nproc_needed)
if __debug__:
debug('SLC', "Starting off %s child processes for nblocks=%i"
% (nproc_needed, params.nblocks))
compute = p_results.manage(
pprocess.MakeParallel(self._proc_block))
seed = mvpa2.get_random_seed()
for iblock, block in enumerate(node_blocks):
# should we maybe deepcopy the measure to have a unique and
# independent one per process?
compute(block, datasets, copy.copy(hmeasure), queryengines,
seed=seed, iblock=iblock)
else:
# otherwise collect the results in an 1-item list
_shpaldebug('Using 1 process to compute mappers.')
if params.nblocks is None:
params.nblocks = 1
params.nblocks = min(len(roi_ids), params.nblocks)
node_blocks = np.array_split(roi_ids, params.nblocks)
p_results = [self._proc_block(block, datasets, hmeasure, queryengines)
for block in node_blocks]
results_ds = self.__handle_all_results(p_results)
# Dummy iterator for, you know, iteration
list(results_ds)
_shpaldebug('Wrapping projection matrices into StaticProjectionMappers')
self.projections = [
StaticProjectionMapper(proj=proj, recon=proj.T) if params.compute_recon
else StaticProjectionMapper(proj=proj)
for proj in self.projections]
return self.projections
def _train(self, ds):
# local binding
fmeasure = self._fmeasure
fselector = self._fselector
scriterion = self._stopping_criterion
bestdetector = self._bestdetector
# init
# Computed error for each tested features set.
errors = []
# feature candidate are all features in the pattern object
candidates = list(range(ds.nfeatures))
# initially empty list of selected features
selected = []
# results in here please
results = None
# as long as there are candidates left
# the loop will most likely get broken earlier if the stopping
# criterion is reached
while len(candidates):
# measures for all candidates
measures = []
# for all possible candidates
for i, candidate in enumerate(candidates):
if __debug__:
debug('IFSC', "Tested %i" % i, cr=True)
# take the new candidate and all already selected features
# select a new temporay feature subset from the dataset
# slice the full dataset, because for the initial iteration
# steps this will be much mure effecient than splitting the
# full ds into train and test at first
fslm = StaticFeatureSelection(selected + [candidate])
fslm.train(ds)
candidate_ds = fslm(ds)
# activate the dataset splitter
dsgen = self._splitter.generate(candidate_ds)
# and derived the dataset part that is used for computing the selection
# criterion
trainds = next(dsgen)
# compute data measure on the training part of this feature set
measures.append(fmeasure(trainds))
# relies on ds.item() to work properly
measures = [np.asscalar(m) for m in measures]
# Select promissing feature candidates (staging)
# IDs are only applicable to the current set of feature candidates
tmp_staging_ids = fselector(measures)
# translate into real candidate ids
staging_ids = [candidates[i] for i in tmp_staging_ids]
# mark them as selected and remove from candidates
selected += staging_ids
for i in staging_ids:
candidates.remove(i)
# actually run the performance measure to estimate "quality" of
# selection
fslm = StaticFeatureSelection(selected)
fslm.train(ds)
selectedds = fslm(ds)
# split into train and test part
trainds, testds = self._get_traintest_ds(selectedds)
# evaluate and store
error = self._evaluate_pmeasure(trainds, testds)
errors.append(np.asscalar(error))
# intermediate cleanup, so the datasets do not hand around while
# the next candidate evaluation is computed
del trainds
del testds
# Check if it is time to stop and if we got
# the best result
stop = scriterion(errors)
isthebest = bestdetector(errors)
if __debug__:
debug('IFSC',
"nselected %i; error: %.4f " \
"best/stop=%d/%d\n" \
% (len(selected), errors[-1], isthebest, stop),
cr=True, lf=True)
if isthebest:
# announce desired features to the underlying slice mapper
# do copy to survive later selections
self._safe_assign_slicearg(copy(selected))
# leave the loop when the criterion is reached
if stop:
break
# charge state
self.ca.errors = errors
def test_multivariate(self):
mv_perf = []
mv_lin_perf = []
uv_perf = []
l_clf = clfswh['linear', 'svm'][0]
nl_clf = clfswh['non-linear', 'svm'][0]
#orig_keys = nl_clf.param._params.keys()
#nl_param_orig = nl_clf.param._params.copy()
# l_clf = LinearNuSVMC()
# XXX ??? not sure what below meant and it is obsolete if
# using SG... commenting out for now
# for some reason order is not preserved thus dictionaries are not
# the same any longer -- lets compare values
#self.assertEqual([nl_clf.param._params[k] for k in orig_keys],
# [nl_param_orig[k] for k in orig_keys],
# msg="New instance mustn't override values in previously created")
## and keys separately
#self.assertEqual(set(nl_clf.param._params.keys()),
# set(orig_keys),
# msg="New instance doesn't change set of parameters in original")
# We must be able to deepcopy not yet trained SVMs now
import mvpa2.support.copy as copy
try:
nl_clf.untrain()
nl_clf_copy_ = copy.copy(nl_clf)
nl_clf_copy = copy.deepcopy(nl_clf)
except:
self.fail(msg="Failed to deepcopy not-yet trained SVM %s" % nl_clf)
for i in range(20):
train = pure_multivariate_signal(20, 3)
test = pure_multivariate_signal(20, 3)
# use non-linear CLF on 2d data
nl_clf.train(train)
p_mv = nl_clf.predict(test.samples)
mv_perf.append(np.mean(p_mv == test.targets))
# use linear CLF on 2d data
l_clf.train(train)
p_lin_mv = l_clf.predict(test.samples)
mv_lin_perf.append(np.mean(p_lin_mv == test.targets))
# use non-linear CLF on 1d data
nl_clf.train(train[:, 0])
p_uv = nl_clf.predict(test[:, 0].samples)
uv_perf.append(np.mean(p_uv == test.targets))
mean_mv_perf = np.mean(mv_perf)
mean_mv_lin_perf = np.mean(mv_lin_perf)
mean_uv_perf = np.mean(uv_perf)
# non-linear CLF has to be close to perfect
self.assertTrue(mean_mv_perf > 0.9)
# linear CLF cannot learn this problem!
self.assertTrue(mean_mv_perf > mean_mv_lin_perf)
# univariate has insufficient information
self.assertTrue(mean_uv_perf < mean_mv_perf)
def _call(self, dataset):
"""Perform the ROI search.
"""
# local binding
nproc = self.nproc
if nproc is None and externals.exists('pprocess'):
import pprocess
try:
nproc = pprocess.get_number_of_cores() or 1
except AttributeError:
warning("pprocess version %s has no API to figure out maximal "
"number of cores. Using 1"
% externals.versions['pprocess'])
nproc = 1
# train the queryengine
self._queryengine.train(dataset)
# decide whether to run on all possible center coords or just a provided
# subset
if isinstance(self.__roi_ids, str):
roi_ids = dataset.fa[self.__roi_ids].value.nonzero()[0]
elif self.__roi_ids is not None:
roi_ids = self.__roi_ids
# safeguard against stupidity
if __debug__:
qe_ids = self._queryengine.ids # known to qe
if not set(qe_ids).issuperset(roi_ids):
raise IndexError(
"Some roi_ids are not known to the query engine %s: %s"
% (self._queryengine,
set(roi_ids).difference(qe_ids)))
else:
roi_ids = self._queryengine.ids
# pass to subclass
results = self._sl_call(dataset, roi_ids, nproc)
if 'mapper' in dataset.a:
# since we know the space we can stick the original mapper into the
# results as well
if self.__roi_ids is None:
results.a['mapper'] = copy.copy(dataset.a.mapper)
else:
# there is an additional selection step that needs to be
# expressed by another mapper
mapper = copy.copy(dataset.a.mapper)
# NNO if the orignal mapper has no append (because it's not a
# chainmapper, for example), we make our own chainmapper.
#
# THe original code was:
# mapper.append(StaticFeatureSelection(roi_ids,
# dshape=dataset.shape[1:]))
feat_sel_mapper = StaticFeatureSelection(roi_ids,
dshape=dataset.shape[1:])
if 'append' in dir(mapper):
mapper.append(feat_sel_mapper)
else:
mapper = ChainMapper([dataset.a.mapper,
feat_sel_mapper])
results.a['mapper'] = mapper
# charge state
self.ca.raw_results = results
# store the center ids as a feature attribute
results.fa['center_ids'] = roi_ids
# return raw results, base-class will take care of transformations
return results
def __copy__(self):
# XXX how do we safely and exhaustively copy a node?
return self.__class__([copy.copy(n) for n in self])
def test_subset():
data = np.array(
[[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
[16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
[32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
[48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63]])
# float array doesn't work
sm = StaticFeatureSelection(np.ones(16))
assert_raises(IndexError, sm.forward, data)
# full mask
sm = StaticFeatureSelection(slice(None))
# should not change single samples
assert_array_equal(sm.forward(data[0:1].copy()), data[0:1])
# or multi-samples
assert_array_equal(sm.forward(data.copy()), data)
sm.train(data)
# same on reverse
assert_array_equal(sm.reverse(data[0:1].copy()), data[0:1])
# or multi-samples
assert_array_equal(sm.reverse(data.copy()), data)
# identical mappers
sm_none = StaticFeatureSelection(slice(None))
sm_int = StaticFeatureSelection(np.arange(16))
sm_bool = StaticFeatureSelection(np.ones(16, dtype='bool'))
sms = [sm_none, sm_int, sm_bool]
# test subsets
sids = [3, 4, 5, 6]
bsubset = np.zeros(16, dtype='bool')
bsubset[sids] = True
subsets = [sids, slice(3, 7), bsubset, [3, 3, 4, 4, 6, 6, 6, 5]]
# all test subset result in equivalent masks, hence should do the same to
# the mapper and result in identical behavior
for st in sms:
for i, sub in enumerate(subsets):
# shallow copy
orig = copy(st)
subsm = StaticFeatureSelection(sub)
# should do copy-on-write for all important stuff!!
orig += subsm
# test if selection did its job
if i == 3:
# special case of multiplying features
assert_array_equal(orig.forward1(data[0].copy()), subsets[i])
else:
assert_array_equal(orig.forward1(data[0].copy()), sids)
## all of the above shouldn't change the original mapper
#assert_array_equal(sm.get_mask(), np.arange(16))
# check for some bug catcher
# no 3D input
#assert_raises(IndexError, sm.forward, np.ones((3,2,1)))
# no input of wrong length
if __debug__:
# checked only in __debug__
assert_raises(ValueError, sm.forward, np.ones(4))
# same on reverse
#assert_raises(ValueError, sm.reverse, np.ones(16))
# invalid ids
#assert_false(subsm.is_valid_inid(-1))
#assert_false(subsm.is_valid_inid(16))
# intended merge failures
fsm = StaticFeatureSelection(np.arange(16))
assert_equal(fsm.__iadd__(None), NotImplemented)
assert_equal(fsm.__iadd__(Dataset([2, 3, 4])), NotImplemented)
def __copy__(self):
return self.__class__([copy.copy(n) for n in self],
copy.copy(self._combine_axis),
copy.copy(self._a))
