def test_gnbsearchlight_matchaccuracy(self):
# was not able to deal with custom errorfx collapsing samples
# after 55e147e0bd30fbf4edede3faef3a15c6c65b33ea
ds = datasets['3dmedium'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
sl_err = sphere_gnbsearchlight(GNB(), NFoldPartitioner(cvtype=1),
radius=0)
sl_acc = sphere_gnbsearchlight(GNB(), NFoldPartitioner(cvtype=1),
radius=0, errorfx=mean_match_accuracy)
assert_array_almost_equal(sl_err(ds), 1.0 - sl_acc(ds).samples)
def test_gnbsearchlight_matchaccuracy(self):
# was not able to deal with custom errorfx collapsing samples
# after 55e147e0bd30fbf4edede3faef3a15c6c65b33ea
ds = datasets['3dmedium'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
sl_err = sphere_gnbsearchlight(GNB(),
NFoldPartitioner(cvtype=1),
radius=0)
sl_acc = sphere_gnbsearchlight(GNB(),
NFoldPartitioner(cvtype=1),
radius=0,
errorfx=mean_match_accuracy)
assert_array_almost_equal(sl_err(ds), 1.0 - sl_acc(ds).samples)
def test_searchlight_errors_per_trial():
# To make sure that searchlight can return error/accuracy per trial
from mvpa2.clfs.gnb import GNB
from mvpa2.generators.partition import OddEvenPartitioner
from mvpa2.measures.base import CrossValidation
from mvpa2.measures.searchlight import sphere_searchlight
from mvpa2.measures.gnbsearchlight import sphere_gnbsearchlight
from mvpa2.testing.datasets import datasets
from mvpa2.misc.errorfx import prediction_target_matches
dataset = datasets['3dsmall'].copy()
# randomly permute samples so we break any random correspondence
# to strengthen tests below
sample_idx = np.arange(len(dataset))
dataset = dataset[np.random.permutation(sample_idx)]
dataset.sa.targets = ['L%d' % l for l in dataset.sa.targets]
dataset.fa['voxel_indices'] = dataset.fa.myspace
sample_clf = GNB() # fast and deterministic
part = OddEvenPartitioner()
# only do partial to save time
cv = CrossValidation(sample_clf, part, errorfx=None) #prediction_target_matches)
# Just to compare error
cv_error = CrossValidation(sample_clf, part)
# Large searchlight radius so we get entire ROI, 2 centers just to make sure
# that all stacking works correctly
sl = sphere_searchlight(cv, radius=10, center_ids=[0, 1])
results = sl(dataset)
sl_gnb = sphere_gnbsearchlight(sample_clf, part, radius=10, errorfx=None,
center_ids=[0, 1])
results_gnbsl = sl_gnb(dataset)
# inspect both results
# verify that partitioning was done correctly
partitions = list(part.generate(dataset))
for res in (results, results_gnbsl):
assert('targets' in res.sa.keys()) # should carry targets
assert('cvfolds' in res.sa.keys()) # should carry cvfolds
for ipart in xrange(len(partitions)):
assert_array_equal(dataset[partitions[ipart].sa.partitions == 2].targets,
res.sa.targets[res.sa.cvfolds == ipart])
assert_datasets_equal(results, results_gnbsl)
# one "accuracy" per each trial
assert_equal(results.shape, (len(dataset), 2))
# with accuracies the same in both searchlights since the same
# features were to be selected in both cases due too large radii
errors_dataset = cv(dataset)
assert_array_equal(errors_dataset.samples[:, 0], results.samples[:, 0])
assert_array_equal(errors_dataset.samples[:, 0], results.samples[:, 1])
# and error matching (up to precision) the one if we run with default error function
assert_array_almost_equal(np.mean(results.targets[:, None] != results.samples, axis=0)[0],
np.mean(cv_error(dataset)))
def test_gnbsearchlight_3partitions_and_splitter(self):
ds = self.dataset[:, :20]
# custom partitioner which provides 3 partitions
part = CustomPartitioner([([2], [3], [1])])
gnb_sl = sphere_gnbsearchlight(GNB(), part)
res_gnb_sl = gnb_sl(ds)
# compare results to full blown searchlight
sl = sphere_searchlight(CrossValidation(GNB(), part))
res_sl = sl(ds)
assert_datasets_equal(res_gnb_sl, res_sl)
# and theoretically for this simple single cross-validation we could
# just use Splitter
splitter = Splitter('chunks', [2, 3])
# we have to put explicit None since can't become a kwarg in 1 day any
# longer here
gnb_sl_ = sphere_gnbsearchlight(GNB(), None, splitter=splitter)
res_gnb_sl_ = gnb_sl_(ds)
assert_datasets_equal(res_gnb_sl, res_gnb_sl_)
def test_gnbsearchlight_3partitions_and_splitter(self):
ds = self.dataset[:, :20]
# custom partitioner which provides 3 partitions
part = CustomPartitioner([([2], [3], [1])])
gnb_sl = sphere_gnbsearchlight(GNB(), part)
res_gnb_sl = gnb_sl(ds)
# compare results to full blown searchlight
sl = sphere_searchlight(CrossValidation(GNB(), part))
res_sl = sl(ds)
assert_datasets_equal(res_gnb_sl, res_sl)
# and theoretically for this simple single cross-validation we could
# just use Splitter
splitter = Splitter('chunks', [2, 3])
# we have to put explicit None since can't become a kwarg in 1 day any
# longer here
gnb_sl_ = sphere_gnbsearchlight(GNB(), None, splitter=splitter)
res_gnb_sl_ = gnb_sl_(ds)
assert_datasets_equal(res_gnb_sl, res_gnb_sl_)
def test_gnbsearchlight_permutations():
import mvpa2
from mvpa2.base.node import ChainNode
from mvpa2.clfs.gnb import GNB
from mvpa2.generators.base import Repeater
from mvpa2.generators.partition import NFoldPartitioner, OddEvenPartitioner
#import mvpa2.generators.permutation
#reload(mvpa2.generators.permutation)
from mvpa2.generators.permutation import AttributePermutator
from mvpa2.testing.datasets import datasets
from mvpa2.measures.base import CrossValidation
from mvpa2.measures.gnbsearchlight import sphere_gnbsearchlight
from mvpa2.measures.searchlight import sphere_searchlight
from mvpa2.mappers.fx import mean_sample
from mvpa2.misc.errorfx import mean_mismatch_error
from mvpa2.clfs.stats import MCNullDist
from mvpa2.testing.tools import assert_raises, ok_, assert_array_less
# mvpa2.debug.active = ['APERM', 'SLC'] #, 'REPM']
# mvpa2.debug.metrics += ['pid']
count = 10
nproc = 1 + int(mvpa2.externals.exists('pprocess'))
ds = datasets['3dsmall'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
slkwargs = dict(radius=3, space='voxel_indices', enable_ca=['roi_sizes'],
center_ids=[1, 10, 70, 100])
mvpa2.seed(mvpa2._random_seed)
clf = GNB()
splt = NFoldPartitioner(cvtype=2, attr='chunks')
repeater = Repeater(count=count)
permutator = AttributePermutator('targets', limit={'partitions': 1}, count=1)
null_sl = sphere_gnbsearchlight(clf, ChainNode([splt, permutator], space=splt.get_space()),
postproc=mean_sample(), errorfx=mean_mismatch_error,
**slkwargs)
distr_est = MCNullDist(repeater, tail='left', measure=null_sl,
enable_ca=['dist_samples'])
sl = sphere_gnbsearchlight(clf, splt,
reuse_neighbors=True,
null_dist=distr_est, postproc=mean_sample(),
errorfx=mean_mismatch_error,
**slkwargs)
if __debug__: # assert is done only without -O mode
assert_raises(NotImplementedError, sl, ds)
# "ad-hoc searchlights can't handle yet varying targets across partitions"
if False:
# after above limitation is removed -- enable
sl_map = sl(ds)
sl_null_prob = sl.ca.null_prob.samples.copy()
mvpa2.seed(mvpa2._random_seed)
### 'normal' Searchlight
clf = GNB()
splt = NFoldPartitioner(cvtype=2, attr='chunks')
repeater = Repeater(count=count)
permutator = AttributePermutator('targets', limit={'partitions': 1}, count=1)
# rng=np.random.RandomState(0)) # to trigger failure since the same np.random state
# would be reused across all pprocesses
null_cv = CrossValidation(clf, ChainNode([splt, permutator], space=splt.get_space()),
postproc=mean_sample())
null_sl_normal = sphere_searchlight(null_cv, nproc=nproc, **slkwargs)
distr_est_normal = MCNullDist(repeater, tail='left', measure=null_sl_normal,
enable_ca=['dist_samples'])
cv = CrossValidation(clf, splt, errorfx=mean_mismatch_error,
enable_ca=['stats'], postproc=mean_sample() )
sl = sphere_searchlight(cv, nproc=nproc, null_dist=distr_est_normal, **slkwargs)
sl_map_normal = sl(ds)
sl_null_prob_normal = sl.ca.null_prob.samples.copy()
# For every feature -- we should get some variance in estimates In
# case of failure they are all really close to each other (up to
# numerical precision), so variance will be close to 0
assert_array_less(-np.var(distr_est_normal.ca.dist_samples.samples[0],
axis=1), -1e-5)
for s in distr_est_normal.ca.dist_samples.samples[0]:
ok_(len(np.unique(s)) > 1)
def test_spatial_searchlight(self, common_variance=True, do_roi=False):
"""Tests both generic and GNBSearchlight
Test of GNBSearchlight anyways requires a ground-truth
comparison to the generic version, so we are doing sweepargs here
"""
# compute N-1 cross-validation for each sphere
# YOH: unfortunately sample_clf_lin is not guaranteed
# to provide exactly the same results due to inherent
# iterative process. Therefore lets use something quick
# and pure Python
gnb = GNB(common_variance=common_variance)
cv = CrossValidation(gnb, NFoldPartitioner())
ds = datasets['3dsmall'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
skwargs = dict(radius=1, enable_ca=['roi_sizes', 'raw_results'])
if do_roi:
# select some random set of features
nroi = rnd.randint(1, ds.nfeatures)
# and lets compute the full one as well once again so we have a reference
# which will be excluded itself from comparisons but values will be compared
# for selected roi_id
sl_all = sphere_gnbsearchlight(gnb, NFoldPartitioner(cvtype=1),
**skwargs)
result_all = sl_all(ds)
# select random features
roi_ids = rnd.permutation(range(ds.nfeatures))[:nroi]
skwargs['center_ids'] = roi_ids
else:
nroi = ds.nfeatures
sls = [sphere_searchlight(cv, **skwargs),
#GNBSearchlight(gnb, NFoldPartitioner(cvtype=1))
sphere_gnbsearchlight(gnb, NFoldPartitioner(cvtype=1),
indexsum='fancy', **skwargs)
]
if externals.exists('scipy'):
sls += [ sphere_gnbsearchlight(gnb, NFoldPartitioner(cvtype=1),
indexsum='sparse', **skwargs)]
# Just test nproc whenever common_variance is True
if externals.exists('pprocess') and common_variance:
sls += [sphere_searchlight(cv, nproc=2, **skwargs)]
all_results = []
for sl in sls:
# run searchlight
results = sl(ds)
all_results.append(results)
#print `sl`
# check for correct number of spheres
self.assertTrue(results.nfeatures == nroi)
# and measures (one per xfold)
self.assertTrue(len(results) == len(ds.UC))
# check for chance-level performance across all spheres
# makes sense only if number of features was big enough
# to get some stable estimate of mean
if not do_roi or nroi > 20:
self.assertTrue(0.4 < results.samples.mean() < 0.6)
mean_errors = results.samples.mean(axis=0)
# that we do get different errors ;)
self.assertTrue(len(np.unique(mean_errors) > 3))
# check resonable sphere sizes
self.assertTrue(len(sl.ca.roi_sizes) == nroi)
if do_roi:
# for roi we should relax conditions a bit
self.assertTrue(max(sl.ca.roi_sizes) <= 7)
self.assertTrue(min(sl.ca.roi_sizes) >= 4)
else:
self.assertTrue(max(sl.ca.roi_sizes) == 7)
self.assertTrue(min(sl.ca.roi_sizes) == 4)
# check base-class state
self.assertEqual(sl.ca.raw_results.nfeatures, nroi)
# Test if we got results correctly for 'selected' roi ids
if do_roi:
assert_array_equal(result_all[:, roi_ids], results)
if len(all_results) > 1:
# if we had multiple searchlights, we can check either they all
# gave the same result (they should have)
aresults = np.array([a.samples for a in all_results])
dresults = np.abs(aresults - aresults.mean(axis=0))
dmax = np.max(dresults)
self.assertTrue(dmax <= 1e-13)
