def test_spatial_searchlight(self, common_variance):
"""Tests both generic and GNBSearchlight
Test of GNBSearchlight anyways requires a ground-truth
comparison to the generic version, so we are doing sweepargs here
"""
# XXX after running at 1 million times we now know that gnb_searchlight
# is totally broken
raise SkipTest
# compute N-1 cross-validation for each sphere
# YOH: unfortunately sample_clf_lin is not guaranteed
# to provide exactly the same results due to inherent
# iterative process. Therefore lets use something quick
# and pure Python
gnb = GNB(common_variance=common_variance)
cv = CrossValidation(gnb, NFoldPartitioner())
skwargs = dict(radius=1, enable_ca=['roi_sizes', 'raw_results'])
sls = [sphere_searchlight(cv, **skwargs),
#GNBSearchlight(gnb, NFoldSplitter(cvtype=1))
sphere_gnbsearchlight(gnb, NFoldSplitter(cvtype=1),
indexsum='fancy', **skwargs)
]
if externals.exists('scipy'):
sls += [ sphere_gnbsearchlight(gnb, NFoldSplitter(cvtype=1),
indexsum='sparse', **skwargs)]
# Just test nproc whenever common_variance is True
if externals.exists('pprocess') and common_variance:
sls += [sphere_searchlight(cv, nproc=2, **skwargs)]
all_results = []
ds = datasets['3dsmall'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
for sl in sls:
# run searchlight
results = sl(ds)
all_results.append(results)
# check for correct number of spheres
self.failUnless(results.nfeatures == 106)
# and measures (one per xfold)
self.failUnless(len(results) == len(ds.UC))
# check for chance-level performance across all spheres
self.failUnless(0.4 < results.samples.mean() < 0.6)
mean_errors = results.samples.mean(axis=0)
# that we do get different errors ;)
self.failUnless(len(np.unique(mean_errors) > 3))
# check resonable sphere sizes
self.failUnless(len(sl.ca.roi_sizes) == 106)
self.failUnless(max(sl.ca.roi_sizes) == 7)
self.failUnless(min(sl.ca.roi_sizes) == 4)
# check base-class state
self.failUnlessEqual(sl.ca.raw_results.nfeatures, 106)
if len(all_results) > 1:
# if we had multiple searchlights, we can check either they all
# gave the same result (they should have)
aresults = np.array([a.samples for a in all_results])
dresults = np.abs(aresults - aresults.mean(axis=0))
dmax = np.max(dresults)
self.failUnless(dmax <= 1e-13)
def test_spatial_searchlight(self, common_variance):
"""Tests both generic and GNBSearchlight
Test of GNBSearchlight anyways requires a ground-truth
comparison to the generic version, so we are doing sweepargs here
"""
# compute N-1 cross-validation for each sphere
# YOH: unfortunately sample_clf_lin is not guaranteed
# to provide exactly the same results due to inherent
# iterative process. Therefore lets use something quick
# and pure Python
gnb = GNB(common_variance=common_variance)
transerror = TransferError(gnb)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1))
skwargs = dict(radius=1, enable_ca=['roi_sizes', 'raw_results'])
sls = [sphere_searchlight(cv, **skwargs),
#GNBSearchlight(gnb, NFoldSplitter(cvtype=1))
sphere_gnbsearchlight(gnb, NFoldSplitter(cvtype=1),
indexsum='fancy', **skwargs)
]
if externals.exists('scipy'):
sls += [ sphere_gnbsearchlight(gnb, NFoldSplitter(cvtype=1),
indexsum='sparse', **skwargs)]
# Just test nproc whenever common_variance is True
if externals.exists('pprocess') and common_variance:
sls += [sphere_searchlight(cv, nproc=2, **skwargs)]
all_results = []
ds = datasets['3dsmall'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
for sl in sls:
# run searchlight
results = sl(ds)
all_results.append(results)
# check for correct number of spheres
self.failUnless(results.nfeatures == 106)
# and measures (one per xfold)
self.failUnless(len(results) == len(ds.UC))
# check for chance-level performance across all spheres
self.failUnless(0.4 < results.samples.mean() < 0.6)
mean_errors = results.samples.mean(axis=0)
# that we do get different errors ;)
self.failUnless(len(np.unique(mean_errors) > 3))
# check resonable sphere sizes
self.failUnless(len(sl.ca.roi_sizes) == 106)
self.failUnless(max(sl.ca.roi_sizes) == 7)
self.failUnless(min(sl.ca.roi_sizes) == 4)
# check base-class state
self.failUnlessEqual(sl.ca.raw_results.nfeatures, 106)
if len(all_results) > 1:
# if we had multiple searchlights, we can check either they all
# gave the same result (they should have)
aresults = np.array([a.samples for a in all_results])
dresults = np.abs(aresults - aresults.mean(axis=0))
dmax = np.max(dresults)
self.failUnless(dmax <= 1e-13)
