def test_corrstability_smoketest(ds):
if not 'chunks' in ds.sa:
return
if len(ds.sa['targets'].unique) > 30:
# was regression dataset
return
# very basic testing since
cs = CorrStability()
#ds = datasets['uni2small']
out = cs(ds)
assert_equal(out.shape, (ds.nfeatures, ))
ok_(np.all(out >= -1.001)) # it should be a correlation after all
ok_(np.all(out <= 1.001))
# and theoretically those nonbogus features should have higher values
if 'nonbogus_targets' in ds.fa:
bogus_features = np.array([x == None for x in ds.fa.nonbogus_targets])
assert_array_less(np.mean(out[bogus_features]),
np.mean(out[~bogus_features]))
# and if we move targets to alternative location
ds = ds.copy(deep=True)
ds.sa['alt'] = ds.T
ds.sa.pop('targets')
assert_raises(KeyError, cs, ds)
cs = CorrStability('alt')
out_ = cs(ds)
assert_array_equal(out, out_)
def test_corrstability_smoketest(ds):
if not 'chunks' in ds.sa:
return
if len(ds.sa['targets'].unique) > 30:
# was regression dataset
return
# very basic testing since
cs = CorrStability()
#ds = datasets['uni2small']
out = cs(ds)
assert_equal(out.shape, (ds.nfeatures,))
ok_(np.all(out >= -1.001)) # it should be a correlation after all
ok_(np.all(out <= 1.001))
# and theoretically those nonbogus features should have higher values
if 'nonbogus_targets' in ds.fa:
bogus_features = np.array([x==None for x in ds.fa.nonbogus_targets])
assert_array_less(np.mean(out[bogus_features]), np.mean(out[~bogus_features]))
# and if we move targets to alternative location
ds = ds.copy(deep=True)
ds.sa['alt'] = ds.T
ds.sa.pop('targets')
assert_raises(KeyError, cs, ds)
cs = CorrStability('alt')
out_ = cs(ds)
assert_array_equal(out, out_)
def test_gnbsearchlight_permutations():
import mvpa2
from mvpa2.base.node import ChainNode
from mvpa2.clfs.gnb import GNB
from mvpa2.generators.base import Repeater
from mvpa2.generators.partition import NFoldPartitioner, OddEvenPartitioner
#import mvpa2.generators.permutation
#reload(mvpa2.generators.permutation)
from mvpa2.generators.permutation import AttributePermutator
from mvpa2.testing.datasets import datasets
from mvpa2.measures.base import CrossValidation
from mvpa2.measures.gnbsearchlight import sphere_gnbsearchlight
from mvpa2.measures.searchlight import sphere_searchlight
from mvpa2.mappers.fx import mean_sample
from mvpa2.misc.errorfx import mean_mismatch_error
from mvpa2.clfs.stats import MCNullDist
from mvpa2.testing.tools import assert_raises, ok_, assert_array_less
# mvpa2.debug.active = ['APERM', 'SLC'] #, 'REPM']
# mvpa2.debug.metrics += ['pid']
count = 10
nproc = 1 + int(mvpa2.externals.exists('pprocess'))
ds = datasets['3dsmall'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
slkwargs = dict(radius=3, space='voxel_indices', enable_ca=['roi_sizes'],
center_ids=[1, 10, 70, 100])
mvpa2.seed(mvpa2._random_seed)
clf = GNB()
splt = NFoldPartitioner(cvtype=2, attr='chunks')
repeater = Repeater(count=count)
permutator = AttributePermutator('targets', limit={'partitions': 1}, count=1)
null_sl = sphere_gnbsearchlight(clf, ChainNode([splt, permutator], space=splt.get_space()),
postproc=mean_sample(), errorfx=mean_mismatch_error,
**slkwargs)
distr_est = MCNullDist(repeater, tail='left', measure=null_sl,
enable_ca=['dist_samples'])
sl = sphere_gnbsearchlight(clf, splt,
reuse_neighbors=True,
null_dist=distr_est, postproc=mean_sample(),
errorfx=mean_mismatch_error,
**slkwargs)
if __debug__: # assert is done only without -O mode
assert_raises(NotImplementedError, sl, ds)
# "ad-hoc searchlights can't handle yet varying targets across partitions"
if False:
# after above limitation is removed -- enable
sl_map = sl(ds)
sl_null_prob = sl.ca.null_prob.samples.copy()
mvpa2.seed(mvpa2._random_seed)
### 'normal' Searchlight
clf = GNB()
splt = NFoldPartitioner(cvtype=2, attr='chunks')
repeater = Repeater(count=count)
permutator = AttributePermutator('targets', limit={'partitions': 1}, count=1)
# rng=np.random.RandomState(0)) # to trigger failure since the same np.random state
# would be reused across all pprocesses
null_cv = CrossValidation(clf, ChainNode([splt, permutator], space=splt.get_space()),
postproc=mean_sample())
null_sl_normal = sphere_searchlight(null_cv, nproc=nproc, **slkwargs)
distr_est_normal = MCNullDist(repeater, tail='left', measure=null_sl_normal,
enable_ca=['dist_samples'])
cv = CrossValidation(clf, splt, errorfx=mean_mismatch_error,
enable_ca=['stats'], postproc=mean_sample() )
sl = sphere_searchlight(cv, nproc=nproc, null_dist=distr_est_normal, **slkwargs)
sl_map_normal = sl(ds)
sl_null_prob_normal = sl.ca.null_prob.samples.copy()
# For every feature -- we should get some variance in estimates In
# case of failure they are all really close to each other (up to
# numerical precision), so variance will be close to 0
assert_array_less(-np.var(distr_est_normal.ca.dist_samples.samples[0],
axis=1), -1e-5)
for s in distr_est_normal.ca.dist_samples.samples[0]:
ok_(len(np.unique(s)) > 1)