def test_mapped_classifier(self):
samples = np.array([[0, 0, -1], [1, 0, 1], [-1, -1, 1], [-1, 0, 1],
[1, -1, 1]])
for mask, res in (([1, 1, 0], [1, 1, 1, -1,
-1]), ([1, 0, 1], [-1, 1, -1, -1, 1]),
([0, 1, 1], [-1, 1, -1, 1, -1])):
clf = MappedClassifier(clf=self.clf_sign,
mapper=mask_mapper(
np.array(mask, dtype=bool)))
self.assertEqual(clf.predict(samples), res)
def test_mapped_classifier(self):
samples = np.array([ [ 0, 0, -1], [ 1, 0, 1],
[-1, -1, 1], [-1, 0, 1],
[ 1, -1, 1] ])
for mask, res in (([1, 1, 0], [ 1, 1, 1, -1, -1]),
([1, 0, 1], [-1, 1, -1, -1, 1]),
([0, 1, 1], [-1, 1, -1, 1, -1])):
clf = MappedClassifier(clf=self.clf_sign,
mapper=mask_mapper(np.array(mask,
dtype=bool)))
self.assertEqual(clf.predict(samples), res)
def test_SplitRFE(self):
# just a smoke test ATM
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.misc.data_generators import normal_feature_dataset
#import mvpa2.featsel.rfe
#reload(mvpa2.featsel.rfe)
from mvpa2.featsel.rfe import RFE, SplitRFE
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.featsel.helpers import FractionTailSelector
from mvpa2.testing import ok_, assert_equal
clf = LinearCSVMC(C=1)
dataset = normal_feature_dataset(perlabel=20,
nlabels=2,
nfeatures=30,
snr=1.,
nonbogus_features=[1, 5])
# flip one of the meaningful features around to see
# if we are still getting proper selection
dataset.samples[:, dataset.a.nonbogus_features[1]] *= -1
# 4 partitions should be enough for testing
partitioner = NFoldPartitioner(count=4)
rfeclf = MappedClassifier(
clf,
SplitRFE(clf,
partitioner,
fselector=FractionTailSelector(0.2,
mode='discard',
tail='lower')))
r0 = repr(rfeclf)
ok_(rfeclf.mapper.nfeatures_min == 0)
rfeclf.train(dataset)
ok_(rfeclf.mapper.nfeatures_min > 0)
predictions = rfeclf(dataset).samples
# at least 1 of the nonbogus-features should be chosen
ok_(
len(
set(dataset.a.nonbogus_features).intersection(
rfeclf.mapper.slicearg)) > 0)
# check repr to have all needed pieces
r = repr(rfeclf)
s = str(rfeclf)
ok_(('partitioner=NFoldP' in r) or
('partitioner=mvpa2.generators.partition.NFoldPartitioner' in r))
ok_('lrn=' in r)
ok_(not 'slicearg=' in r)
assert_equal(r, r0)
def _test_mcasey20120222(): # pragma: no cover
# http://lists.alioth.debian.org/pipermail/pkg-exppsy-pymvpa/2012q1/002034.html
# This one is conditioned on allowing # of samples to be changed
# by the mapper provided to MappedClassifier. See
# https://github.com/yarikoptic/PyMVPA/tree/_tent/allow_ch_nsamples
import numpy as np
from mvpa2.datasets.base import dataset_wizard
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.mappers.base import ChainMapper
from mvpa2.mappers.svd import SVDMapper
from mvpa2.mappers.fx import mean_group_sample
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.measures.base import CrossValidation
mapper = ChainMapper([mean_group_sample(['targets','chunks']),
SVDMapper()])
clf = MappedClassifier(LinearCSVMC(), mapper)
cvte = CrossValidation(clf, NFoldPartitioner(),
enable_ca=['repetition_results', 'stats'])
ds = dataset_wizard(
samples=np.arange(32).reshape((8, -1)),
targets=[1, 1, 2, 2, 1, 1, 2, 2],
chunks=[1, 1, 1, 1, 2, 2, 2, 2])
errors = cvte(ds)
def test_chained_crossvalidation_searchlight():
from mvpa2.clfs.gnb import GNB
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.mappers.base import ChainMapper
from mvpa2.mappers.base import Mapper
from mvpa2.measures.base import CrossValidation
from mvpa2.measures.searchlight import sphere_searchlight
from mvpa2.testing.datasets import datasets
dataset = datasets['3dlarge'].copy()
dataset.fa['voxel_indices'] = dataset.fa.myspace
sample_clf = GNB() # fast and deterministic
class ZScoreFeaturesMapper(Mapper):
"""Very basic mapper which would take care about standardizing
all features within each sample separately
"""
def _forward_data(self, data):
return (data - np.mean(data, axis=1)[:, None])/np.std(data, axis=1)[:, None]
# only do partial to save time
sl_kwargs = dict(radius=2, center_ids=[3, 50])
clf_mapped = MappedClassifier(sample_clf, ZScoreFeaturesMapper())
cv = CrossValidation(clf_mapped, NFoldPartitioner())
sl = sphere_searchlight(cv, **sl_kwargs)
results_mapped = sl(dataset)
cv_chained = ChainMapper([ZScoreFeaturesMapper(auto_train=True),
CrossValidation(sample_clf, NFoldPartitioner())])
sl_chained = sphere_searchlight(cv_chained, **sl_kwargs)
results_chained = sl_chained(dataset)
assert_array_equal(results_mapped, results_chained)
def test_SplitRFE(self):
# just a smoke test ATM
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.misc.data_generators import normal_feature_dataset
#import mvpa2.featsel.rfe
#reload(mvpa2.featsel.rfe)
from mvpa2.featsel.rfe import RFE, SplitRFE
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.featsel.helpers import FractionTailSelector
from mvpa2.testing import ok_, assert_equal
clf = LinearCSVMC(C=1)
dataset = normal_feature_dataset(perlabel=20, nlabels=2, nfeatures=30,
snr=1., nonbogus_features=[1,5])
# flip one of the meaningful features around to see
# if we are still getting proper selection
dataset.samples[:, dataset.a.nonbogus_features[1]] *= -1
# 4 partitions should be enough for testing
partitioner = NFoldPartitioner(count=4)
rfeclf = MappedClassifier(
clf, SplitRFE(clf,
partitioner,
fselector=FractionTailSelector(
0.2, mode='discard', tail='lower')))
r0 = repr(rfeclf)
ok_(rfeclf.mapper.nfeatures_min == 0)
rfeclf.train(dataset)
ok_(rfeclf.mapper.nfeatures_min > 0)
predictions = rfeclf(dataset).samples
# at least 1 of the nonbogus-features should be chosen
ok_(len(set(dataset.a.nonbogus_features).intersection(
rfeclf.mapper.slicearg)) > 0)
# check repr to have all needed pieces
r = repr(rfeclf)
s = str(rfeclf)
ok_(('partitioner=NFoldP' in r) or
('partitioner=mvpa2.generators.partition.NFoldPartitioner' in r))
ok_('lrn=' in r)
ok_(not 'slicearg=' in r)
assert_equal(r, r0)
def test_gideon_weird_case(self):
"""Test if MappedClassifier could handle a mapper altering number of samples
'The utter collapse' -- communicated by Peter J. Kohler
Desire to collapse all samples per each category in training
and testing sets, thus resulting only in a single
sample/category per training and per testing.
It is a peculiar scenario which pin points the problem that so
far mappers assumed not to change number of samples
"""
from mvpa2.mappers.fx import mean_group_sample
from mvpa2.clfs.knn import kNN
from mvpa2.mappers.base import ChainMapper
ds = datasets['uni2large'].copy()
#ds = ds[ds.sa.chunks < 9]
accs = []
k = 1 # for kNN
nf = 1 # for NFoldPartitioner
for i in xrange(1): # # of random runs
ds.samples = np.random.randn(*ds.shape)
#
# There are 3 ways to accomplish needed goal
#
# 0. Hard way: overcome the problem by manually
# pre-splitting/meaning in a loop
from mvpa2.clfs.transerror import ConfusionMatrix
partitioner = NFoldPartitioner(nf)
meaner = mean_group_sample(['targets', 'partitions'])
cm = ConfusionMatrix()
te = TransferMeasure(kNN(k),
Splitter('partitions'),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'),
enable_ca=['stats'])
errors = []
for part in partitioner.generate(ds):
ds_meaned = meaner(part)
errors.append(np.asscalar(te(ds_meaned)))
cm += te.ca.stats
#print i, cm.stats['ACC']
accs.append(cm.stats['ACC'])
if False: # not yet working -- see _tent/allow_ch_nsamples
# branch for attempt to make it work
# 1. This is a "native way" IF we allow change of number
# of samples via _call to be done by MappedClassifier
# while operating solely on the mapped dataset
clf2 = MappedClassifier(
clf=kNN(k), #clf,
mapper=mean_group_sample(['targets', 'partitions']))
cv = CrossValidation(clf2,
NFoldPartitioner(nf),
postproc=None,
enable_ca=['stats'])
# meaning all should be ok since we should have ballanced
# sets across all chunks here
errors_native = cv(ds)
self.assertEqual(
np.max(np.abs(errors_native.samples[:, 0] - errors)), 0)
# 2. Work without fixes to MappedClassifier allowing
# change of # of samples
#
# CrossValidation will operate on a chain mapper which
# would perform necessary meaning first before dealing with
# kNN cons: .stats would not be exposed since ChainMapper
# doesn't expose them from ChainMapper (yet)
if __debug__ and 'ENFORCE_CA_ENABLED' in debug.active:
raise SkipTest("Known to fail while trying to enable "
"training_stats for the ChainMapper")
cv2 = CrossValidation(ChainMapper(
[mean_group_sample(['targets', 'partitions']),
kNN(k)],
space='targets'),
NFoldPartitioner(nf),
postproc=None)
errors_native2 = cv2(ds)
self.assertEqual(
np.max(np.abs(errors_native2.samples[:, 0] - errors)), 0)
# All of the ways should provide the same results
#print i, np.max(np.abs(errors_native.samples[:,0] - errors)), \
# np.max(np.abs(errors_native2.samples[:,0] - errors))
if False: # just to investigate the distribution if we have enough iterations
import pylab as pl
uaccs = np.unique(accs)
step = np.asscalar(np.unique(np.round(uaccs[1:] - uaccs[:-1], 4)))
bins = np.linspace(0., 1., np.round(1. / step + 1))
xx = pl.hist(accs, bins=bins, align='left')
pl.xlim((0. - step / 2, 1. + step / 2))
def test_SplitRFE(self, fmeasure):
# just a smoke test ATM
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.misc.data_generators import normal_feature_dataset
#import mvpa2.featsel.rfe
#reload(mvpa2.featsel.rfe)
from mvpa2.featsel.rfe import RFE, SplitRFE
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.featsel.helpers import FractionTailSelector
from mvpa2.testing import ok_, assert_equal
clf = LinearCSVMC(C=1)
dataset = normal_feature_dataset(perlabel=20, nlabels=2, nfeatures=11,
snr=1., nonbogus_features=[1, 5])
# flip one of the meaningful features around to see
# if we are still getting proper selection
dataset.samples[:, dataset.a.nonbogus_features[1]] *= -1
# 3 partitions should be enough for testing
partitioner = NFoldPartitioner(count=3)
rfeclf = MappedClassifier(
clf, SplitRFE(clf,
partitioner,
fselector=FractionTailSelector(
0.5, mode='discard', tail='lower'),
fmeasure=fmeasure,
# need to update only when using clf's sens anal
update_sensitivity=fmeasure is None))
r0 = repr(rfeclf)
ok_(rfeclf.mapper.nfeatures_min == 0)
rfeclf.train(dataset)
ok_(rfeclf.mapper.nfeatures_min > 0)
predictions = rfeclf(dataset).samples
# at least 1 of the nonbogus-features should be chosen
ok_(len(set(dataset.a.nonbogus_features).intersection(
rfeclf.mapper.slicearg)) > 0)
# check repr to have all needed pieces
r = repr(rfeclf)
s = str(rfeclf)
ok_(('partitioner=NFoldP' in r) or
('partitioner=mvpa2.generators.partition.NFoldPartitioner' in r))
ok_('lrn=' in r)
ok_(not 'slicearg=' in r)
assert_equal(r, r0)
if externals.exists('joblib'):
rfeclf.mapper.nproc = -1
# compare results against the one ran in parallel
_slicearg = rfeclf.mapper.slicearg
_predictions = predictions
rfeclf.train(dataset)
predictions = rfeclf(dataset).samples
assert_array_equal(predictions, _predictions)
assert_array_equal(_slicearg, rfeclf.mapper.slicearg)
# Test that we can collect stats from cas within cross-validation
sensitivities = []
nested_errors = []
nested_nfeatures = []
def store_me(data, node, result):
sens = node.measure.get_sensitivity_analyzer(force_train=False)(data)
sensitivities.append(sens)
nested_errors.append(node.measure.mapper.ca.nested_errors)
nested_nfeatures.append(node.measure.mapper.ca.nested_nfeatures)
cv = CrossValidation(rfeclf, NFoldPartitioner(count=1), callback=store_me,
enable_ca=['stats'])
_ = cv(dataset)
# just to make sure we collected them
assert_equal(len(sensitivities), 1)
assert_equal(len(nested_errors), 1)
assert_equal(len(nested_nfeatures), 1)
def test_SplitRFE(self, fmeasure):
# just a smoke test ATM
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.misc.data_generators import normal_feature_dataset
#import mvpa2.featsel.rfe
#reload(mvpa2.featsel.rfe)
from mvpa2.featsel.rfe import RFE, SplitRFE
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.featsel.helpers import FractionTailSelector
from mvpa2.testing import ok_, assert_equal
clf = LinearCSVMC(C=1)
dataset = normal_feature_dataset(perlabel=20,
nlabels=2,
nfeatures=11,
snr=1.,
nonbogus_features=[1, 5])
# flip one of the meaningful features around to see
# if we are still getting proper selection
dataset.samples[:, dataset.a.nonbogus_features[1]] *= -1
# 3 partitions should be enough for testing
partitioner = NFoldPartitioner(count=3)
rfeclf = MappedClassifier(
clf,
SplitRFE(
clf,
partitioner,
fselector=FractionTailSelector(0.5,
mode='discard',
tail='lower'),
fmeasure=fmeasure,
# need to update only when using clf's sens anal
update_sensitivity=fmeasure is None))
r0 = repr(rfeclf)
ok_(rfeclf.mapper.nfeatures_min == 0)
rfeclf.train(dataset)
ok_(rfeclf.mapper.nfeatures_min > 0)
predictions = rfeclf(dataset).samples
# at least 1 of the nonbogus-features should be chosen
ok_(
len(
set(dataset.a.nonbogus_features).intersection(
rfeclf.mapper.slicearg)) > 0)
# check repr to have all needed pieces
r = repr(rfeclf)
s = str(rfeclf)
ok_(('partitioner=NFoldP' in r) or
('partitioner=mvpa2.generators.partition.NFoldPartitioner' in r))
ok_('lrn=' in r)
ok_(not 'slicearg=' in r)
assert_equal(r, r0)
if externals.exists('joblib'):
rfeclf.mapper.nproc = -1
# compare results against the one ran in parallel
_slicearg = rfeclf.mapper.slicearg
_predictions = predictions
rfeclf.train(dataset)
predictions = rfeclf(dataset).samples
assert_array_equal(predictions, _predictions)
assert_array_equal(_slicearg, rfeclf.mapper.slicearg)
# Test that we can collect stats from cas within cross-validation
sensitivities = []
nested_errors = []
nested_nfeatures = []
def store_me(data, node, result):
sens = node.measure.get_sensitivity_analyzer(
force_train=False)(data)
sensitivities.append(sens)
nested_errors.append(node.measure.mapper.ca.nested_errors)
nested_nfeatures.append(node.measure.mapper.ca.nested_nfeatures)
cv = CrossValidation(rfeclf,
NFoldPartitioner(count=1),
callback=store_me,
enable_ca=['stats'])
_ = cv(dataset)
# just to make sure we collected them
assert_equal(len(sensitivities), 1)
assert_equal(len(nested_errors), 1)
assert_equal(len(nested_nfeatures), 1)
