def test_feature_selection_classifier(self):
from mvpa2.featsel.base import \
SensitivityBasedFeatureSelection
from mvpa2.featsel.helpers import \
FixedNElementTailSelector
# should give lowest weight to the feature with lowest index
sens_ana = SillySensitivityAnalyzer()
# should give lowest weight to the feature with highest index
sens_ana_rev = SillySensitivityAnalyzer(mult=-1)
# corresponding feature selections
feat_sel = SensitivityBasedFeatureSelection(
sens_ana, FixedNElementTailSelector(1, mode='discard'))
feat_sel_rev = SensitivityBasedFeatureSelection(
sens_ana_rev, FixedNElementTailSelector(1))
samples = np.array([[0, 0, -1], [1, 0, 1], [-1, -1, 1], [-1, 0, 1],
[1, -1, 1]])
testdata3 = dataset_wizard(samples=samples, targets=1)
# dummy train data so proper mapper gets created
traindata = dataset_wizard(samples=np.array([[0, 0, -1], [1, 0, 1]]),
targets=[1, 2])
# targets
res110 = [1, 1, 1, -1, -1]
res011 = [-1, 1, -1, 1, -1]
# first classifier -- 0th feature should be discarded
clf011 = FeatureSelectionClassifier(self.clf_sign,
feat_sel,
enable_ca=['feature_ids'])
self.clf_sign.ca.change_temporarily(enable_ca=['estimates'])
clf011.train(traindata)
self.assertEqual(clf011.predict(testdata3.samples), res011)
# just silly test if we get values assigned in the 'ProxyClassifier'
self.assertTrue(len(clf011.ca.estimates) == len(res110),
msg="We need to pass values into ProxyClassifier")
self.clf_sign.ca.reset_changed_temporarily()
self.assertEqual(clf011.mapper._oshape, (2, ))
"Feature selection classifier had to be trained on 2 features"
# first classifier -- last feature should be discarded
clf011 = FeatureSelectionClassifier(self.clf_sign, feat_sel_rev)
clf011.train(traindata)
self.assertEqual(clf011.predict(testdata3.samples), res110)
def test_feature_selection_classifier(self):
from mvpa2.featsel.base import \
SensitivityBasedFeatureSelection
from mvpa2.featsel.helpers import \
FixedNElementTailSelector
# should give lowest weight to the feature with lowest index
sens_ana = SillySensitivityAnalyzer()
# should give lowest weight to the feature with highest index
sens_ana_rev = SillySensitivityAnalyzer(mult=-1)
# corresponding feature selections
feat_sel = SensitivityBasedFeatureSelection(sens_ana,
FixedNElementTailSelector(1, mode='discard'))
feat_sel_rev = SensitivityBasedFeatureSelection(sens_ana_rev,
FixedNElementTailSelector(1))
samples = np.array([ [0, 0, -1], [1, 0, 1], [-1, -1, 1],
[-1, 0, 1], [1, -1, 1] ])
testdata3 = dataset_wizard(samples=samples, targets=1)
# dummy train data so proper mapper gets created
traindata = dataset_wizard(samples=np.array([ [0, 0, -1], [1, 0, 1] ]),
targets=[1, 2])
# targets
res110 = [1, 1, 1, -1, -1]
res011 = [-1, 1, -1, 1, -1]
# first classifier -- 0th feature should be discarded
clf011 = FeatureSelectionClassifier(self.clf_sign, feat_sel,
enable_ca=['feature_ids'])
self.clf_sign.ca.change_temporarily(enable_ca=['estimates'])
clf011.train(traindata)
self.assertEqual(clf011.predict(testdata3.samples), res011)
# just silly test if we get values assigned in the 'ProxyClassifier'
self.assertTrue(len(clf011.ca.estimates) == len(res110),
msg="We need to pass values into ProxyClassifier")
self.clf_sign.ca.reset_changed_temporarily()
self.assertEqual(clf011.mapper._oshape, (2,))
"Feature selection classifier had to be trained on 2 features"
# first classifier -- last feature should be discarded
clf011 = FeatureSelectionClassifier(self.clf_sign, feat_sel_rev)
clf011.train(traindata)
self.assertEqual(clf011.predict(testdata3.samples), res110)
def test_feature_selection_classifier_with_regression(self):
from mvpa2.featsel.base import \
SensitivityBasedFeatureSelection
from mvpa2.featsel.helpers import \
FixedNElementTailSelector
if sample_clf_reg is None:
# none regression was found, so nothing to test
return
# should give lowest weight to the feature with lowest index
sens_ana = SillySensitivityAnalyzer()
# corresponding feature selections
feat_sel = SensitivityBasedFeatureSelection(sens_ana,
FixedNElementTailSelector(1, mode='discard'))
# now test with regression-based classifier. The problem is
# that it is determining predictions twice from values and
# then setting the values from the results, which the second
# time is set to predictions. The final outcome is that the
# values are actually predictions...
dat = dataset_wizard(samples=np.random.randn(4, 10),
targets=[-1, -1, 1, 1])
clf_reg = FeatureSelectionClassifier(sample_clf_reg, feat_sel)
clf_reg.train(dat)
_ = clf_reg.predict(dat.samples)
self.failIf((np.array(clf_reg.ca.estimates)
- clf_reg.ca.predictions).sum()==0,
msg="Values were set to the predictions in %s." %
sample_clf_reg)
def test_feature_selection_classifier_with_regression(self):
from mvpa2.featsel.base import \
SensitivityBasedFeatureSelection
from mvpa2.featsel.helpers import \
FixedNElementTailSelector
if sample_clf_reg is None:
# none regression was found, so nothing to test
return
# should give lowest weight to the feature with lowest index
sens_ana = SillySensitivityAnalyzer()
# corresponding feature selections
feat_sel = SensitivityBasedFeatureSelection(
sens_ana, FixedNElementTailSelector(1, mode='discard'))
# now test with regression-based classifier. The problem is
# that it is determining predictions twice from values and
# then setting the values from the results, which the second
# time is set to predictions. The final outcome is that the
# values are actually predictions...
dat = dataset_wizard(samples=np.random.randn(4, 10),
targets=[-1, -1, 1, 1])
clf_reg = FeatureSelectionClassifier(sample_clf_reg, feat_sel)
clf_reg.train(dat)
_ = clf_reg.predict(dat.samples)
self.failIf(
(np.array(clf_reg.ca.estimates) -
clf_reg.ca.predictions).sum() == 0,
msg="Values were set to the predictions in %s." % sample_clf_reg)
def test_rfe_sensmap():
# http://lists.alioth.debian.org/pipermail/pkg-exppsy-pymvpa/2013q3/002538.html
# just a smoke test. fails with
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import FeatureSelectionClassifier
from mvpa2.measures.base import CrossValidation, RepeatedMeasure
from mvpa2.generators.splitters import Splitter
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.misc.errorfx import mean_mismatch_error
from mvpa2.mappers.fx import mean_sample
from mvpa2.mappers.fx import maxofabs_sample
from mvpa2.generators.base import Repeater
from mvpa2.featsel.rfe import RFE
from mvpa2.featsel.helpers import FractionTailSelector, BestDetector
from mvpa2.featsel.helpers import NBackHistoryStopCrit
from mvpa2.datasets import vstack
from mvpa2.misc.data_generators import normal_feature_dataset
# Let's simulate the beast -- 6 categories total groupped into 3
# super-ordinate, and actually without any 'superordinate' effect
# since subordinate categories independent
fds = normal_feature_dataset(nlabels=3,
snr=1, # 100, # pure signal! ;)
perlabel=9,
nfeatures=6,
nonbogus_features=range(3),
nchunks=3)
clfsvm = LinearCSVMC()
rfesvm = RFE(clfsvm.get_sensitivity_analyzer(postproc=maxofabs_sample()),
CrossValidation(
clfsvm,
NFoldPartitioner(),
errorfx=mean_mismatch_error, postproc=mean_sample()),
Repeater(2),
fselector=FractionTailSelector(0.70, mode='select', tail='upper'),
stopping_criterion=NBackHistoryStopCrit(BestDetector(), 10),
update_sensitivity=True)
fclfsvm = FeatureSelectionClassifier(clfsvm, rfesvm)
sensanasvm = fclfsvm.get_sensitivity_analyzer(postproc=maxofabs_sample())
# manually repeating/splitting so we do both RFE sensitivity and classification
senses, errors = [], []
for i, pset in enumerate(NFoldPartitioner().generate(fds)):
# split partitioned dataset
split = [d for d in Splitter('partitions').generate(pset)]
senses.append(sensanasvm(split[0])) # and it also should train the classifier so we would ask it about error
errors.append(mean_mismatch_error(fclfsvm.predict(split[1]), split[1].targets))
senses = vstack(senses)
errors = vstack(errors)
# Let's compare against rerunning the beast simply for classification with CV
errors_cv = CrossValidation(fclfsvm, NFoldPartitioner(), errorfx=mean_mismatch_error)(fds)
# and they should match
assert_array_equal(errors, errors_cv)
# buggy!
cv_sensana_svm = RepeatedMeasure(sensanasvm, NFoldPartitioner())
senses_rm = cv_sensana_svm(fds)
#print senses.samples, senses_rm.samples
#print errors, errors_cv.samples
assert_raises(AssertionError,
assert_array_almost_equal,
senses.samples, senses_rm.samples)
raise SkipTest("Known failure for repeated measures: https://github.com/PyMVPA/PyMVPA/issues/117")
# should i zscore?
lidx = np.arange(32)
pidx = np.arange(32, 64)
lres = cv(betas[lidx].copy())
lresults.append(lres)
print "language: " + str(np.mean(lres.samples))
cv.untrain()
pres = cv(betas[pidx].copy())
presults.append(pres)
print "pictures: " + str(np.mean(pres.samples))
cv.untrain()
fclf.train(betas[lidx].copy())
l2presults.append(np.mean(fclf.predict(betas[pidx]) == betas[pidx].sa.targets))
fclf.untrain()
fclf.train(betas[pidx])
p2lresults.append(np.mean(fclf.predict(betas[lidx]) == betas[lidx].sa.targets))
import matplotlib.pyplot as plt
def plotsubs(lr, pr, l2p, p2l, c=None, title=None, bar_width=.2, opacity=.4, error_config={'ecolor': '0.3'}):
# results is the concatenated output of cv across subjects... or something.
f, (ax1, ax2) = plt.subplots(2, figsize=(12,6))
index = np.arange(len(lr))
lheights = []
lerrbars = []
pheights = []
perrbars = []
