def test_feature_selection_pipeline(self):
sens_ana = SillySensitivityAnalyzer()
data = self.get_data()
data_nfeatures = data.nfeatures
# test silly one first ;-)
self.assertEqual(
sens_ana(data).samples[0, 0], -int(data_nfeatures / 2))
# OLD: first remove 25% == 6, and then 4, total removing 10
# NOW: test should be independent of the numerical number of features
feature_selections = [
SensitivityBasedFeatureSelection(sens_ana,
FractionTailSelector(0.25)),
SensitivityBasedFeatureSelection(sens_ana,
FixedNElementTailSelector(4))
]
# create a FeatureSelection pipeline
feat_sel_pipeline = ChainMapper(feature_selections)
feat_sel_pipeline.train(data)
resds = feat_sel_pipeline(data)
self.assertEqual(len(feat_sel_pipeline),
len(feature_selections),
msg="Test the property feature_selections")
desired_nfeatures = int(np.ceil(data_nfeatures * 0.75))
self.assertEqual([fe._oshape[0] for fe in feat_sel_pipeline],
[desired_nfeatures, desired_nfeatures - 4])
def __init__(self,
sensitivity_analyzer,
feature_selector=FractionTailSelector(0.05),
train_analyzer=True,
**kwargs
):
"""Initialize feature selection
Parameters
----------
sensitivity_analyzer : FeaturewiseMeasure
sensitivity analyzer to come up with sensitivity
feature_selector : Functor
Given a sensitivity map it has to return the ids of those
features that should be kept.
train_analyzer : bool
Flag whether to train the sensitivity analyzer on the input dataset
during train(). If False, the employed sensitivity measure has to be
already trained before.
"""
# base init first
FeatureSelection.__init__(self, **kwargs)
self.__sensitivity_analyzer = sensitivity_analyzer
"""Sensitivity analyzer to use once"""
self.__feature_selector = feature_selector
"""Functor which takes care about removing some features."""
self.__train_analyzer = train_analyzer
def __test_matthias_question(self):
rfe_clf = LinearCSVMC(C=1)
rfesvm_split = SplitClassifier(rfe_clf)
clf = \
FeatureSelectionClassifier(
clf = LinearCSVMC(C=1),
feature_selection = RFE(
sensitivity_analyzer = rfesvm_split.get_sensitivity_analyzer(
combiner=first_axis_mean,
transformer=np.abs),
transfer_error=ConfusionBasedError(
rfesvm_split,
confusion_state="confusion"),
stopping_criterion=FixedErrorThresholdStopCrit(0.20),
feature_selector=FractionTailSelector(
0.2, mode='discard', tail='lower'),
update_sensitivity=True))
no_permutations = 1000
permutator = AttributePermutator('targets', count=no_permutations)
cv = CrossValidation(clf,
NFoldPartitioner(),
null_dist=MCNullDist(permutator, tail='left'),
enable_ca=['stats'])
error = cv(datasets['uni2small'])
self.assertTrue(error < 0.4)
self.assertTrue(cv.ca.null_prob < 0.05)
def test_split_samples_probability_mapper(self):
skip_if_no_external('scipy')
nf = 10
ns = 100
nsubj = 5
nchunks = 5
data = np.random.normal(size=(ns, nf))
ds = AttrDataset(data,
sa=dict(sidx=np.arange(ns),
targets=np.arange(ns) % nchunks,
chunks=np.floor(np.arange(ns) * nchunks / ns),
subjects=np.arange(ns) /
(ns / nsubj / nchunks) % nsubj),
fa=dict(fidx=np.arange(nf)))
analyzer = OneWayAnova()
element_selector = FractionTailSelector(.4,
mode='select',
tail='upper')
common = True
m = SplitSamplesProbabilityMapper(analyzer,
'subjects',
probability_label='fprob',
select_common_features=common,
selector=element_selector)
m.train(ds)
y = m(ds)
z = m(ds.samples)
assert_array_equal(z, y.samples)
assert_equal(y.shape, (100, 4))
def __init__(self,
sensitivity_analyzer,
split_by_labels,
select_common_features=True,
probability_label=None,
probability_combiner=None,
selector=FractionTailSelector(0.05),
**kwargs):
'''
Parameters
----------
sensitivity_analyzer: FeaturewiseMeasure
Sensitivity analyzer to come up with sensitivity.
split_by_labels: str or list of str
Sample labels on which input datasets are split before
data is selected.
select_common_features: bool
True means that the same features are selected after the split.
probablity_label: None or str
If None, then the output dataset ds from the
sensitivity_analyzer is taken to select the samples.
If not None it takes ds.sa['probablity_label'].
For example if sensitivity_analyzer=OneWayAnova then
probablity_label='fprob' is a sensible value.
probability_combiner: function
If select_common_features is True, then this function is
applied to the feature scores across splits. If None,
it uses lambda x:np.sum(-np.log(x)) which is sensible if
the scores are probability values
selector: Selector
function that returns the indices to keep.
'''
SliceMapper.__init__(self, None, **kwargs)
if probability_combiner is None:
def f(x):
y = -np.log(x.ravel())
# address potential NaNs
# set to max value in y
m = np.isnan(y)
if np.all(m):
return 0 # p=1
y[m] = np.max(y[np.logical_not(m)])
return np.sum(y)
probability_combiner = f # avoid lambda as h5py doesn't like it
self._sensitivity_analyzer = sensitivity_analyzer
self._split_by_labels = split_by_labels
self._select_common_features = select_common_features
self._probability_label = probability_label
self._probability_combiner = probability_combiner
self._selector = selector
def test_james_problem_multiclass(self):
percent = 80
dataset = datasets['uni4large']
#dataset = dataset[:, dataset.a.nonbogus_features]
rfesvm_split = LinearCSVMC()
fs = \
RFE(rfesvm_split.get_sensitivity_analyzer(
postproc=ChainMapper([
#FxMapper('features', l2_normed),
#FxMapper('samples', np.mean),
#FxMapper('samples', np.abs)
FxMapper('features', lambda x: np.argsort(np.abs(x))),
#maxofabs_sample()
mean_sample()
])),
ProxyMeasure(rfesvm_split,
postproc=BinaryFxNode(mean_mismatch_error,
'targets')),
Splitter('train'),
fselector=FractionTailSelector(
percent / 100.0,
mode='select', tail='upper'), update_sensitivity=True)
clf = FeatureSelectionClassifier(
LinearCSVMC(),
# on features selected via RFE
fs)
# update sensitivity at each step (since we're not using the
# same CLF as sensitivity analyzer)
class StoreResults(object):
def __init__(self):
self.storage = []
def __call__(self, data, node, result):
self.storage.append((node.measure.mapper.ca.history,
node.measure.mapper.ca.errors)),
cv_storage = StoreResults()
cv = CrossValidation(clf,
NFoldPartitioner(),
postproc=mean_sample(),
callback=cv_storage,
enable_ca=['stats'])
#cv = SplitClassifier(clf)
try:
error = cv(dataset).samples.squeeze()
except Exception, e:
self.fail('CrossValidation cannot handle classifier with RFE '
'feature selection. Got exception: %s' % (e, ))
def test_SplitRFE(self):
# just a smoke test ATM
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.misc.data_generators import normal_feature_dataset
#import mvpa2.featsel.rfe
#reload(mvpa2.featsel.rfe)
from mvpa2.featsel.rfe import RFE, SplitRFE
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.featsel.helpers import FractionTailSelector
from mvpa2.testing import ok_, assert_equal
clf = LinearCSVMC(C=1)
dataset = normal_feature_dataset(perlabel=20,
nlabels=2,
nfeatures=30,
snr=1.,
nonbogus_features=[1, 5])
# flip one of the meaningful features around to see
# if we are still getting proper selection
dataset.samples[:, dataset.a.nonbogus_features[1]] *= -1
# 4 partitions should be enough for testing
partitioner = NFoldPartitioner(count=4)
rfeclf = MappedClassifier(
clf,
SplitRFE(clf,
partitioner,
fselector=FractionTailSelector(0.2,
mode='discard',
tail='lower')))
r0 = repr(rfeclf)
ok_(rfeclf.mapper.nfeatures_min == 0)
rfeclf.train(dataset)
ok_(rfeclf.mapper.nfeatures_min > 0)
predictions = rfeclf(dataset).samples
# at least 1 of the nonbogus-features should be chosen
ok_(
len(
set(dataset.a.nonbogus_features).intersection(
rfeclf.mapper.slicearg)) > 0)
# check repr to have all needed pieces
r = repr(rfeclf)
s = str(rfeclf)
ok_(('partitioner=NFoldP' in r) or
('partitioner=mvpa2.generators.partition.NFoldPartitioner' in r))
ok_('lrn=' in r)
ok_(not 'slicearg=' in r)
assert_equal(r, r0)
def test_james_problem(self):
percent = 80
dataset = datasets['uni2small']
rfesvm_split = LinearCSVMC()
fs = \
RFE(rfesvm_split.get_sensitivity_analyzer(),
ProxyMeasure(rfesvm_split,
postproc=BinaryFxNode(mean_mismatch_error,
'targets')),
Splitter('train'),
fselector=FractionTailSelector(
percent / 100.0,
mode='select', tail='upper'), update_sensitivity=True)
clf = FeatureSelectionClassifier(
LinearCSVMC(),
# on features selected via RFE
fs)
# update sensitivity at each step (since we're not using the
# same CLF as sensitivity analyzer)
class StoreResults(object):
def __init__(self):
self.storage = []
def __call__(self, data, node, result):
self.storage.append((node.measure.mapper.ca.history,
node.measure.mapper.ca.errors)),
cv_storage = StoreResults()
cv = CrossValidation(clf,
NFoldPartitioner(),
postproc=mean_sample(),
callback=cv_storage,
enable_ca=['confusion']) # TODO -- it is stats
#cv = SplitClassifier(clf)
try:
error = cv(dataset).samples.squeeze()
except Exception as e:
self.fail('CrossValidation cannot handle classifier with RFE '
'feature selection. Got exception: %s' % (e, ))
assert (len(cv_storage.storage) == len(dataset.sa['chunks'].unique))
assert (len(cv_storage.storage[0]) == 2)
assert (len(cv_storage.storage[0][0]) == dataset.nfeatures)
self.assertTrue(error < 0.2)
def test_mapped_classifier_sensitivity_analyzer(self, clf):
"""Test sensitivity of the mapped classifier
"""
# Assuming many defaults it is as simple as
mclf = FeatureSelectionClassifier(
clf,
SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.5, mode='select', tail='upper')),
enable_ca=['training_stats'])
sana = mclf.get_sensitivity_analyzer(postproc=sumofabs_sample(),
enable_ca=["sensitivities"])
# and lets look at all sensitivities
dataset = datasets['uni2small']
# and we get sensitivity analyzer which works on splits
sens = sana(dataset)
self.assertEqual(sens.shape, (1, dataset.nfeatures))
def __init__(self,
fmeasure,
pmeasure,
splitter,
fselector=FractionTailSelector(0.05),
update_sensitivity=True,
nfeatures_min=0,
**kwargs):
# XXX Allow for multiple stopping criterions, e.g. error not decreasing
# anymore OR number of features less than threshold
"""Initialize recursive feature elimination
Parameters
----------
fmeasure : FeaturewiseMeasure
pmeasure : Measure
used to compute the transfer error of a classifier based on a
certain feature set on the test dataset.
NOTE: If sensitivity analyzer is based on the same
classifier as transfer_error is using, make sure you
initialize transfer_error with train=False, otherwise
it would train classifier twice without any necessity.
splitter: Splitter
This splitter instance has to generate at least two dataset splits
when called with the input dataset. The first split serves as the
training dataset and the second as the evaluation dataset.
fselector : Functor
Given a sensitivity map it has to return the ids of those
features that should be kept.
update_sensitivity : bool
If False the sensitivity map is only computed once and reused
for each iteration. Otherwise the sensitivities are
recomputed at each selection step.
nfeatures_min : int
Number of features for RFE to stop if reached.
"""
# bases init first
IterativeFeatureSelection.__init__(self, fmeasure, pmeasure, splitter,
fselector, **kwargs)
self.__update_sensitivity = update_sensitivity
"""Flag whether sensitivity map is recomputed for each step."""
self._nfeatures_min = nfeatures_min
def test_union_feature_selection(self):
# two methods: 5% highes F-scores, non-zero SMLR weights
fss = [SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.05, mode='select', tail='upper')),
SensitivityBasedFeatureSelection(
SMLRWeights(SMLR(lm=1, implementation="C"),
postproc=sumofabs_sample()),
RangeElementSelector(mode='select'))]
fs = CombinedFeatureSelection(fss, method='union')
od_union = fs(self.dataset)
self.assertTrue(fs.method == 'union')
# check output dataset
self.assertTrue(od_union.nfeatures <= self.dataset.nfeatures)
# again for intersection
fs = CombinedFeatureSelection(fss, method='intersection')
od_intersect = fs(self.dataset)
assert_true(od_intersect.nfeatures < od_union.nfeatures)
def __test_fspipeline_with_split_classifier(self, basic_clf):
#basic_clf = LinearNuSVMC()
multi_clf = MulticlassClassifier(clf=basic_clf)
#svm_weigths = LinearSVMWeights(svm)
# Proper RFE: aggregate sensitivities across multiple splits,
# but also due to multi class those need to be aggregated
# somehow. Transfer error here should be 'leave-1-out' error
# of split classifier itself
sclf = SplitClassifier(clf=basic_clf)
rfe = RFE(sensitivity_analyzer=sclf.get_sensitivity_analyzer(
enable_ca=["sensitivities"]),
transfer_error=trans_error,
feature_selector=FeatureSelectionPipeline([
FractionTailSelector(0.5),
FixedNElementTailSelector(1)
]),
train_pmeasure=True)
# and we get sensitivity analyzer which works on splits and uses
# sensitivity
selected_features = rfe(self.dataset)
def test_rfe(self, clf):
# sensitivity analyser and transfer error quantifier use the SAME clf!
sens_ana = clf.get_sensitivity_analyzer(postproc=maxofabs_sample())
pmeasure = ProxyMeasure(clf,
postproc=BinaryFxNode(mean_mismatch_error,
'targets'))
cvmeasure = CrossValidation(clf,
NFoldPartitioner(),
errorfx=mean_mismatch_error,
postproc=mean_sample())
rfesvm_split = SplitClassifier(clf, OddEvenPartitioner())
# explore few recipes
for rfe, data in [
# because the clf is already trained when computing the sensitivity
# map, prevent retraining for transfer error calculation
# Use absolute of the svm weights as sensitivity
(RFE(sens_ana,
pmeasure,
Splitter('train'),
fselector=FixedNElementTailSelector(1),
train_pmeasure=False), self.get_data()),
# use cross-validation within training to get error for the stopping point
# but use full training data to derive sensitivity
(
RFE(
sens_ana,
cvmeasure,
Repeater(
2
), # give the same full dataset to sens_ana and cvmeasure
fselector=FractionTailSelector(0.70,
mode='select',
tail='upper'),
train_pmeasure=True),
normal_feature_dataset(perlabel=20,
nchunks=5,
nfeatures=200,
nonbogus_features=[0, 1],
snr=1.5)),
# use cross-validation (via SplitClassifier) and get mean
# of normed sensitivities across those splits
(
RFE(
rfesvm_split.get_sensitivity_analyzer(
postproc=ChainMapper([
FxMapper('features', l2_normed),
FxMapper('samples', np.mean),
FxMapper('samples', np.abs)
])),
ConfusionBasedError(rfesvm_split, confusion_state='stats'),
Repeater(
2), # we will use the same full cv-training dataset
fselector=FractionTailSelector(0.50,
mode='select',
tail='upper'),
stopping_criterion=NBackHistoryStopCrit(
BestDetector(), 10),
train_pmeasure=
False, # we just extract it from existing confusion
update_sensitivity=True),
normal_feature_dataset(perlabel=28,
nchunks=7,
nfeatures=200,
nonbogus_features=[0, 1],
snr=1.5))
]:
# prep data
# data = datasets['uni2medium']
data_nfeatures = data.nfeatures
rfe.train(data)
resds = rfe(data)
# fail if orig datasets are changed
self.assertTrue(data.nfeatures == data_nfeatures)
# check that the features set with the least error is selected
if len(rfe.ca.errors):
e = np.array(rfe.ca.errors)
if isinstance(rfe._fselector, FixedNElementTailSelector):
self.assertTrue(resds.nfeatures == data_nfeatures -
e.argmin())
else:
imin = np.argmin(e)
if 'does_feature_selection' in clf.__tags__:
# if clf is smart it might figure it out right away
assert_array_less(imin, len(e))
else:
# in this case we can even check if we had actual
# going down/up trend... although -- why up???
self.assertTrue(1 < imin < len(e) - 1)
else:
self.assertTrue(resds.nfeatures == data_nfeatures)
# silly check if nfeatures is in decreasing order
nfeatures = np.array(rfe.ca.nfeatures).copy()
nfeatures.sort()
self.assertTrue((nfeatures[::-1] == rfe.ca.nfeatures).all())
# check if history has elements for every step
self.assertTrue(
set(rfe.ca.history) == set(range(len(np.array(
rfe.ca.errors)))))
# Last (the largest number) can be present multiple times even
# if we remove 1 feature at a time -- just need to stop well
# in advance when we have more than 1 feature left ;)
self.assertTrue(rfe.ca.nfeatures[-1] == len(
np.where(rfe.ca.history == max(rfe.ca.history))[0]))
def test_feature_selector(self):
"""Test feature selector"""
# remove 10% weekest
selector = FractionTailSelector(0.1)
data = np.array([3.5, 10, 7, 5, -0.4, 0, 0, 2, 10, 9])
# == rank [4, 5, 6, 7, 0, 3, 2, 9, 1, 8]
target10 = np.array([0, 1, 2, 3, 5, 6, 7, 8, 9])
target30 = np.array([0, 1, 2, 3, 7, 8, 9])
self.assertRaises(UnknownStateError, selector.ca.__getattribute__,
'ndiscarded')
self.assertTrue((selector(data) == target10).all())
selector.felements = 0.30 # discard 30%
self.assertTrue(selector.felements == 0.3)
self.assertTrue((selector(data) == target30).all())
self.assertTrue(selector.ca.ndiscarded == 3) # se 3 were discarded
selector = FixedNElementTailSelector(1)
# 0 1 2 3 4 5 6 7 8 9
data = np.array([3.5, 10, 7, 5, -0.4, 0, 0, 2, 10, 9])
self.assertTrue((selector(data) == target10).all())
selector.nelements = 3
self.assertTrue(selector.nelements == 3)
self.assertTrue((selector(data) == target30).all())
self.assertTrue(selector.ca.ndiscarded == 3)
# test range selector
# simple range 'above'
self.assertTrue((RangeElementSelector(lower=0)(data) == \
np.array([0,1,2,3,7,8,9])).all())
self.assertTrue((RangeElementSelector(lower=0,
inclusive=True)(data) == \
np.array([0,1,2,3,5,6,7,8,9])).all())
self.assertTrue((RangeElementSelector(lower=0, mode='discard',
inclusive=True)(data) == \
np.array([4])).all())
# simple range 'below'
self.assertTrue((RangeElementSelector(upper=2)(data) == \
np.array([4,5,6])).all())
self.assertTrue((RangeElementSelector(upper=2,
inclusive=True)(data) == \
np.array([4,5,6,7])).all())
self.assertTrue((RangeElementSelector(upper=2, mode='discard',
inclusive=True)(data) == \
np.array([0,1,2,3,8,9])).all())
# ranges
self.assertTrue((RangeElementSelector(lower=2, upper=9)(data) == \
np.array([0,2,3])).all())
self.assertTrue((RangeElementSelector(lower=2, upper=9,
inclusive=True)(data) == \
np.array([0,2,3,7,9])).all())
self.assertTrue((RangeElementSelector(
upper=2, lower=9, mode='discard',
inclusive=True)(data) == RangeElementSelector(
lower=2, upper=9, inclusive=False)(data)).all())
# non-0 elements -- should be equivalent to np.nonzero()[0]
self.assertTrue((RangeElementSelector()(data) == \
np.nonzero(data)[0]).all())
clfswh += \
FeatureSelectionClassifier(
kNN(),
SensitivityBasedFeatureSelection(
SMLRWeights(SMLR(lm=1.0, implementation="C"),
postproc=maxofabs_sample()),
RangeElementSelector(mode='select')),
descr="kNN on SMLR(lm=1) non-0")
clfswh += \
FeatureSelectionClassifier(
kNN(),
SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.05, mode='select', tail='upper')),
descr="kNN on 5%(ANOVA)")
clfswh += \
FeatureSelectionClassifier(
kNN(),
SensitivityBasedFeatureSelection(
OneWayAnova(),
FixedNElementTailSelector(50, mode='select', tail='upper')),
descr="kNN on 50(ANOVA)")
# GNB
clfswh += GNB(descr="GNB()")
clfswh += GNB(common_variance=True, descr="GNB(common_variance=True)")
clfswh += GNB(prior='uniform', descr="GNB(prior='uniform')")
clfswh += \
def test_SplitRFE(self, fmeasure):
# just a smoke test ATM
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import MappedClassifier
from mvpa2.misc.data_generators import normal_feature_dataset
#import mvpa2.featsel.rfe
#reload(mvpa2.featsel.rfe)
from mvpa2.featsel.rfe import RFE, SplitRFE
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.featsel.helpers import FractionTailSelector
from mvpa2.testing import ok_, assert_equal
clf = LinearCSVMC(C=1)
dataset = normal_feature_dataset(perlabel=20,
nlabels=2,
nfeatures=11,
snr=1.,
nonbogus_features=[1, 5])
# flip one of the meaningful features around to see
# if we are still getting proper selection
dataset.samples[:, dataset.a.nonbogus_features[1]] *= -1
# 3 partitions should be enough for testing
partitioner = NFoldPartitioner(count=3)
rfeclf = MappedClassifier(
clf,
SplitRFE(
clf,
partitioner,
fselector=FractionTailSelector(0.5,
mode='discard',
tail='lower'),
fmeasure=fmeasure,
# need to update only when using clf's sens anal
update_sensitivity=fmeasure is None))
r0 = repr(rfeclf)
ok_(rfeclf.mapper.nfeatures_min == 0)
rfeclf.train(dataset)
ok_(rfeclf.mapper.nfeatures_min > 0)
predictions = rfeclf(dataset).samples
# at least 1 of the nonbogus-features should be chosen
ok_(
len(
set(dataset.a.nonbogus_features).intersection(
rfeclf.mapper.slicearg)) > 0)
# check repr to have all needed pieces
r = repr(rfeclf)
s = str(rfeclf)
ok_(('partitioner=NFoldP' in r) or
('partitioner=mvpa2.generators.partition.NFoldPartitioner' in r))
ok_('lrn=' in r)
ok_(not 'slicearg=' in r)
assert_equal(r, r0)
if externals.exists('joblib'):
rfeclf.mapper.nproc = -1
# compare results against the one ran in parallel
_slicearg = rfeclf.mapper.slicearg
_predictions = predictions
rfeclf.train(dataset)
predictions = rfeclf(dataset).samples
assert_array_equal(predictions, _predictions)
assert_array_equal(_slicearg, rfeclf.mapper.slicearg)
# Test that we can collect stats from cas within cross-validation
sensitivities = []
nested_errors = []
nested_nfeatures = []
def store_me(data, node, result):
sens = node.measure.get_sensitivity_analyzer(
force_train=False)(data)
sensitivities.append(sens)
nested_errors.append(node.measure.mapper.ca.nested_errors)
nested_nfeatures.append(node.measure.mapper.ca.nested_nfeatures)
cv = CrossValidation(rfeclf,
NFoldPartitioner(count=1),
callback=store_me,
enable_ca=['stats'])
_ = cv(dataset)
# just to make sure we collected them
assert_equal(len(sensitivities), 1)
assert_equal(len(nested_errors), 1)
assert_equal(len(nested_nfeatures), 1)
def test_rfe_sensmap():
# http://lists.alioth.debian.org/pipermail/pkg-exppsy-pymvpa/2013q3/002538.html
# just a smoke test. fails with
from mvpa2.clfs.svm import LinearCSVMC
from mvpa2.clfs.meta import FeatureSelectionClassifier
from mvpa2.measures.base import CrossValidation, RepeatedMeasure
from mvpa2.generators.splitters import Splitter
from mvpa2.generators.partition import NFoldPartitioner
from mvpa2.misc.errorfx import mean_mismatch_error
from mvpa2.mappers.fx import mean_sample
from mvpa2.mappers.fx import maxofabs_sample
from mvpa2.generators.base import Repeater
from mvpa2.featsel.rfe import RFE
from mvpa2.featsel.helpers import FractionTailSelector, BestDetector
from mvpa2.featsel.helpers import NBackHistoryStopCrit
from mvpa2.datasets import vstack
from mvpa2.misc.data_generators import normal_feature_dataset
# Let's simulate the beast -- 6 categories total groupped into 3
# super-ordinate, and actually without any 'superordinate' effect
# since subordinate categories independent
fds = normal_feature_dataset(nlabels=3,
snr=1, # 100, # pure signal! ;)
perlabel=9,
nfeatures=6,
nonbogus_features=range(3),
nchunks=3)
clfsvm = LinearCSVMC()
rfesvm = RFE(clfsvm.get_sensitivity_analyzer(postproc=maxofabs_sample()),
CrossValidation(
clfsvm,
NFoldPartitioner(),
errorfx=mean_mismatch_error, postproc=mean_sample()),
Repeater(2),
fselector=FractionTailSelector(0.70, mode='select', tail='upper'),
stopping_criterion=NBackHistoryStopCrit(BestDetector(), 10),
update_sensitivity=True)
fclfsvm = FeatureSelectionClassifier(clfsvm, rfesvm)
sensanasvm = fclfsvm.get_sensitivity_analyzer(postproc=maxofabs_sample())
# manually repeating/splitting so we do both RFE sensitivity and classification
senses, errors = [], []
for i, pset in enumerate(NFoldPartitioner().generate(fds)):
# split partitioned dataset
split = [d for d in Splitter('partitions').generate(pset)]
senses.append(sensanasvm(split[0])) # and it also should train the classifier so we would ask it about error
errors.append(mean_mismatch_error(fclfsvm.predict(split[1]), split[1].targets))
senses = vstack(senses)
errors = vstack(errors)
# Let's compare against rerunning the beast simply for classification with CV
errors_cv = CrossValidation(fclfsvm, NFoldPartitioner(), errorfx=mean_mismatch_error)(fds)
# and they should match
assert_array_equal(errors, errors_cv)
# buggy!
cv_sensana_svm = RepeatedMeasure(sensanasvm, NFoldPartitioner())
senses_rm = cv_sensana_svm(fds)
#print senses.samples, senses_rm.samples
#print errors, errors_cv.samples
assert_raises(AssertionError,
assert_array_almost_equal,
senses.samples, senses_rm.samples)
raise SkipTest("Known failure for repeated measures: https://github.com/PyMVPA/PyMVPA/issues/117")
def test_feature_selector(self):
"""Test feature selector"""
# remove 10% weekest
selector = FractionTailSelector(0.1)
data = np.array([3.5, 10, 7, 5, -0.4, 0, 0, 2, 10, 9])
# == rank [4, 5, 6, 7, 0, 3, 2, 9, 1, 8]
target10 = np.array([0, 1, 2, 3, 5, 6, 7, 8, 9])
target30 = np.array([0, 1, 2, 3, 7, 8, 9])
self.assertRaises(UnknownStateError,
selector.ca.__getattribute__, 'ndiscarded')
self.assertTrue((selector(data) == target10).all())
selector.felements = 0.30 # discard 30%
self.assertTrue(selector.felements == 0.3)
self.assertTrue((selector(data) == target30).all())
self.assertTrue(selector.ca.ndiscarded == 3) # se 3 were discarded
selector = FixedNElementTailSelector(1)
# 0 1 2 3 4 5 6 7 8 9
data = np.array([3.5, 10, 7, 5, -0.4, 0, 0, 2, 10, 9])
self.assertTrue((selector(data) == target10).all())
selector.nelements = 3
self.assertTrue(selector.nelements == 3)
self.assertTrue((selector(data) == target30).all())
self.assertTrue(selector.ca.ndiscarded == 3)
# test range selector
# simple range 'above'
self.assertTrue((RangeElementSelector(lower=0)(data) == \
np.array([0,1,2,3,7,8,9])).all())
self.assertTrue((RangeElementSelector(lower=0,
inclusive=True)(data) == \
np.array([0,1,2,3,5,6,7,8,9])).all())
self.assertTrue((RangeElementSelector(lower=0, mode='discard',
inclusive=True)(data) == \
np.array([4])).all())
# simple range 'below'
self.assertTrue((RangeElementSelector(upper=2)(data) == \
np.array([4,5,6])).all())
self.assertTrue((RangeElementSelector(upper=2,
inclusive=True)(data) == \
np.array([4,5,6,7])).all())
self.assertTrue((RangeElementSelector(upper=2, mode='discard',
inclusive=True)(data) == \
np.array([0,1,2,3,8,9])).all())
# ranges
self.assertTrue((RangeElementSelector(lower=2, upper=9)(data) == \
np.array([0,2,3])).all())
self.assertTrue((RangeElementSelector(lower=2, upper=9,
inclusive=True)(data) == \
np.array([0,2,3,7,9])).all())
self.assertTrue((RangeElementSelector(upper=2, lower=9, mode='discard',
inclusive=True)(data) ==
RangeElementSelector(lower=2, upper=9,
inclusive=False)(data)).all())
# non-0 elements -- should be equivalent to np.nonzero()[0]
self.assertTrue((RangeElementSelector()(data) == \
np.nonzero(data)[0]).all())
def custom_tail_selector(seq):
seq1 = FractionTailSelector(0.01, mode='discard',
tail='upper')(seq)
seq2 = FractionTailSelector(0.05, mode='select', tail='upper')(seq)
return list(set(seq1).intersection(seq2))
clfswh += \
FeatureSelectionClassifier(
kNN(),
SensitivityBasedFeatureSelection(
SMLRWeights(SMLR(lm=1.0, implementation="C"),
postproc=maxofabs_sample()),
RangeElementSelector(mode='select')),
descr="kNN on SMLR(lm=1) non-0")
clfswh += \
FeatureSelectionClassifier(
kNN(),
SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.05, mode='select', tail='upper')),
descr="kNN on 5%(ANOVA)")
clfswh += \
FeatureSelectionClassifier(
kNN(),
SensitivityBasedFeatureSelection(
OneWayAnova(),
FixedNElementTailSelector(50, mode='select', tail='upper')),
descr="kNN on 50(ANOVA)")
# GNB
clfswh += GNB(descr="GNB()")
clfswh += GNB(common_variance=True, descr="GNB(common_variance=True)")
clfswh += GNB(prior='uniform', descr="GNB(prior='uniform')")
clfswh += \
