def test_mapped_classifier_sensitivity_analyzer(self, clf):
"""Test sensitivity of the mapped classifier
"""
# Assuming many defaults it is as simple as
mclf = FeatureSelectionClassifier(
clf,
SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.5, mode='select', tail='upper')),
enable_ca=['training_stats'])
sana = mclf.get_sensitivity_analyzer(postproc=sumofabs_sample(),
enable_ca=["sensitivities"])
# and lets look at all sensitivities
dataset = datasets['uni2medium']
# and we get sensitivity analyzer which works on splits
sens = sana(dataset)
self.failUnlessEqual(sens.shape, (1, dataset.nfeatures))
def test_union_feature_selection(self):
# two methods: 5% highes F-scores, non-zero SMLR weights
fss = [SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.05, mode='select', tail='upper')),
SensitivityBasedFeatureSelection(
SMLRWeights(SMLR(lm=1, implementation="C"),
postproc=sumofabs_sample()),
RangeElementSelector(mode='select'))]
fs = CombinedFeatureSelection(fss, method='union')
od_union = fs(self.dataset)
self.failUnless(fs.method == 'union')
# check output dataset
self.failUnless(od_union.nfeatures <= self.dataset.nfeatures)
# again for intersection
fs = CombinedFeatureSelection(fss, method='intersection')
od_intersect = fs(self.dataset)
assert_true(od_intersect.nfeatures < od_union.nfeatures)
def test_mapped_classifier_sensitivity_analyzer(self, clf):
"""Test sensitivity of the mapped classifier
"""
# Assuming many defaults it is as simple as
mclf = FeatureSelectionClassifier(clf,
SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(
0.5,
mode='select',
tail='upper')),
enable_ca=['training_confusion'])
sana = mclf.get_sensitivity_analyzer(postproc=sumofabs_sample(),
enable_ca=["sensitivities"])
# and lets look at all sensitivities
dataset = datasets['uni2medium']
# and we get sensitivity analyzer which works on splits
sens = sana(dataset)
self.failUnlessEqual(sens.shape, (1, dataset.nfeatures))
def test_union_feature_selection(self):
# two methods: 5% highes F-scores, non-zero SMLR weights
fss = [
SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.05, mode='select', tail='upper')),
SensitivityBasedFeatureSelection(
SMLRWeights(SMLR(lm=1, implementation="C"),
postproc=sumofabs_sample()),
RangeElementSelector(mode='select'))
]
fs = CombinedFeatureSelection(
fss,
combiner='union',
enable_ca=['selected_ids', 'selections_ids'])
od = fs(self.dataset)
self.failUnless(fs.combiner == 'union')
self.failUnless(len(fs.ca.selections_ids))
self.failUnless(len(fs.ca.selections_ids) <= self.dataset.nfeatures)
# should store one set per methods
self.failUnless(len(fs.ca.selections_ids) == len(fss))
# no individual can be larger than union
for s in fs.ca.selections_ids:
self.failUnless(len(s) <= len(fs.ca.selected_ids))
# check output dataset
self.failUnless(od.nfeatures == len(fs.ca.selected_ids))
for i, id in enumerate(fs.ca.selected_ids):
self.failUnless(
(od.samples[:, i] == self.dataset.samples[:, id]).all())
# again for intersection
fs = CombinedFeatureSelection(
fss,
combiner='intersection',
enable_ca=['selected_ids', 'selections_ids'])
# simply run it for now -- can't think of additional tests
od = fs(self.dataset)
def test_union_feature_selection(self):
# two methods: 5% highes F-scores, non-zero SMLR weights
fss = [SensitivityBasedFeatureSelection(
OneWayAnova(),
FractionTailSelector(0.05, mode='select', tail='upper')),
SensitivityBasedFeatureSelection(
SMLRWeights(SMLR(lm=1, implementation="C"),
postproc=sumofabs_sample()),
RangeElementSelector(mode='select'))]
fs = CombinedFeatureSelection(fss, combiner='union',
enable_ca=['selected_ids',
'selections_ids'])
od = fs(self.dataset)
self.failUnless(fs.combiner == 'union')
self.failUnless(len(fs.ca.selections_ids))
self.failUnless(len(fs.ca.selections_ids) <= self.dataset.nfeatures)
# should store one set per methods
self.failUnless(len(fs.ca.selections_ids) == len(fss))
# no individual can be larger than union
for s in fs.ca.selections_ids:
self.failUnless(len(s) <= len(fs.ca.selected_ids))
# check output dataset
self.failUnless(od.nfeatures == len(fs.ca.selected_ids))
for i, id in enumerate(fs.ca.selected_ids):
self.failUnless((od.samples[:,i]
== self.dataset.samples[:,id]).all())
# again for intersection
fs = CombinedFeatureSelection(fss, combiner='intersection',
enable_ca=['selected_ids',
'selections_ids'])
# simply run it for now -- can't think of additional tests
od = fs(self.dataset)
from mvpa.generators.partition import NFoldPartitioner
from mvpa.generators.resampling import Balancer
from mvpa.misc.errorfx import mean_mismatch_error
from mvpa.misc.transformers import Absolute, \
DistPValue
from mvpa.measures.base import Measure, \
TransferMeasure, RepeatedMeasure, CrossValidation
from mvpa.measures.anova import OneWayAnova, CompoundOneWayAnova
from mvpa.measures.irelief import IterativeRelief, IterativeReliefOnline, \
IterativeRelief_Devel, IterativeReliefOnline_Devel
_MEASURES_2_SWEEP = [ OneWayAnova(),
CompoundOneWayAnova(postproc=sumofabs_sample()),
IterativeRelief(), IterativeReliefOnline(),
IterativeRelief_Devel(), IterativeReliefOnline_Devel()
]
if externals.exists('scipy'):
from mvpa.measures.corrcoef import CorrCoef
_MEASURES_2_SWEEP += [ CorrCoef(),
# that one is good when small... handle later
#CorrCoef(pvalue=True)
]
class SensitivityAnalysersTests(unittest.TestCase):
def setUp(self):
self.dataset = datasets['uni2large']
from mvpa.mappers.zscore import zscore
from mvpa.mappers.fx import sumofabs_sample, absolute_features, FxMapper, \
maxofabs_sample
from mvpa.datasets.splitters import NFoldSplitter, NoneSplitter
from mvpa.misc.transformers import Absolute, \
DistPValue
from mvpa.measures.base import SplitFeaturewiseDatasetMeasure
from mvpa.measures.anova import OneWayAnova, CompoundOneWayAnova
from mvpa.measures.irelief import IterativeRelief, IterativeReliefOnline, \
IterativeRelief_Devel, IterativeReliefOnline_Devel
_MEASURES_2_SWEEP = [
OneWayAnova(),
CompoundOneWayAnova(postproc=sumofabs_sample()),
IterativeRelief(),
IterativeReliefOnline(),
IterativeRelief_Devel(),
IterativeReliefOnline_Devel()
]
if externals.exists('scipy'):
from mvpa.measures.corrcoef import CorrCoef
_MEASURES_2_SWEEP += [
CorrCoef(),
# that one is good when small... handle later
#CorrCoef(pvalue=True)
]
class SensitivityAnalysersTests(unittest.TestCase):
