def test_james_problem(self):
percent = 80
dataset = datasets['uni2small']
rfesvm_split = LinearCSVMC()
fs = \
RFE(sensitivity_analyzer=rfesvm_split.get_sensitivity_analyzer(),
transfer_error=TransferError(rfesvm_split),
feature_selector=FractionTailSelector(
percent / 100.0,
mode='select', tail='upper'), update_sensitivity=True)
clf = FeatureSelectionClassifier(
clf = LinearCSVMC(),
# on features selected via RFE
feature_selection = fs)
# update sensitivity at each step (since we're not using the
# same CLF as sensitivity analyzer)
clf.ca.enable('feature_ids')
cv = CrossValidatedTransferError(
TransferError(clf),
NFoldSplitter(cvtype=1),
postproc=mean_sample(),
enable_ca=['confusion'],
expose_testdataset=True)
#cv = SplitClassifier(clf)
try:
error = cv(dataset).samples.squeeze()
except Exception, e:
self.fail('CrossValidation cannot handle classifier with RFE '
'feature selection. Got exception: %s' % (e,))
def test_custom_targets(self, lrn):
"""Simple test if a learner could cope with custom sa not targets
"""
# Since we are comparing performances of two learners, we need
# to assure that if they depend on some random seed -- they
# would use the same value. Currently we have such stochastic
# behavior in SMLR
if 'seed' in lrn.params:
from mvpa import _random_seed
lrn = lrn.clone() # clone the beast
lrn.params.seed = _random_seed # reuse the same seed
lrn_ = lrn.clone()
lrn_.params.targets_attr = 'custom'
te = CrossValidatedTransferError(TransferError(lrn), NFoldSplitter())
te_ = CrossValidatedTransferError(TransferError(lrn_), NFoldSplitter())
nclasses = 2 * (1 + int('multiclass' in lrn.__tags__))
dsname = ('uni%dsmall' % nclasses,
'sin_modulated')[int(lrn.__is_regression__)]
ds = datasets[dsname]
ds_ = ds.copy()
ds_.sa['custom'] = ds_.sa['targets']
ds_.sa.pop('targets')
self.failUnless('targets' in ds.sa,
msg="'targets' should remain in original ds")
try:
cve = te(ds)
cve_ = te_(ds_)
except Exception, e:
self.fail("Failed with %r" % e)
def test_chi_square_searchlight(self):
# only do partial to save time
# Can't yet do this since test_searchlight isn't yet "under nose"
#skip_if_no_external('scipy')
if not externals.exists('scipy'):
return
from mvpa.misc.stats import chisquare
transerror = TransferError(sample_clf_lin)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1),
enable_ca=['confusion'])
def getconfusion(data):
cv(data)
return chisquare(cv.ca.confusion.matrix)[0]
sl = sphere_searchlight(getconfusion, radius=0,
center_ids=[3,50])
# run searchlight
results = sl(self.dataset)
self.failUnless(results.nfeatures == 2)
def __test_matthias_question(self):
rfe_clf = LinearCSVMC(C=1)
rfesvm_split = SplitClassifier(rfe_clf)
clf = \
FeatureSelectionClassifier(
clf = LinearCSVMC(C=1),
feature_selection = RFE(
sensitivity_analyzer = rfesvm_split.get_sensitivity_analyzer(
combiner=first_axis_mean,
transformer=np.abs),
transfer_error=ConfusionBasedError(
rfesvm_split,
confusion_state="confusion"),
stopping_criterion=FixedErrorThresholdStopCrit(0.20),
feature_selector=FractionTailSelector(
0.2, mode='discard', tail='lower'),
update_sensitivity=True))
splitter = NFoldSplitter(cvtype=1)
no_permutations = 1000
cv = CrossValidatedTransferError(
TransferError(clf),
splitter,
null_dist=MCNullDist(permutations=no_permutations,
tail='left'),
enable_ca=['confusion'])
error = cv(datasets['uni2small'])
self.failUnless(error < 0.4)
self.failUnless(cv.ca.null_prob < 0.05)
def test_regression_as_classifier(self, regr):
"""Basic tests of metaclass for using regressions as classifiers
"""
for dsname in 'uni2small', 'uni4small':
ds = datasets[dsname]
clf = RegressionAsClassifier(regr, enable_ca=['distances'])
cv = CrossValidatedTransferError(
TransferError(clf),
OddEvenSplitter(),
postproc=mean_sample(),
enable_ca=['confusion', 'training_confusion'])
error = cv(ds).samples.squeeze()
nlabels = len(ds.uniquetargets)
if nlabels == 2 \
and cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(error < 0.3)
# Check if does not puke on repr and str
self.failUnless(str(clf) != "")
self.failUnless(repr(clf) != "")
self.failUnlessEqual(clf.ca.distances.shape,
(ds.nsamples / 2, nlabels))
def test_split_classifier_extended(self, clf_):
clf2 = clf_.clone()
ds = datasets['uni2medium'] #self.data_bin_1
clf = SplitClassifier(
clf=clf_, #SameSignClassifier(),
splitter=NFoldSplitter(1),
enable_ca=['confusion', 'feature_ids'])
clf.train(ds) # train the beast
error = clf.ca.confusion.error
cv = CrossValidatedTransferError(
TransferError(clf2),
NFoldSplitter(),
postproc=mean_sample(),
enable_ca=['confusion', 'training_confusion'])
cverror = cv(ds).samples.squeeze()
self.failUnless(
abs(error - cverror) < 0.01,
msg="We should get the same error using split classifier as"
" using CrossValidatedTransferError. Got %s and %s" %
(error, cverror))
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(error < 0.25,
msg="clf should generalize more or less fine. "
"Got error %s" % error)
self.failUnlessEqual(len(clf.ca.confusion.sets),
len(ds.UC),
msg="Should have 1 confusion per each split")
self.failUnlessEqual(
len(clf.clfs),
len(ds.UC),
msg="Should have number of classifiers equal # of epochs")
def test_split_classifier(self):
ds = self.data_bin_1
clf = SplitClassifier(
clf=SameSignClassifier(),
splitter=NFoldSplitter(1),
enable_ca=['confusion', 'training_confusion', 'feature_ids'])
clf.train(ds) # train the beast
error = clf.ca.confusion.error
tr_error = clf.ca.training_confusion.error
clf2 = clf.clone()
cv = CrossValidatedTransferError(
TransferError(clf2),
NFoldSplitter(),
postproc=mean_sample(),
enable_ca=['confusion', 'training_confusion'])
cverror = cv(ds).samples.squeeze()
tr_cverror = cv.ca.training_confusion.error
self.failUnlessEqual(
error,
cverror,
msg="We should get the same error using split classifier as"
" using CrossValidatedTransferError. Got %s and %s" %
(error, cverror))
self.failUnlessEqual(
tr_error,
tr_cverror,
msg="We should get the same training error using split classifier as"
" using CrossValidatedTransferError. Got %s and %s" %
(tr_error, tr_cverror))
self.failUnlessEqual(clf.ca.confusion.percent_correct,
100,
msg="Dummy clf should train perfectly")
self.failUnlessEqual(len(clf.ca.confusion.sets),
len(ds.UC),
msg="Should have 1 confusion per each split")
self.failUnlessEqual(
len(clf.clfs),
len(ds.UC),
msg="Should have number of classifiers equal # of epochs")
self.failUnlessEqual(clf.predict(ds.samples),
list(ds.targets),
msg="Should classify correctly")
# feature_ids must be list of lists, and since it is not
# feature-selecting classifier used - we expect all features
# to be utilized
# NOT ANYMORE -- for BoostedClassifier we have now union of all
# used features across slave classifiers. That makes
# semantics clear. If you need to get deeper -- use upcoming
# harvesting facility ;-)
# self.failUnlessEqual(len(clf.feature_ids), len(ds.uniquechunks))
# self.failUnless(np.array([len(ids)==ds.nfeatures
# for ids in clf.feature_ids]).all())
# Just check if we get it at all ;-)
summary = clf.summary()
def test_simple_n_minus_one_cv(self):
data = get_mv_pattern(3)
data.init_origids('samples')
self.failUnless(data.nsamples == 120)
self.failUnless(data.nfeatures == 2)
self.failUnless(
(data.sa.targets == \
[0,0,0,0,0,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0] * 6).all())
self.failUnless(
(data.sa.chunks == \
[k for k in range(1, 7) for i in range(20)]).all())
assert_equal(len(np.unique(data.sa.origids)), data.nsamples)
transerror = TransferError(sample_clf_nl)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1),
enable_ca=['confusion', 'training_confusion', 'samples_error'])
results = cv(data)
self.failUnless((results.samples < 0.2).all()
and (results.samples >= 0.0).all())
# TODO: test accessibility of {training_,}confusion{,s} of
# CrossValidatedTransferError
self.failUnless(isinstance(cv.ca.samples_error, dict))
self.failUnless(len(cv.ca.samples_error) == data.nsamples)
# one value for each origid
assert_array_equal(sorted(cv.ca.samples_error.keys()),
sorted(data.sa.origids))
for k, v in cv.ca.samples_error.iteritems():
self.failUnless(len(v) == 1)
def test_tree_classifier(self):
"""Basic tests for TreeClassifier
"""
ds = datasets['uni4small']
clfs = clfswh['binary'] # pool of classifiers
# Lets permute so each time we try some different combination
# of the classifiers
clfs = [clfs[i] for i in np.random.permutation(len(clfs))]
# Test conflicting definition
tclf = TreeClassifier(clfs[0], {
'L0+2': (('L0', 'L2'), clfs[1]),
'L2+3': (('L2', 'L3'), clfs[2])
})
self.failUnlessRaises(ValueError, tclf.train, ds)
"""Should raise exception since label 2 is in both"""
# Test insufficient definition
tclf = TreeClassifier(clfs[0], {
'L0+5': (('L0', 'L5'), clfs[1]),
'L2+3': (('L2', 'L3'), clfs[2])
})
self.failUnlessRaises(ValueError, tclf.train, ds)
"""Should raise exception since no group for L1"""
# proper definition now
tclf = TreeClassifier(clfs[0], {
'L0+1': (('L0', 'L1'), clfs[1]),
'L2+3': (('L2', 'L3'), clfs[2])
})
# Lets test train/test cycle using CVTE
cv = CrossValidatedTransferError(
TransferError(tclf),
OddEvenSplitter(),
postproc=mean_sample(),
enable_ca=['confusion', 'training_confusion'])
cverror = cv(ds).samples.squeeze()
try:
rtclf = repr(tclf)
except:
self.fail(msg="Could not obtain repr for TreeClassifier")
# Test accessibility of .clfs
self.failUnless(tclf.clfs['L0+1'] is clfs[1])
self.failUnless(tclf.clfs['L2+3'] is clfs[2])
cvtrc = cv.ca.training_confusion
cvtc = cv.ca.confusion
if cfg.getboolean('tests', 'labile', default='yes'):
# just a dummy check to make sure everything is working
self.failUnless(cvtrc != cvtc)
self.failUnless(cverror < 0.3)
# TODO: whenever implemented
tclf = TreeClassifier(clfs[0], {
'L0': (('L0', ), clfs[1]),
'L1+2+3': (('L1', 'L2', 'L3'), clfs[2])
})
def test_cache_speedup(self):
skip_if_no_external('shogun', ver_dep='shogun:rev', min_version=4455)
ck = sgSVM(kernel=CachedKernel(kernel=RbfSGKernel(sigma=2)), C=1)
sk = sgSVM(kernel=RbfSGKernel(sigma=2), C=1)
cv_c = CrossValidatedTransferError(TransferError(ck),
splitter=NFoldSplitter())
cv_s = CrossValidatedTransferError(TransferError(sk),
splitter=NFoldSplitter())
#data = datasets['uni4large']
P = 5000
data = normal_feature_dataset(snr=2,
perlabel=200,
nchunks=10,
means=np.random.randn(2, P),
nfeatures=P)
t0 = time()
ck.params.kernel.compute(data)
cachetime = time() - t0
t0 = time()
cached_err = cv_c(data)
ccv_time = time() - t0
t0 = time()
norm_err = cv_s(data)
ncv_time = time() - t0
assert_almost_equal(np.asanyarray(cached_err), np.asanyarray(norm_err))
ok_(cachetime < ncv_time)
ok_(ccv_time < ncv_time)
#print 'Regular CV time: %s seconds'%ncv_time
#print 'Caching time: %s seconds'%cachetime
#print 'Cached CV time: %s seconds'%ccv_time
speedup = ncv_time / (ccv_time + cachetime)
#print 'Speedup factor: %s'%speedup
# Speedup ideally should be 10, though it's not purely linear
self.failIf(speedup < 2, 'Problem caching data - too slow!')
def test_cached_kernel_different_datasets(self):
skip_if_no_external('shogun', ver_dep='shogun:rev', min_version=4455)
# Inspired by the problem Swaroop ran into
k = LinearSGKernel(normalizer_cls=False)
k_ = LinearSGKernel(normalizer_cls=False) # to be cached
ck = CachedKernel(k_)
clf = sgSVM(svm_impl='libsvm', kernel=k, C=-1)
clf_ = sgSVM(svm_impl='libsvm', kernel=ck, C=-1)
cvte = CrossValidatedTransferError(TransferError(clf), NFoldSplitter())
cvte_ = CrossValidatedTransferError(TransferError(clf_),
NFoldSplitter())
te = TransferError(clf)
te_ = TransferError(clf_)
for r in xrange(2):
ds1 = datasets['uni2medium']
errs1 = cvte(ds1)
ck.compute(ds1)
ok_(ck._recomputed)
errs1_ = cvte_(ds1)
ok_(~ck._recomputed)
assert_array_equal(errs1, errs1_)
ds2 = datasets['uni3small']
errs2 = cvte(ds2)
ck.compute(ds2)
ok_(ck._recomputed)
errs2_ = cvte_(ds2)
ok_(~ck._recomputed)
assert_array_equal(errs2, errs2_)
ssel = np.round(datasets['uni2large'].samples[:5, 0]).astype(int)
terr = te(datasets['uni3small_test'][ssel],
datasets['uni3small_train'][::2])
terr_ = te_(datasets['uni3small_test'][ssel],
datasets['uni3small_train'][::2])
ok_(~ck._recomputed)
ok_(terr == terr_)
def test_null_dist_prob(self, l_clf):
train = datasets['uni2medium']
num_perm = 10
# define class to estimate NULL distribution of errors
# use left tail of the distribution since we use MeanMatchFx as error
# function and lower is better
terr = TransferError(clf=l_clf,
null_dist=MCNullDist(permutations=num_perm,
tail='left'))
# check reasonable error range
err = terr(train, train)
self.failUnless(err < 0.4)
# Lets do the same for CVTE
cvte = CrossValidatedTransferError(TransferError(clf=l_clf),
OddEvenSplitter(),
null_dist=MCNullDist(
permutations=num_perm,
tail='left',
enable_ca=['dist_samples']),
postproc=mean_sample())
cv_err = cvte(train)
# check that the result is highly significant since we know that the
# data has signal
null_prob = terr.ca.null_prob
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(
null_prob <= 0.1,
msg="Failed to check that the result is highly significant "
"(got %f) since we know that the data has signal" % null_prob)
self.failUnless(
cvte.ca.null_prob <= 0.1,
msg="Failed to check that the result is highly significant "
"(got p(cvte)=%f) since we know that the data has signal" %
cvte.ca.null_prob)
# and we should be able to access the actual samples of the distribution
self.failUnlessEqual(len(cvte.null_dist.ca.dist_samples), num_perm)
def test_harvesting(self):
# get a dataset with a very high SNR
data = get_mv_pattern(10)
# do crossval with default errorfx and 'mean' combiner
transerror = TransferError(clfswh['linear'][0])
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1),
harvest_attribs=['transerror.clf.ca.training_time'])
result = cv(data)
ok_(cv.ca.harvested.has_key('transerror.clf.ca.training_time'))
assert_equal(len(cv.ca.harvested['transerror.clf.ca.training_time']),
len(data.UC))
def test_per_sample_error(self, l_clf):
train = datasets['uni2medium']
train.init_origids('samples')
terr = TransferError(clf=l_clf, enable_ca=['samples_error'])
err = terr(train, train)
se = terr.ca.samples_error
# one error per sample
self.failUnless(len(se) == train.nsamples)
# for this simple test it can only be correct or misclassified
# (boolean)
self.failUnless(
np.sum(np.array(se.values(), dtype='float') \
- np.array(se.values(), dtype='b')) == 0)
def test_vstack_and_origids_issue(self):
# That is actually what swaroop hit
skip_if_no_external('shogun', ver_dep='shogun:rev', min_version=4455)
# Inspired by the problem Swaroop ran into
k = LinearSGKernel(normalizer_cls=False)
k_ = LinearSGKernel(normalizer_cls=False) # to be cached
ck = CachedKernel(k_)
clf = sgSVM(svm_impl='libsvm', kernel=k, C=-1)
clf_ = sgSVM(svm_impl='libsvm', kernel=ck, C=-1)
cvte = CrossValidatedTransferError(TransferError(clf), NFoldSplitter())
cvte_ = CrossValidatedTransferError(TransferError(clf_),
NFoldSplitter())
ds = datasets['uni2large_test'].copy(deep=True)
ok_(~('orig_ids' in ds.sa)) # assure that there are None
ck.compute(ds) # so we initialize origids
ok_('origids' in ds.sa)
ds2 = ds.copy(deep=True)
ds2.samples = np.zeros(ds2.shape)
from mvpa.base.dataset import vstack
ds_vstacked = vstack((ds2, ds))
# should complaint now since there would not be unique
# samples' origids
if __debug__:
assert_raises(ValueError, ck.compute, ds_vstacked)
ds_vstacked.init_origids('samples') # reset origids
ck.compute(ds_vstacked)
errs = cvte(ds_vstacked)
errs_ = cvte_(ds_vstacked)
# Following test would have failed since origids
# were just ints, and then non-unique after vstack
assert_array_equal(errs.samples, errs_.samples)
def test_classifier_generalization(self, clf):
"""Simple test if classifiers can generalize ok on simple data
"""
te = CrossValidatedTransferError(TransferError(clf),
NFoldSplitter(),
postproc=mean_sample())
# check the default
self.failUnless(isinstance(te.transerror.errorfx, MeanMismatchErrorFx))
nclasses = 2 * (1 + int('multiclass' in clf.__tags__))
ds = datasets['uni%dmedium' % nclasses]
try:
cve = te(ds).samples.squeeze()
except Exception, e:
self.fail("Failed with %s" % e)
def test_values(self, clf):
if isinstance(clf, MulticlassClassifier):
# TODO: handle those values correctly
return
ds = datasets['uni2small']
clf.ca.change_temporarily(enable_ca=['estimates'])
cv = CrossValidatedTransferError(
TransferError(clf),
OddEvenSplitter(),
enable_ca=['confusion', 'training_confusion'])
_ = cv(ds)
#print clf.descr, clf.values[0]
# basic test either we get 1 set of values per each sample
self.failUnlessEqual(len(clf.ca.estimates), ds.nsamples / 2)
clf.ca.reset_changed_temporarily()
def test_rfe(self, clf):
# sensitivity analyser and transfer error quantifier use the SAME clf!
sens_ana = clf.get_sensitivity_analyzer(postproc=maxofabs_sample())
trans_error = TransferError(clf)
# because the clf is already trained when computing the sensitivity
# map, prevent retraining for transfer error calculation
# Use absolute of the svm weights as sensitivity
rfe = RFE(sens_ana,
trans_error,
feature_selector=FixedNElementTailSelector(1),
train_clf=False)
wdata = self.get_data()
wdata_nfeatures = wdata.nfeatures
tdata = self.get_data_t()
tdata_nfeatures = tdata.nfeatures
sdata, stdata = rfe(wdata, tdata)
# fail if orig datasets are changed
self.failUnless(wdata.nfeatures == wdata_nfeatures)
self.failUnless(tdata.nfeatures == tdata_nfeatures)
# check that the features set with the least error is selected
if len(rfe.ca.errors):
e = np.array(rfe.ca.errors)
self.failUnless(sdata.nfeatures == wdata_nfeatures - e.argmin())
else:
self.failUnless(sdata.nfeatures == wdata_nfeatures)
# silly check if nfeatures is in decreasing order
nfeatures = np.array(rfe.ca.nfeatures).copy()
nfeatures.sort()
self.failUnless( (nfeatures[::-1] == rfe.ca.nfeatures).all() )
# check if history has elements for every step
self.failUnless(set(rfe.ca.history)
== set(range(len(np.array(rfe.ca.errors)))))
# Last (the largest number) can be present multiple times even
# if we remove 1 feature at a time -- just need to stop well
# in advance when we have more than 1 feature left ;)
self.failUnless(rfe.ca.nfeatures[-1]
== len(np.where(rfe.ca.history
==max(rfe.ca.history))[0]))
def test_regressions_classifiers(self, clf):
"""Simple tests on regressions being used as classifiers
"""
# check if we get values set correctly
clf.ca.change_temporarily(enable_ca=['estimates'])
self.failUnlessRaises(UnknownStateError, clf.ca['estimates']._get)
cv = CrossValidatedTransferError(
TransferError(clf),
NFoldSplitter(),
enable_ca=['confusion', 'training_confusion'])
ds = datasets['uni2small'].copy()
# we want numeric labels to maintain the previous behavior, especially
# since we deal with regressions here
ds.sa.targets = AttributeMap().to_numeric(ds.targets)
cverror = cv(ds)
self.failUnless(len(clf.ca.estimates) == ds[ds.chunks == 1].nsamples)
clf.ca.reset_changed_temporarily()
def test_partial_searchlight_with_full_report(self):
# compute N-1 cross-validation for each sphere
transerror = TransferError(sample_clf_lin)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1))
# contruct diameter 1 (or just radius 0) searchlight
sl = sphere_searchlight(cv, radius=0,
center_ids=[3,50])
# run searchlight
results = sl(self.dataset)
# only two spheres but error for all CV-folds
self.failUnlessEqual(results.shape, (len(self.dataset.UC), 2))
# test if we graciously puke if center_ids are out of bounds
dataset0 = self.dataset[:, :50] # so we have no 50th feature
self.failUnlessRaises(IndexError, sl, dataset0)
def test_noise_classification(self):
# get a dataset with a very high SNR
data = get_mv_pattern(10)
# do crossval with default errorfx and 'mean' combiner
transerror = TransferError(sample_clf_nl)
cv = CrossValidatedTransferError(transerror, NFoldSplitter(cvtype=1))
# must return a scalar value
result = cv(data)
# must be perfect
self.failUnless((result.samples < 0.05).all())
# do crossval with permuted regressors
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1, permute_attr='targets', nrunspersplit=10))
results = cv(data)
# must be at chance level
pmean = np.array(results).mean()
self.failUnless(pmean < 0.58 and pmean > 0.42)
def test_auc(self, clf):
"""Test AUC computation
"""
if isinstance(clf, MulticlassClassifier):
# TODO: handle those values correctly
return
clf.ca.change_temporarily(enable_ca=['estimates'])
# uni2 dataset with reordered labels
ds2 = datasets['uni2small'].copy()
# revert labels
ds2.sa['targets'].value = ds2.targets[::-1].copy()
# same with uni3
ds3 = datasets['uni3small'].copy()
ul = ds3.sa['targets'].unique
nl = ds3.targets.copy()
for l in xrange(3):
nl[ds3.targets == ul[l]] = ul[(l + 1) % 3]
ds3.sa.targets = nl
for ds in [datasets['uni2small'], ds2, datasets['uni3small'], ds3]:
cv = CrossValidatedTransferError(
TransferError(clf),
OddEvenSplitter(),
enable_ca=['confusion', 'training_confusion'])
cverror = cv(ds)
stats = cv.ca.confusion.stats
Nlabels = len(ds.uniquetargets)
# so we at least do slightly above chance
self.failUnless(stats['ACC'] > 1.2 / Nlabels)
auc = stats['AUC']
if (Nlabels == 2) or (Nlabels > 2 and auc[0] is not np.nan):
mauc = np.min(stats['AUC'])
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(
mauc > 0.55,
msg='All AUCs must be above chance. Got minimal '
'AUC=%.2g among %s' % (mauc, stats['AUC']))
clf.ca.reset_changed_temporarily()
def test_ifs(self, svm):
# data measure and transfer error quantifier use the SAME clf!
trans_error = TransferError(svm)
data_measure = CrossValidatedTransferError(trans_error,
NFoldSplitter(1),
postproc=mean_sample())
ifs = IFS(data_measure,
trans_error,
feature_selector=\
# go for lower tail selection as data_measure will return
# errors -> low is good
FixedNElementTailSelector(1, tail='lower', mode='select'),
)
wdata = self.get_data()
wdata_nfeatures = wdata.nfeatures
tdata = self.get_data()
tdata_nfeatures = tdata.nfeatures
sdata, stdata = ifs(wdata, tdata)
# fail if orig datasets are changed
self.failUnless(wdata.nfeatures == wdata_nfeatures)
self.failUnless(tdata.nfeatures == tdata_nfeatures)
# check that the features set with the least error is selected
self.failUnless(len(ifs.ca.errors))
e = np.array(ifs.ca.errors)
self.failUnless(sdata.nfeatures == e.argmin() + 1)
# repeat with dataset where selection order is known
signal = datasets['dumb2']
sdata, stdata = ifs(signal, signal)
self.failUnless((sdata.samples[:, 0] == signal.samples[:, 0]).all())
def test_confusion_based_error(self, l_clf):
train = datasets['uni2medium_train']
# to check if we fail to classify for 3 labels
test3 = datasets['uni3medium_train']
err = ConfusionBasedError(clf=l_clf)
terr = TransferError(clf=l_clf)
self.failUnlessRaises(UnknownStateError, err, None)
"""Shouldn't be able to access the state yet"""
l_clf.train(train)
e, te = err(None), terr(train)
self.failUnless(
abs(e - te) < 1e-10,
msg="ConfusionBasedError (%.2g) should be equal to TransferError "
"(%.2g) on traindataset" % (e, te))
# this will print nasty WARNING but it is ok -- it is just checking code
# NB warnings are not printed while doing whole testing
warning("Don't worry about the following warning.")
self.failIf(terr(test3) is None)
# try copying the beast
terr_copy = copy(terr)
def test_spatial_searchlight(self, common_variance):
"""Tests both generic and GNBSearchlight
Test of GNBSearchlight anyways requires a ground-truth
comparison to the generic version, so we are doing sweepargs here
"""
# compute N-1 cross-validation for each sphere
# YOH: unfortunately sample_clf_lin is not guaranteed
# to provide exactly the same results due to inherent
# iterative process. Therefore lets use something quick
# and pure Python
gnb = GNB(common_variance=common_variance)
transerror = TransferError(gnb)
cv = CrossValidatedTransferError(
transerror,
NFoldSplitter(cvtype=1))
skwargs = dict(radius=1, enable_ca=['roi_sizes', 'raw_results'])
sls = [sphere_searchlight(cv, **skwargs),
#GNBSearchlight(gnb, NFoldSplitter(cvtype=1))
sphere_gnbsearchlight(gnb, NFoldSplitter(cvtype=1),
indexsum='fancy', **skwargs)
]
if externals.exists('scipy'):
sls += [ sphere_gnbsearchlight(gnb, NFoldSplitter(cvtype=1),
indexsum='sparse', **skwargs)]
# Just test nproc whenever common_variance is True
if externals.exists('pprocess') and common_variance:
sls += [sphere_searchlight(cv, nproc=2, **skwargs)]
all_results = []
ds = datasets['3dsmall'].copy()
ds.fa['voxel_indices'] = ds.fa.myspace
for sl in sls:
# run searchlight
results = sl(ds)
all_results.append(results)
# check for correct number of spheres
self.failUnless(results.nfeatures == 106)
# and measures (one per xfold)
self.failUnless(len(results) == len(ds.UC))
# check for chance-level performance across all spheres
self.failUnless(0.4 < results.samples.mean() < 0.6)
mean_errors = results.samples.mean(axis=0)
# that we do get different errors ;)
self.failUnless(len(np.unique(mean_errors) > 3))
# check resonable sphere sizes
self.failUnless(len(sl.ca.roi_sizes) == 106)
self.failUnless(max(sl.ca.roi_sizes) == 7)
self.failUnless(min(sl.ca.roi_sizes) == 4)
# check base-class state
self.failUnlessEqual(sl.ca.raw_results.nfeatures, 106)
if len(all_results) > 1:
# if we had multiple searchlights, we can check either they all
# gave the same result (they should have)
aresults = np.array([a.samples for a in all_results])
dresults = np.abs(aresults - aresults.mean(axis=0))
dmax = np.max(dresults)
self.failUnless(dmax <= 1e-13)
def test_regressions(self, regr):
"""Simple tests on regressions
"""
ds = datasets['chirp_linear']
# we want numeric labels to maintain the previous behavior, especially
# since we deal with regressions here
ds.sa.targets = AttributeMap().to_numeric(ds.targets)
cve = CrossValidatedTransferError(
TransferError(regr),
splitter=NFoldSplitter(),
postproc=mean_sample(),
enable_ca=['training_confusion', 'confusion'])
# check the default
self.failUnless(isinstance(cve.transerror.errorfx, CorrErrorFx))
corr = np.asscalar(cve(ds).samples)
# Our CorrErrorFx should never return NaN
self.failUnless(not np.isnan(corr))
self.failUnless(corr == cve.ca.confusion.stats['CCe'])
splitregr = SplitClassifier(
regr,
splitter=OddEvenSplitter(),
enable_ca=['training_confusion', 'confusion'])
splitregr.train(ds)
split_corr = splitregr.ca.confusion.stats['CCe']
split_corr_tr = splitregr.ca.training_confusion.stats['CCe']
for confusion, error in (
(cve.ca.confusion, corr),
(splitregr.ca.confusion, split_corr),
(splitregr.ca.training_confusion, split_corr_tr),
):
#TODO: test confusion statistics
# Part of it for now -- CCe
for conf in confusion.summaries:
stats = conf.stats
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(stats['CCe'] < 0.5)
self.failUnlessEqual(stats['CCe'], stats['Summary CCe'])
s0 = confusion.as_string(short=True)
s1 = confusion.as_string(short=False)
for s in [s0, s1]:
self.failUnless(len(s) > 10,
msg="We should get some string representation "
"of regression summary. Got %s" % s)
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(
error < 0.2,
msg="Regressions should perform well on a simple "
"dataset. Got correlation error of %s " % error)
# Test access to summary statistics
# YOH: lets start making testing more reliable.
# p-value for such accident to have is verrrry tiny,
# so if regression works -- it better has at least 0.5 ;)
# otherwise fix it! ;)
# YOH: not now -- issues with libsvr in SG and linear kernel
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(confusion.stats['CCe'] < 0.5)
# just to check if it works fine
split_predictions = splitregr.predict(ds.samples)
def test_retrainables(self, clf):
# we need a copy since will tune its internals later on
clf = clf.clone()
clf.ca.change_temporarily(
enable_ca=['estimates'],
# ensure that it does do predictions
# while training
disable_ca=['training_confusion'])
clf_re = clf.clone()
# TODO: .retrainable must have a callback to call smth like
# _set_retrainable
clf_re._set_retrainable(True)
# need to have high snr so we don't 'cope' with problematic
# datasets since otherwise unittests would fail.
dsargs = {
'perlabel': 50,
'nlabels': 2,
'nfeatures': 5,
'nchunks': 1,
'nonbogus_features': [2, 4],
'snr': 5.0
}
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# NB datasets will be changed by the end of testing, so if
# are to change to use generic datasets - make sure to copy
# them here
dstrain = deepcopy(datasets['uni2large_train'])
dstest = deepcopy(datasets['uni2large_test'])
clf.untrain()
clf_re.untrain()
trerr, trerr_re = TransferError(clf), \
TransferError(clf_re,
disable_ca=['training_confusion'])
# Just check for correctness of retraining
err_1 = trerr(dstest, dstrain)
self.failUnless(
err_1 < 0.3,
msg="We should test here on easy dataset. Got error of %s" % err_1)
values_1 = clf.ca.estimates[:]
# some times retraining gets into deeper optimization ;-)
eps = 0.05
corrcoef_eps = 0.85 # just to get no failures... usually > 0.95
def batch_test(retrain=True, retest=True, closer=True):
err = trerr(dstest, dstrain)
err_re = trerr_re(dstest, dstrain)
corr = np.corrcoef(clf.ca.estimates, clf_re.ca.estimates)[0, 1]
corr_old = np.corrcoef(values_1, clf_re.ca.estimates)[0, 1]
if __debug__:
debug(
'TEST', "Retraining stats: errors %g %g corr %g "
"with old error %g corr %g" %
(err, err_re, corr, err_1, corr_old))
self.failUnless(clf_re.ca.retrained == retrain,
("Must fully train",
"Must retrain instead of full training")[retrain])
self.failUnless(clf_re.ca.repredicted == retest,
("Must fully test",
"Must retest instead of full testing")[retest])
self.failUnless(
corr > corrcoef_eps,
msg="Result must be close to the one without retraining."
" Got corrcoef=%s" % (corr))
if closer:
self.failUnless(
corr >= corr_old,
msg="Result must be closer to current without retraining"
" than to old one. Got corrcoef=%s" % (corr_old))
# Check sequential retraining/retesting
for i in xrange(3):
flag = bool(i != 0)
# ok - on 1st call we should train/test, then retrain/retest
# and we can't compare for closinest to old result since
# we are working on the same data/classifier
batch_test(retrain=flag, retest=flag, closer=False)
# should retrain nicely if we change a parameter
if 'C' in clf.params:
clf.params.C *= 0.1
clf_re.params.C *= 0.1
batch_test()
elif 'sigma_noise' in clf.params:
clf.params.sigma_noise *= 100
clf_re.params.sigma_noise *= 100
batch_test()
else:
raise RuntimeError, \
'Please implement testing while changing some of the ' \
'params for clf %s' % clf
# should retrain nicely if we change kernel parameter
if hasattr(clf, 'kernel_params') and len(clf.kernel_params):
clf.kernel_params.gamma = 0.1
clf_re.kernel_params.gamma = 0.1
# retest is false since kernel got recomputed thus
# can't expect to use the same kernel
batch_test(retest=not ('gamma' in clf.kernel_params))
# should retrain nicely if we change labels
oldlabels = dstrain.targets[:]
dstrain.permute_attr(assure_permute=True)
self.failUnless(
(oldlabels != dstrain.targets).any(),
msg="We should succeed at permutting -- now got the same targets")
batch_test()
# Change labels in testing
oldlabels = dstest.targets[:]
dstest.permute_attr(assure_permute=True)
self.failUnless(
(oldlabels != dstest.targets).any(),
msg="We should succeed at permutting -- now got the same targets")
batch_test()
# should re-train if we change data
# reuse trained SVM and its 'final' optimization point
if not clf.__class__.__name__ in [
'GPR'
]: # on GPR everything depends on the data ;-)
oldsamples = dstrain.samples.copy()
dstrain.samples[:] += dstrain.samples * 0.05
self.failUnless((oldsamples != dstrain.samples).any())
batch_test(retest=False)
clf.ca.reset_changed_temporarily()
# test retrain()
# TODO XXX -- check validity
clf_re.retrain(dstrain)
self.failUnless(clf_re.ca.retrained)
clf_re.retrain(dstrain, labels=True)
self.failUnless(clf_re.ca.retrained)
clf_re.retrain(dstrain, traindataset=True)
self.failUnless(clf_re.ca.retrained)
# test repredict()
clf_re.repredict(dstest.samples)
self.failUnless(clf_re.ca.repredicted)
self.failUnlessRaises(RuntimeError,
clf_re.repredict,
dstest.samples,
labels=True)
"""for now retesting with anything changed makes no sense"""
clf_re._set_retrainable(False)
from mvpa.datasets.splitters import OddEvenSplitter
from mvpa.clfs.svm import LinearCSVMC
from mvpa.clfs.transerror import TransferError
from mvpa.algorithms.cvtranserror import CrossValidatedTransferError
from mvpa.measures.searchlight import sphere_searchlight
from mvpa.testing.datasets import datasets
from mvpa.mappers.fx import mean_sample
"""For the sake of simplicity, let's use a small artificial dataset."""
# Lets just use our tiny 4D dataset from testing battery
dataset = datasets['3dlarge']
"""Now it only takes three lines for a searchlight analysis."""
# setup measure to be computed in each sphere (cross-validated
# generalization error on odd/even splits)
cv = CrossValidatedTransferError(TransferError(LinearCSVMC()),
OddEvenSplitter())
# setup searchlight with 2 voxels radius and measure configured above
sl = sphere_searchlight(cv, radius=2, space='myspace', postproc=mean_sample())
# run searchlight on dataset
sl_map = sl(dataset)
print 'Best performing sphere error:', np.min(sl_map.samples)
"""
If this analysis is done on a fMRI dataset using `NiftiDataset` the resulting
searchlight map (`sl_map`) can be mapped back into the original dataspace
and viewed as a brain overlay. :ref:`Another example <example_searchlight>`
shows a typical application of this algorithm.
"Failed to load due to %r" % (e,)
ok_(isinstance(lrn_, Classifier))
# Verify that we have the same ca enabled
# XXX FAILS atm!
#ok_(set(lrn.ca.enabled) == set(lrn_.ca.enabled))
# lets choose a dataset
dsname, errorfx = \
{False: ('uni2large', MeanMismatchErrorFx()),
True: ('sin_modulated', CorrErrorFx())}\
['regression' in lrn.__tags__]
ds_train = datasets['%s_train' % dsname]
ds_test = datasets['%s_test' % dsname]
te = TransferError(lrn, errorfx)
te_ = TransferError(lrn_, errorfx)
error = te(ds_test, ds_train)
error_ = te_(ds_test, ds_train)
ok_(error == error_)
if len(set(['swig', 'rpy2']).intersection(lrn.__tags__)):
raise SkipTest("Trained swigged and R-interfaced classifiers can't "
"be stored/reloaded yet")
# now lets store/reload the trained one
try:
h5save(f.name, lrn_)
except Exception, e:
def test_cper_class(self, clf):
if not (clf.params.has_key('C')):
# skip those without C
return
ds = datasets['uni2medium'].copy()
ds__ = datasets['uni2medium'].copy()
#
# ballanced set
# Lets add a bit of noise to drive classifier nuts. same
# should be done for disballanced set
ds__.samples = ds__.samples + \
0.5 * np.random.normal(size=(ds__.samples.shape))
#
# disballanced set
# lets overpopulate label 0
times = 20
ds_ = ds[(range(ds.nsamples) + range(ds.nsamples/2) * times)]
ds_.samples = ds_.samples + \
0.5 * np.random.normal(size=(ds_.samples.shape))
spl = get_nsamples_per_attr(ds_, 'targets') #_.samplesperlabel
#print ds_.targets, ds_.chunks
cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(),
enable_ca='confusion')
# on balanced
e = cve(ds__)
tpr_1 = cve.ca.confusion.stats["TPR"][1]
# on disbalanced
e = cve(ds_)
tpr_2 = cve.ca.confusion.stats["TPR"][1]
# Set '1 C per label'
# recreate cvte since previous might have operated on copies
cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(),
enable_ca='confusion')
oldC = clf.params.C
# TODO: provide clf.params.C not with a tuple but dictionary
# with C per label (now order is deduced in a cruel way)
ratio = np.sqrt(float(spl[ds_.UT[0]])/spl[ds_.UT[1]])
clf.params.C = (-1/ratio, -1*ratio)
try:
# on disbalanced but with balanced C
e_ = cve(ds_)
# reassign C
clf.params.C = oldC
except:
clf.params.C = oldC
raise
tpr_3 = cve.ca.confusion.stats["TPR"][1]
# Actual tests
if cfg.getboolean('tests', 'labile', default='yes'):
self.failUnless(tpr_1 > 0.25,
msg="Without disballance we should have some "
"hits, but got TPR=%.3f" % tpr_1)
self.failUnless(tpr_2 < 0.25,
msg="With disballance we should have almost no "
"hits for minor, but got TPR=%.3f" % tpr_2)
self.failUnless(tpr_3 > 0.25,
msg="With disballanced data but ratio-based Cs "
"we should have some hits for minor, but got "
"TPR=%.3f" % tpr_3)