def test_confusion_based_error(self, l_clf):
train = datasets['uni2medium']
train = train[train.sa.train == 1]
# to check if we fail to classify for 3 labels
test3 = datasets['uni3medium']
test3 = test3[test3.sa.train == 1]
err = ConfusionBasedError(clf=l_clf)
terr = TransferMeasure(l_clf,
Splitter('train', attr_values=[1, 1]),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'))
self.assertRaises(UnknownStateError, err, None)
"""Shouldn't be able to access the state yet"""
l_clf.train(train)
e, te = err(None), terr(train)
te = np.asscalar(te)
self.assertTrue(
abs(e - te) < 1e-10,
msg="ConfusionBasedError (%.2g) should be equal to TransferError "
"(%.2g) on traindataset" % (e, te))
# this will print nasty WARNING but it is ok -- it is just checking code
# NB warnings are not printed while doing whole testing
warning("Don't worry about the following warning.")
if 'multiclass' in l_clf.__tags__:
self.assertFalse(terr(test3) is None)
# try copying the beast
terr_copy = copy(terr)
def test_null_dist_prob(self, l_clf):
train = datasets['uni2medium']
num_perm = 10
permutator = AttributePermutator('targets',
count=num_perm,
limit='chunks')
# define class to estimate NULL distribution of errors
# use left tail of the distribution since we use MeanMatchFx as error
# function and lower is better
terr = TransferMeasure(l_clf,
Repeater(count=2),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'),
null_dist=MCNullDist(permutator, tail='left'))
# check reasonable error range
err = terr(train)
self.assertTrue(np.mean(err) < 0.4)
# Lets do the same for CVTE
cvte = CrossValidation(l_clf,
OddEvenPartitioner(),
null_dist=MCNullDist(permutator,
tail='left',
enable_ca=['dist_samples'
]),
postproc=mean_sample())
cv_err = cvte(train)
# check that the result is highly significant since we know that the
# data has signal
null_prob = np.asscalar(terr.ca.null_prob)
if cfg.getboolean('tests', 'labile', default='yes'):
self.assertTrue(
null_prob <= 0.1,
msg="Failed to check that the result is highly significant "
"(got %f) since we know that the data has signal" % null_prob)
self.assertTrue(
np.asscalar(cvte.ca.null_prob) <= 0.1,
msg="Failed to check that the result is highly significant "
"(got p(cvte)=%f) since we know that the data has signal" %
np.asscalar(cvte.ca.null_prob))
# we should be able to access the actual samples of the distribution
# yoh: why it is 3D really?
# mih: because these are the distribution samples for the ONE error
# collapsed into ONE value across all folds. It will also be
# 3d if the return value of the measure isn't a scalar and it is
# not collapsed across folds. it simply corresponds to the shape
# of the output dataset of the respective measure (+1 axis)
# Some permutations could have been skipped since classifier failed
# to train due to degenerate situation etc, thus accounting for them
self.assertEqual(cvte.null_dist.ca.dist_samples.shape[2],
num_perm - cvte.null_dist.ca.skipped)
def _train(self, dataset):
pmeasure = ProxyMeasure(
self.lrn,
postproc=BinaryFxNode(self.errorfx, self.lrn.space),
skip_train=True # do not train since fmeasure will
)
# First we need to replicate our RFE construct but this time
# with pmeasure for the classifier
rfe = RFE(
self.fmeasure,
pmeasure,
Splitter('partitions'),
fselector=self.fselector,
bestdetector=None,
train_pmeasure=False,
stopping_criterion=None, # full "track"
update_sensitivity=self.update_sensitivity,
enable_ca=['errors', 'nfeatures'])
errors, nfeatures = [], []
if __debug__:
debug("RFEC", "Stage 1: initial nested CV/RFE for %s", (dataset, ))
for partition in self.partitioner.generate(dataset):
rfe.train(partition)
errors.append(rfe.ca.errors)
nfeatures.append(rfe.ca.nfeatures)
# mean errors across splits and find optimal number
errors_mean = np.mean(errors, axis=0)
nfeatures_mean = np.mean(nfeatures, axis=0)
# we will take the "mean location" of the min to stay
# within the most 'stable' choice
mins_idx = np.where(errors_mean == np.min(errors_mean))[0]
min_idx = mins_idx[int(len(mins_idx) / 2)]
min_error = errors_mean[min_idx]
assert (min_error == np.min(errors_mean))
nfeatures_min = nfeatures_mean[min_idx]
if __debug__:
debug(
"RFEC", "Choosing among %d choices to have %d features with "
"mean error=%.2g (initial mean error %.2g)",
(len(mins_idx), nfeatures_min, min_error, errors_mean[0]))
self.nfeatures_min = nfeatures_min
if __debug__:
debug(
"RFEC", "Stage 2: running RFE on full training dataset to "
"distil best %d features" % nfeatures_min)
super(SplitRFE, self)._train(dataset)
def test_james_problem_multiclass(self):
percent = 80
dataset = datasets['uni4large']
#dataset = dataset[:, dataset.a.nonbogus_features]
rfesvm_split = LinearCSVMC()
fs = \
RFE(rfesvm_split.get_sensitivity_analyzer(
postproc=ChainMapper([
#FxMapper('features', l2_normed),
#FxMapper('samples', np.mean),
#FxMapper('samples', np.abs)
FxMapper('features', lambda x: np.argsort(np.abs(x))),
#maxofabs_sample()
mean_sample()
])),
ProxyMeasure(rfesvm_split,
postproc=BinaryFxNode(mean_mismatch_error,
'targets')),
Splitter('train'),
fselector=FractionTailSelector(
percent / 100.0,
mode='select', tail='upper'), update_sensitivity=True)
clf = FeatureSelectionClassifier(
LinearCSVMC(),
# on features selected via RFE
fs)
# update sensitivity at each step (since we're not using the
# same CLF as sensitivity analyzer)
class StoreResults(object):
def __init__(self):
self.storage = []
def __call__(self, data, node, result):
self.storage.append((node.measure.mapper.ca.history,
node.measure.mapper.ca.errors)),
cv_storage = StoreResults()
cv = CrossValidation(clf,
NFoldPartitioner(),
postproc=mean_sample(),
callback=cv_storage,
enable_ca=['stats'])
#cv = SplitClassifier(clf)
try:
error = cv(dataset).samples.squeeze()
except Exception, e:
self.fail('CrossValidation cannot handle classifier with RFE '
'feature selection. Got exception: %s' % (e, ))
def test_clf_transfer_measure(self):
# and now on a classifier
clf = SMLR()
enode = BinaryFxNode(mean_mismatch_error, 'targets')
tm = TransferMeasure(clf, Splitter('chunks', count=2),
enable_ca=['stats'])
res = tm(self.dataset)
manual_error = np.mean(res.samples.squeeze() != res.sa.targets)
postproc_error = enode(res)
tm_err = TransferMeasure(clf, Splitter('chunks', count=2),
postproc=enode)
auto_error = tm_err(self.dataset)
ok_(manual_error == postproc_error.samples[0, 0])
def test_pseudo_cv_measure(self):
clf = SMLR()
enode = BinaryFxNode(mean_mismatch_error, 'targets')
tm = TransferMeasure(clf, Splitter('partitions'), postproc=enode)
cvgen = NFoldPartitioner()
rm = RepeatedMeasure(tm, cvgen)
res = rm(self.dataset)
# one error per fold
assert_equal(res.shape, (len(self.dataset.sa['chunks'].unique), 1))
# we can do the same with Crossvalidation
cv = CrossValidation(clf, cvgen, enable_ca=['stats', 'training_stats',
'datasets'])
res = cv(self.dataset)
assert_equal(res.shape, (len(self.dataset.sa['chunks'].unique), 1))
def test_james_problem(self):
percent = 80
dataset = datasets['uni2small']
rfesvm_split = LinearCSVMC()
fs = \
RFE(rfesvm_split.get_sensitivity_analyzer(),
ProxyMeasure(rfesvm_split,
postproc=BinaryFxNode(mean_mismatch_error,
'targets')),
Splitter('train'),
fselector=FractionTailSelector(
percent / 100.0,
mode='select', tail='upper'), update_sensitivity=True)
clf = FeatureSelectionClassifier(
LinearCSVMC(),
# on features selected via RFE
fs)
# update sensitivity at each step (since we're not using the
# same CLF as sensitivity analyzer)
class StoreResults(object):
def __init__(self):
self.storage = []
def __call__(self, data, node, result):
self.storage.append((node.measure.mapper.ca.history,
node.measure.mapper.ca.errors)),
cv_storage = StoreResults()
cv = CrossValidation(clf,
NFoldPartitioner(),
postproc=mean_sample(),
callback=cv_storage,
enable_ca=['confusion']) # TODO -- it is stats
#cv = SplitClassifier(clf)
try:
error = cv(dataset).samples.squeeze()
except Exception as e:
self.fail('CrossValidation cannot handle classifier with RFE '
'feature selection. Got exception: %s' % (e, ))
assert (len(cv_storage.storage) == len(dataset.sa['chunks'].unique))
assert (len(cv_storage.storage[0]) == 2)
assert (len(cv_storage.storage[0][0]) == dataset.nfeatures)
self.assertTrue(error < 0.2)
def test_single_class(self, clf):
"""Test if binary and multiclass can handle single class training/testing
"""
ds = datasets['uni2small']
ds = ds[ds.sa.targets == 'L0'] # only 1 label
assert(ds.sa['targets'].unique == ['L0'])
ds_ = list(OddEvenPartitioner().generate(ds))[0]
# Here is our "nice" 0.6 substitute for TransferError:
trerr = TransferMeasure(clf, Splitter('train'),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'))
try:
err = np.asscalar(trerr(ds_))
except Exception, e:
self.fail(str(e))
def test_ifs(self, svm):
# measure for feature selection criterion and performance assesment
# use the SAME clf!
errorfx = mean_mismatch_error
fmeasure = CrossValidation(svm,
NFoldPartitioner(),
postproc=mean_sample())
pmeasure = ProxyMeasure(svm, postproc=BinaryFxNode(errorfx, 'targets'))
ifs = IFS(fmeasure,
pmeasure,
Splitter('purpose', attr_values=['train', 'test']),
fselector=\
# go for lower tail selection as data_measure will return
# errors -> low is good
FixedNElementTailSelector(1, tail='lower', mode='select'),
)
wdata = self.get_data()
wdata.sa['purpose'] = np.repeat('train', len(wdata))
tdata = self.get_data()
tdata.sa['purpose'] = np.repeat('test', len(tdata))
ds = vstack((wdata, tdata))
orig_nfeatures = ds.nfeatures
ifs.train(ds)
resds = ifs(ds)
# fail if orig datasets are changed
self.assertTrue(ds.nfeatures == orig_nfeatures)
# check that the features set with the least error is selected
self.assertTrue(len(ifs.ca.errors))
e = np.array(ifs.ca.errors)
self.assertTrue(resds.nfeatures == e.argmin() + 1)
# repeat with dataset where selection order is known
wsignal = datasets['dumb2'].copy()
wsignal.sa['purpose'] = np.repeat('train', len(wsignal))
tsignal = datasets['dumb2'].copy()
tsignal.sa['purpose'] = np.repeat('test', len(tsignal))
signal = vstack((wsignal, tsignal))
ifs.train(signal)
resds = ifs(signal)
self.assertTrue((resds.samples[:, 0] == signal.samples[:, 0]).all())
def test_cached_kernel_different_datasets(self):
skip_if_no_external('shogun', ver_dep='shogun:rev', min_version=4455)
# Inspired by the problem Swaroop ran into
k = LinearSGKernel(normalizer_cls=False)
k_ = LinearSGKernel(normalizer_cls=False) # to be cached
ck = CachedKernel(k_)
clf = sgSVM(svm_impl='libsvm', kernel=k, C=-1)
clf_ = sgSVM(svm_impl='libsvm', kernel=ck, C=-1)
cvte = CrossValidation(clf, NFoldPartitioner())
cvte_ = CrossValidation(clf_, NFoldPartitioner())
postproc=BinaryFxNode(mean_mismatch_error, 'targets')
te = ProxyMeasure(clf, postproc=postproc)
te_ = ProxyMeasure(clf_, postproc=postproc)
for r in xrange(2):
ds1 = datasets['uni2medium']
errs1 = cvte(ds1)
ck.compute(ds1)
ok_(ck._recomputed)
errs1_ = cvte_(ds1)
ok_(~ck._recomputed)
assert_array_equal(errs1, errs1_)
ds2 = datasets['uni3small']
errs2 = cvte(ds2)
ck.compute(ds2)
ok_(ck._recomputed)
errs2_ = cvte_(ds2)
ok_(~ck._recomputed)
assert_array_equal(errs2, errs2_)
ssel = np.round(datasets['uni2large'].samples[:5, 0]).astype(int)
te.train(datasets['uni3small'][::2])
terr = np.asscalar(te(datasets['uni3small'][ssel]))
te_.train(datasets['uni3small'][::2])
terr_ = np.asscalar(te_(datasets['uni3small'][ssel]))
ok_(~ck._recomputed)
ok_(terr == terr_)
def __init__(self,
learner,
generator=None,
errorfx=mean_mismatch_error,
splitter=None,
**kwargs):
"""
Parameters
----------
learner : Learner
Any trainable node that shall be run on the dataset folds.
generator : Node, optional
Generator used to resample the input dataset into multiple instances
(i.e. partitioning it). The number of datasets yielded by this
generator determines the number of cross-validation folds.
IMPORTANT: The ``space`` of this generator determines the attribute
that will be used to split all generated datasets into training and
testing sets. If None provided, a single original dataset will be
passed to the ``splitter`` as is
errorfx : Node or callable
Custom implementation of an error function. The callable needs to
accept two arguments (1. predicted values, 2. target values). If not
a Node, it gets wrapped into a `BinaryFxNode`.
splitter : Splitter or None
A Splitter instance to split the dataset into training and testing
part. The first split will be used for training and the second for
testing -- all other splits will be ignored. If None, a default
splitter is auto-generated using the ``space`` setting of the
``generator``. If no ``generator`` provided, splitter uses 'partitions'
sample attribute. The default splitter is configured to return the
``1``-labeled partition of the input dataset at first, and the
``2``-labeled partition second. This behavior corresponds to most
Partitioners that label the taken-out portion ``2`` and the remainder
with ``1``.
"""
# compile the appropriate repeated measure to do cross-validation from
# pieces
if errorfx is not None:
# error node -- postproc of transfer measure
if isinstance(errorfx, Node):
enode = errorfx
else:
# wrap into BinaryFxNode
enode = BinaryFxNode(errorfx, learner.get_space())
else:
enode = None
if splitter is None:
# default splitter splits into "1" and "2" partition.
# that will effectively ignore 'deselected' samples (e.g. by
# Balancer). It is done this way (and not by ignoring '0' samples
# because it is guaranteed to yield two splits) and is more likely
# to fail in visible ways if the attribute does not have 0,1,2
# values at all (i.e. a literal train/test/spareforlater attribute)
splitter = Splitter(
generator.get_space() if generator else 'partitions',
attr_values=(1, 2))
# transfer measure to wrap the learner
# splitter used the output space of the generator to know what to split
tm = TransferMeasure(learner, splitter, postproc=enode)
space = kwargs.pop('space', 'sa.cvfolds')
# and finally the repeated measure to perform the x-val
RepeatedMeasure.__init__(self,
tm,
generator=generator,
space=space,
**kwargs)
for ca in ['stats', 'training_stats']:
if self.ca.is_enabled(ca):
# enforce ca if requested
tm.ca.enable(ca)
if self.ca.is_enabled('training_stats'):
# also enable training stats in the learner
learner.ca.enable('training_stats')
def test_rfe(self, clf):
# sensitivity analyser and transfer error quantifier use the SAME clf!
sens_ana = clf.get_sensitivity_analyzer(postproc=maxofabs_sample())
pmeasure = ProxyMeasure(clf,
postproc=BinaryFxNode(mean_mismatch_error,
'targets'))
cvmeasure = CrossValidation(clf,
NFoldPartitioner(),
errorfx=mean_mismatch_error,
postproc=mean_sample())
rfesvm_split = SplitClassifier(clf, OddEvenPartitioner())
# explore few recipes
for rfe, data in [
# because the clf is already trained when computing the sensitivity
# map, prevent retraining for transfer error calculation
# Use absolute of the svm weights as sensitivity
(RFE(sens_ana,
pmeasure,
Splitter('train'),
fselector=FixedNElementTailSelector(1),
train_pmeasure=False), self.get_data()),
# use cross-validation within training to get error for the stopping point
# but use full training data to derive sensitivity
(
RFE(
sens_ana,
cvmeasure,
Repeater(
2
), # give the same full dataset to sens_ana and cvmeasure
fselector=FractionTailSelector(0.70,
mode='select',
tail='upper'),
train_pmeasure=True),
normal_feature_dataset(perlabel=20,
nchunks=5,
nfeatures=200,
nonbogus_features=[0, 1],
snr=1.5)),
# use cross-validation (via SplitClassifier) and get mean
# of normed sensitivities across those splits
(
RFE(
rfesvm_split.get_sensitivity_analyzer(
postproc=ChainMapper([
FxMapper('features', l2_normed),
FxMapper('samples', np.mean),
FxMapper('samples', np.abs)
])),
ConfusionBasedError(rfesvm_split, confusion_state='stats'),
Repeater(
2), # we will use the same full cv-training dataset
fselector=FractionTailSelector(0.50,
mode='select',
tail='upper'),
stopping_criterion=NBackHistoryStopCrit(
BestDetector(), 10),
train_pmeasure=
False, # we just extract it from existing confusion
update_sensitivity=True),
normal_feature_dataset(perlabel=28,
nchunks=7,
nfeatures=200,
nonbogus_features=[0, 1],
snr=1.5))
]:
# prep data
# data = datasets['uni2medium']
data_nfeatures = data.nfeatures
rfe.train(data)
resds = rfe(data)
# fail if orig datasets are changed
self.assertTrue(data.nfeatures == data_nfeatures)
# check that the features set with the least error is selected
if len(rfe.ca.errors):
e = np.array(rfe.ca.errors)
if isinstance(rfe._fselector, FixedNElementTailSelector):
self.assertTrue(resds.nfeatures == data_nfeatures -
e.argmin())
else:
imin = np.argmin(e)
if 'does_feature_selection' in clf.__tags__:
# if clf is smart it might figure it out right away
assert_array_less(imin, len(e))
else:
# in this case we can even check if we had actual
# going down/up trend... although -- why up???
self.assertTrue(1 < imin < len(e) - 1)
else:
self.assertTrue(resds.nfeatures == data_nfeatures)
# silly check if nfeatures is in decreasing order
nfeatures = np.array(rfe.ca.nfeatures).copy()
nfeatures.sort()
self.assertTrue((nfeatures[::-1] == rfe.ca.nfeatures).all())
# check if history has elements for every step
self.assertTrue(
set(rfe.ca.history) == set(range(len(np.array(
rfe.ca.errors)))))
# Last (the largest number) can be present multiple times even
# if we remove 1 feature at a time -- just need to stop well
# in advance when we have more than 1 feature left ;)
self.assertTrue(rfe.ca.nfeatures[-1] == len(
np.where(rfe.ca.history == max(rfe.ca.history))[0]))
raise AssertionError, \
"Failed to load due to %r" % (e,)
ok_(isinstance(lrn_, Classifier))
# Verify that we have the same ca enabled
# XXX FAILS atm!
#ok_(set(lrn.ca.enabled) == set(lrn_.ca.enabled))
# lets choose a dataset
dsname, errorfx = \
{False: ('uni2large', mean_mismatch_error),
True: ('sin_modulated', corr_error)}\
['regression' in lrn.__tags__]
ds = datasets[dsname]
splitter = Splitter('train')
postproc = BinaryFxNode(errorfx, 'targets')
te = TransferMeasure(lrn, splitter, postproc=postproc)
te_ = TransferMeasure(lrn_, splitter, postproc=postproc)
error = te(ds)
error_ = te_(ds)
if len(set(['swig', 'rpy2']).intersection(lrn.__tags__)):
raise SkipTest("Trained swigged and R-interfaced classifiers can't "
"be stored/reloaded yet")
# now lets store/reload the trained one
try:
h5save(fname, lrn_)
except Exception, e:
raise AssertionError, \
def _train(self, dataset):
pmeasure = ProxyMeasure(
self.lrn,
postproc=BinaryFxNode(self.errorfx, self.lrn.space),
skip_train=not self.
train_pmeasure # do not train since fmeasure will
)
# First we need to replicate our RFE construct but this time
# with pmeasure for the classifier
rfe = RFE(
self.fmeasure,
pmeasure,
Splitter('partitions'),
fselector=self.fselector,
bestdetector=None,
train_pmeasure=self.train_pmeasure,
stopping_criterion=None, # full "track"
update_sensitivity=self.update_sensitivity,
enable_ca=['errors', 'nfeatures'])
errors, nfeatures = [], []
if __debug__:
debug("RFEC", "Stage 1: initial nested CV/RFE for %s", (dataset, ))
if self.nproc != 1 and externals.exists('joblib'):
nested_results = jl.Parallel(self.nproc)(
jl.delayed(_process_partition)(rfe, partition)
for partition in self.partitioner.generate(dataset))
else:
nested_results = [
_process_partition(rfe, partition)
for partition in self.partitioner.generate(dataset)
]
# unzip
errors = [x[0] for x in nested_results]
nfeatures = [x[1] for x in nested_results]
self.ca.nested_nfeatures = nfeatures
self.ca.nested_errors = errors
# mean errors across splits and find optimal number
errors_mean = np.mean(errors, axis=0)
nfeatures_mean = np.mean(nfeatures, axis=0)
# we will take the "mean location" of the min to stay
# within the most 'stable' choice
mins_idx = np.where(errors_mean == np.min(errors_mean))[0]
min_idx = mins_idx[int(len(mins_idx) / 2)]
min_error = errors_mean[min_idx]
assert (min_error == np.min(errors_mean))
nfeatures_min = nfeatures_mean[min_idx]
if __debug__:
debug(
"RFEC", "Choosing among %d choices to have %d features with "
"mean error=%.2g (initial mean error %.2g)",
(len(mins_idx), nfeatures_min, min_error, errors_mean[0]))
self.nfeatures_min = nfeatures_min
if __debug__:
debug(
"RFEC", "Stage 2: running RFE on full training dataset to "
"obtain the best %d features" % nfeatures_min)
super(SplitRFE, self)._train(dataset)
def test_retrainables(self, clf):
# XXX we agreed to not worry about this for the initial 0.6 release
raise SkipTest
# we need a copy since will tune its internals later on
clf = clf.clone()
clf.ca.change_temporarily(
enable_ca=['estimates'],
# ensure that it does do predictions
# while training
disable_ca=['training_stats'])
clf_re = clf.clone()
# TODO: .retrainable must have a callback to call smth like
# _set_retrainable
clf_re._set_retrainable(True)
# need to have high snr so we don't 'cope' with problematic
# datasets since otherwise unittests would fail.
dsargs = {
'perlabel': 50,
'nlabels': 2,
'nfeatures': 5,
'nchunks': 1,
'nonbogus_features': [2, 4],
'snr': 5.0
}
## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# NB datasets will be changed by the end of testing, so if
# are to change to use generic datasets - make sure to copy
# them here
ds = deepcopy(datasets['uni2large'])
clf.untrain()
clf_re.untrain()
trerr = TransferMeasure(clf,
Splitter('train'),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'))
trerr_re = TransferMeasure(clf_re,
Splitter('train'),
disable_ca=['training_stats'],
postproc=BinaryFxNode(
mean_mismatch_error, 'targets'))
# Just check for correctness of retraining
err_1 = np.asscalar(trerr(ds))
self.assertTrue(
err_1 < 0.3,
msg="We should test here on easy dataset. Got error of %s" % err_1)
values_1 = clf.ca.estimates[:]
# some times retraining gets into deeper optimization ;-)
eps = 0.05
corrcoef_eps = 0.85 # just to get no failures... usually > 0.95
def batch_test(retrain=True, retest=True, closer=True):
err = np.asscalar(trerr(ds))
err_re = np.asscalar(trerr_re(ds))
corr = np.corrcoef(clf.ca.estimates, clf_re.ca.estimates)[0, 1]
corr_old = np.corrcoef(values_1, clf_re.ca.estimates)[0, 1]
if __debug__:
debug(
'TEST', "Retraining stats: errors %g %g corr %g "
"with old error %g corr %g" %
(err, err_re, corr, err_1, corr_old))
self.assertTrue(clf_re.ca.retrained == retrain,
("Must fully train",
"Must retrain instead of full training")[retrain])
self.assertTrue(clf_re.ca.repredicted == retest,
("Must fully test",
"Must retest instead of full testing")[retest])
self.assertTrue(
corr > corrcoef_eps,
msg="Result must be close to the one without retraining."
" Got corrcoef=%s" % (corr))
if closer:
self.assertTrue(
corr >= corr_old,
msg="Result must be closer to current without retraining"
" than to old one. Got corrcoef=%s" % (corr_old))
# Check sequential retraining/retesting
for i in xrange(3):
flag = bool(i != 0)
# ok - on 1st call we should train/test, then retrain/retest
# and we can't compare for closinest to old result since
# we are working on the same data/classifier
batch_test(retrain=flag, retest=flag, closer=False)
# should retrain nicely if we change a parameter
if 'C' in clf.params:
clf.params.C *= 0.1
clf_re.params.C *= 0.1
batch_test()
elif 'sigma_noise' in clf.params:
clf.params.sigma_noise *= 100
clf_re.params.sigma_noise *= 100
batch_test()
else:
raise RuntimeError, \
'Please implement testing while changing some of the ' \
'params for clf %s' % clf
# should retrain nicely if we change kernel parameter
if hasattr(clf, 'kernel_params') and len(clf.kernel_params):
clf.kernel_params.gamma = 0.1
clf_re.kernel_params.gamma = 0.1
# retest is false since kernel got recomputed thus
# can't expect to use the same kernel
batch_test(retest=not ('gamma' in clf.kernel_params))
# should retrain nicely if we change labels
permute = AttributePermutator('targets', assure=True)
oldlabels = dstrain.targets[:]
dstrain = permute(dstrain)
self.assertTrue(
(oldlabels != dstrain.targets).any(),
msg="We should succeed at permutting -- now got the same targets")
ds = vstack((dstrain, dstest))
batch_test()
# Change labels in testing
oldlabels = dstest.targets[:]
dstest = permute(dstest)
self.assertTrue(
(oldlabels != dstest.targets).any(),
msg="We should succeed at permutting -- now got the same targets")
ds = vstack((dstrain, dstest))
batch_test()
# should re-train if we change data
# reuse trained SVM and its 'final' optimization point
if not clf.__class__.__name__ in [
'GPR'
]: # on GPR everything depends on the data ;-)
oldsamples = dstrain.samples.copy()
dstrain.samples[:] += dstrain.samples * 0.05
self.assertTrue((oldsamples != dstrain.samples).any())
ds = vstack((dstrain, dstest))
batch_test(retest=False)
clf.ca.reset_changed_temporarily()
# test retrain()
# TODO XXX -- check validity
clf_re.retrain(dstrain)
self.assertTrue(clf_re.ca.retrained)
clf_re.retrain(dstrain, labels=True)
self.assertTrue(clf_re.ca.retrained)
clf_re.retrain(dstrain, traindataset=True)
self.assertTrue(clf_re.ca.retrained)
# test repredict()
clf_re.repredict(dstest.samples)
self.assertTrue(clf_re.ca.repredicted)
self.assertRaises(RuntimeError,
clf_re.repredict,
dstest.samples,
labels=True)
"""for now retesting with anything changed makes no sense"""
clf_re._set_retrainable(False)
def __init__(self, **kwargs):
BinaryFxNode.__init__(self, fx=mean_mismatch_error, space='targets', **kwargs)
def __init__(self, fx, space, subj_space, **kwargs):
self.subj_space = subj_space
BinaryFxNode.__init__(self, fx, space, **kwargs)
def test_multiclass_ties(clf):
if 'lars' in clf.__tags__:
raise SkipTest("Known to crash while running this test")
ds = _dsties1
# reassign data between ties, so we know that decision is data, not order driven
ds_ = ds.copy(deep=True)
ds_.samples[ds.a.ties_idx[1]] = ds.samples[ds.a.ties_idx[0]]
ds_.samples[ds.a.ties_idx[0]] = ds.samples[ds.a.ties_idx[1]]
ok_(np.any(ds_.samples != ds.samples))
clf_ = clf.clone()
clf = clf.clone()
clf.ca.enable(['estimates', 'predictions'])
clf_.ca.enable(['estimates', 'predictions'])
te = TransferMeasure(clf,
Splitter('train'),
postproc=BinaryFxNode(mean_mismatch_error, 'targets'),
enable_ca=['stats'])
te_ = TransferMeasure(clf_,
Splitter('train'),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'),
enable_ca=['stats'])
te = CrossValidation(clf,
NFoldPartitioner(),
postproc=mean_sample(),
enable_ca=['stats'])
te_ = CrossValidation(clf_,
NFoldPartitioner(),
postproc=mean_sample(),
enable_ca=['stats'])
error = te(ds)
matrix = te.ca.stats.matrix
# if ties were broken randomly we should have got nearly the same
# number of hits for tied targets
ties_indices = [te.ca.stats.labels.index(c) for c in ds.a.ties]
hits = np.diag(te.ca.stats.matrix)[ties_indices]
# First check is to see if we swap data between tied labels we
# are getting the same results if we permute labels accordingly,
# i.e. that tie resolution is not dependent on the labels order
# but rather on the data
te_(ds_)
matrix_swapped = te_.ca.stats.matrix
if False: #0 in hits:
print clf, matrix, matrix_swapped
print clf.ca.estimates[:, 2] - clf.ca.estimates[:, 0]
#print clf.ca.estimates
# TODO: for now disabled all the non-compliant ones to pass the
# tests. For visibility decided to skip them instead of just
# exclusion and skipping only here to possibly catch crashes
# which might happen before
if len(
set(('libsvm', 'sg', 'skl', 'gpr',
'blr')).intersection(clf.__tags__)):
raise SkipTest("Skipped %s because it is known to fail")
ok_(not (np.array_equal(matrix, matrix_swapped) and 0 in hits))
# this check is valid only if ties are not broken randomly
# like it is the case with SMLR
if not ('random_tie_breaking' in clf.__tags__
or # since __tags__ would not go that high up e.g. in
# <knn on SMLR non-0>
'SMLR' in str(clf)):
assert_array_equal(hits, np.diag(matrix_swapped)[ties_indices[::-1]])
# Second check is to just see if we didn't get an obvious bias and
# got 0 in one of the hits, although it is labile
if cfg.getboolean('tests', 'labile', default='yes'):
ok_(not 0 in hits)
def test_gideon_weird_case(self):
"""Test if MappedClassifier could handle a mapper altering number of samples
'The utter collapse' -- communicated by Peter J. Kohler
Desire to collapse all samples per each category in training
and testing sets, thus resulting only in a single
sample/category per training and per testing.
It is a peculiar scenario which pin points the problem that so
far mappers assumed not to change number of samples
"""
from mvpa2.mappers.fx import mean_group_sample
from mvpa2.clfs.knn import kNN
from mvpa2.mappers.base import ChainMapper
ds = datasets['uni2large'].copy()
#ds = ds[ds.sa.chunks < 9]
accs = []
k = 1 # for kNN
nf = 1 # for NFoldPartitioner
for i in xrange(1): # # of random runs
ds.samples = np.random.randn(*ds.shape)
#
# There are 3 ways to accomplish needed goal
#
# 0. Hard way: overcome the problem by manually
# pre-splitting/meaning in a loop
from mvpa2.clfs.transerror import ConfusionMatrix
partitioner = NFoldPartitioner(nf)
meaner = mean_group_sample(['targets', 'partitions'])
cm = ConfusionMatrix()
te = TransferMeasure(kNN(k),
Splitter('partitions'),
postproc=BinaryFxNode(mean_mismatch_error,
'targets'),
enable_ca=['stats'])
errors = []
for part in partitioner.generate(ds):
ds_meaned = meaner(part)
errors.append(np.asscalar(te(ds_meaned)))
cm += te.ca.stats
#print i, cm.stats['ACC']
accs.append(cm.stats['ACC'])
if False: # not yet working -- see _tent/allow_ch_nsamples
# branch for attempt to make it work
# 1. This is a "native way" IF we allow change of number
# of samples via _call to be done by MappedClassifier
# while operating solely on the mapped dataset
clf2 = MappedClassifier(
clf=kNN(k), #clf,
mapper=mean_group_sample(['targets', 'partitions']))
cv = CrossValidation(clf2,
NFoldPartitioner(nf),
postproc=None,
enable_ca=['stats'])
# meaning all should be ok since we should have ballanced
# sets across all chunks here
errors_native = cv(ds)
self.assertEqual(
np.max(np.abs(errors_native.samples[:, 0] - errors)), 0)
# 2. Work without fixes to MappedClassifier allowing
# change of # of samples
#
# CrossValidation will operate on a chain mapper which
# would perform necessary meaning first before dealing with
# kNN cons: .stats would not be exposed since ChainMapper
# doesn't expose them from ChainMapper (yet)
if __debug__ and 'ENFORCE_CA_ENABLED' in debug.active:
raise SkipTest("Known to fail while trying to enable "
"training_stats for the ChainMapper")
cv2 = CrossValidation(ChainMapper(
[mean_group_sample(['targets', 'partitions']),
kNN(k)],
space='targets'),
NFoldPartitioner(nf),
postproc=None)
errors_native2 = cv2(ds)
self.assertEqual(
np.max(np.abs(errors_native2.samples[:, 0] - errors)), 0)
# All of the ways should provide the same results
#print i, np.max(np.abs(errors_native.samples[:,0] - errors)), \
# np.max(np.abs(errors_native2.samples[:,0] - errors))
if False: # just to investigate the distribution if we have enough iterations
import pylab as pl
uaccs = np.unique(accs)
step = np.asscalar(np.unique(np.round(uaccs[1:] - uaccs[:-1], 4)))
bins = np.linspace(0., 1., np.round(1. / step + 1))
xx = pl.hist(accs, bins=bins, align='left')
pl.xlim((0. - step / 2, 1. + step / 2))
