def test_confusion_based_error(self, l_clf): train = datasets['uni2medium'] train = train[train.sa.train == 1] # to check if we fail to classify for 3 labels test3 = datasets['uni3medium'] test3 = test3[test3.sa.train == 1] err = ConfusionBasedError(clf=l_clf) terr = TransferMeasure(l_clf, Splitter('train', attr_values=[1, 1]), postproc=BinaryFxNode(mean_mismatch_error, 'targets')) self.assertRaises(UnknownStateError, err, None) """Shouldn't be able to access the state yet""" l_clf.train(train) e, te = err(None), terr(train) te = np.asscalar(te) self.assertTrue( abs(e - te) < 1e-10, msg="ConfusionBasedError (%.2g) should be equal to TransferError " "(%.2g) on traindataset" % (e, te)) # this will print nasty WARNING but it is ok -- it is just checking code # NB warnings are not printed while doing whole testing warning("Don't worry about the following warning.") if 'multiclass' in l_clf.__tags__: self.assertFalse(terr(test3) is None) # try copying the beast terr_copy = copy(terr)
def test_null_dist_prob(self, l_clf): train = datasets['uni2medium'] num_perm = 10 permutator = AttributePermutator('targets', count=num_perm, limit='chunks') # define class to estimate NULL distribution of errors # use left tail of the distribution since we use MeanMatchFx as error # function and lower is better terr = TransferMeasure(l_clf, Repeater(count=2), postproc=BinaryFxNode(mean_mismatch_error, 'targets'), null_dist=MCNullDist(permutator, tail='left')) # check reasonable error range err = terr(train) self.assertTrue(np.mean(err) < 0.4) # Lets do the same for CVTE cvte = CrossValidation(l_clf, OddEvenPartitioner(), null_dist=MCNullDist(permutator, tail='left', enable_ca=['dist_samples' ]), postproc=mean_sample()) cv_err = cvte(train) # check that the result is highly significant since we know that the # data has signal null_prob = np.asscalar(terr.ca.null_prob) if cfg.getboolean('tests', 'labile', default='yes'): self.assertTrue( null_prob <= 0.1, msg="Failed to check that the result is highly significant " "(got %f) since we know that the data has signal" % null_prob) self.assertTrue( np.asscalar(cvte.ca.null_prob) <= 0.1, msg="Failed to check that the result is highly significant " "(got p(cvte)=%f) since we know that the data has signal" % np.asscalar(cvte.ca.null_prob)) # we should be able to access the actual samples of the distribution # yoh: why it is 3D really? # mih: because these are the distribution samples for the ONE error # collapsed into ONE value across all folds. It will also be # 3d if the return value of the measure isn't a scalar and it is # not collapsed across folds. it simply corresponds to the shape # of the output dataset of the respective measure (+1 axis) # Some permutations could have been skipped since classifier failed # to train due to degenerate situation etc, thus accounting for them self.assertEqual(cvte.null_dist.ca.dist_samples.shape[2], num_perm - cvte.null_dist.ca.skipped)
def _train(self, dataset): pmeasure = ProxyMeasure( self.lrn, postproc=BinaryFxNode(self.errorfx, self.lrn.space), skip_train=True # do not train since fmeasure will ) # First we need to replicate our RFE construct but this time # with pmeasure for the classifier rfe = RFE( self.fmeasure, pmeasure, Splitter('partitions'), fselector=self.fselector, bestdetector=None, train_pmeasure=False, stopping_criterion=None, # full "track" update_sensitivity=self.update_sensitivity, enable_ca=['errors', 'nfeatures']) errors, nfeatures = [], [] if __debug__: debug("RFEC", "Stage 1: initial nested CV/RFE for %s", (dataset, )) for partition in self.partitioner.generate(dataset): rfe.train(partition) errors.append(rfe.ca.errors) nfeatures.append(rfe.ca.nfeatures) # mean errors across splits and find optimal number errors_mean = np.mean(errors, axis=0) nfeatures_mean = np.mean(nfeatures, axis=0) # we will take the "mean location" of the min to stay # within the most 'stable' choice mins_idx = np.where(errors_mean == np.min(errors_mean))[0] min_idx = mins_idx[int(len(mins_idx) / 2)] min_error = errors_mean[min_idx] assert (min_error == np.min(errors_mean)) nfeatures_min = nfeatures_mean[min_idx] if __debug__: debug( "RFEC", "Choosing among %d choices to have %d features with " "mean error=%.2g (initial mean error %.2g)", (len(mins_idx), nfeatures_min, min_error, errors_mean[0])) self.nfeatures_min = nfeatures_min if __debug__: debug( "RFEC", "Stage 2: running RFE on full training dataset to " "distil best %d features" % nfeatures_min) super(SplitRFE, self)._train(dataset)
def test_james_problem_multiclass(self): percent = 80 dataset = datasets['uni4large'] #dataset = dataset[:, dataset.a.nonbogus_features] rfesvm_split = LinearCSVMC() fs = \ RFE(rfesvm_split.get_sensitivity_analyzer( postproc=ChainMapper([ #FxMapper('features', l2_normed), #FxMapper('samples', np.mean), #FxMapper('samples', np.abs) FxMapper('features', lambda x: np.argsort(np.abs(x))), #maxofabs_sample() mean_sample() ])), ProxyMeasure(rfesvm_split, postproc=BinaryFxNode(mean_mismatch_error, 'targets')), Splitter('train'), fselector=FractionTailSelector( percent / 100.0, mode='select', tail='upper'), update_sensitivity=True) clf = FeatureSelectionClassifier( LinearCSVMC(), # on features selected via RFE fs) # update sensitivity at each step (since we're not using the # same CLF as sensitivity analyzer) class StoreResults(object): def __init__(self): self.storage = [] def __call__(self, data, node, result): self.storage.append((node.measure.mapper.ca.history, node.measure.mapper.ca.errors)), cv_storage = StoreResults() cv = CrossValidation(clf, NFoldPartitioner(), postproc=mean_sample(), callback=cv_storage, enable_ca=['stats']) #cv = SplitClassifier(clf) try: error = cv(dataset).samples.squeeze() except Exception, e: self.fail('CrossValidation cannot handle classifier with RFE ' 'feature selection. Got exception: %s' % (e, ))
def test_clf_transfer_measure(self): # and now on a classifier clf = SMLR() enode = BinaryFxNode(mean_mismatch_error, 'targets') tm = TransferMeasure(clf, Splitter('chunks', count=2), enable_ca=['stats']) res = tm(self.dataset) manual_error = np.mean(res.samples.squeeze() != res.sa.targets) postproc_error = enode(res) tm_err = TransferMeasure(clf, Splitter('chunks', count=2), postproc=enode) auto_error = tm_err(self.dataset) ok_(manual_error == postproc_error.samples[0, 0])
def test_pseudo_cv_measure(self): clf = SMLR() enode = BinaryFxNode(mean_mismatch_error, 'targets') tm = TransferMeasure(clf, Splitter('partitions'), postproc=enode) cvgen = NFoldPartitioner() rm = RepeatedMeasure(tm, cvgen) res = rm(self.dataset) # one error per fold assert_equal(res.shape, (len(self.dataset.sa['chunks'].unique), 1)) # we can do the same with Crossvalidation cv = CrossValidation(clf, cvgen, enable_ca=['stats', 'training_stats', 'datasets']) res = cv(self.dataset) assert_equal(res.shape, (len(self.dataset.sa['chunks'].unique), 1))
def test_james_problem(self): percent = 80 dataset = datasets['uni2small'] rfesvm_split = LinearCSVMC() fs = \ RFE(rfesvm_split.get_sensitivity_analyzer(), ProxyMeasure(rfesvm_split, postproc=BinaryFxNode(mean_mismatch_error, 'targets')), Splitter('train'), fselector=FractionTailSelector( percent / 100.0, mode='select', tail='upper'), update_sensitivity=True) clf = FeatureSelectionClassifier( LinearCSVMC(), # on features selected via RFE fs) # update sensitivity at each step (since we're not using the # same CLF as sensitivity analyzer) class StoreResults(object): def __init__(self): self.storage = [] def __call__(self, data, node, result): self.storage.append((node.measure.mapper.ca.history, node.measure.mapper.ca.errors)), cv_storage = StoreResults() cv = CrossValidation(clf, NFoldPartitioner(), postproc=mean_sample(), callback=cv_storage, enable_ca=['confusion']) # TODO -- it is stats #cv = SplitClassifier(clf) try: error = cv(dataset).samples.squeeze() except Exception as e: self.fail('CrossValidation cannot handle classifier with RFE ' 'feature selection. Got exception: %s' % (e, )) assert (len(cv_storage.storage) == len(dataset.sa['chunks'].unique)) assert (len(cv_storage.storage[0]) == 2) assert (len(cv_storage.storage[0][0]) == dataset.nfeatures) self.assertTrue(error < 0.2)
def test_single_class(self, clf): """Test if binary and multiclass can handle single class training/testing """ ds = datasets['uni2small'] ds = ds[ds.sa.targets == 'L0'] # only 1 label assert(ds.sa['targets'].unique == ['L0']) ds_ = list(OddEvenPartitioner().generate(ds))[0] # Here is our "nice" 0.6 substitute for TransferError: trerr = TransferMeasure(clf, Splitter('train'), postproc=BinaryFxNode(mean_mismatch_error, 'targets')) try: err = np.asscalar(trerr(ds_)) except Exception, e: self.fail(str(e))
def test_ifs(self, svm): # measure for feature selection criterion and performance assesment # use the SAME clf! errorfx = mean_mismatch_error fmeasure = CrossValidation(svm, NFoldPartitioner(), postproc=mean_sample()) pmeasure = ProxyMeasure(svm, postproc=BinaryFxNode(errorfx, 'targets')) ifs = IFS(fmeasure, pmeasure, Splitter('purpose', attr_values=['train', 'test']), fselector=\ # go for lower tail selection as data_measure will return # errors -> low is good FixedNElementTailSelector(1, tail='lower', mode='select'), ) wdata = self.get_data() wdata.sa['purpose'] = np.repeat('train', len(wdata)) tdata = self.get_data() tdata.sa['purpose'] = np.repeat('test', len(tdata)) ds = vstack((wdata, tdata)) orig_nfeatures = ds.nfeatures ifs.train(ds) resds = ifs(ds) # fail if orig datasets are changed self.assertTrue(ds.nfeatures == orig_nfeatures) # check that the features set with the least error is selected self.assertTrue(len(ifs.ca.errors)) e = np.array(ifs.ca.errors) self.assertTrue(resds.nfeatures == e.argmin() + 1) # repeat with dataset where selection order is known wsignal = datasets['dumb2'].copy() wsignal.sa['purpose'] = np.repeat('train', len(wsignal)) tsignal = datasets['dumb2'].copy() tsignal.sa['purpose'] = np.repeat('test', len(tsignal)) signal = vstack((wsignal, tsignal)) ifs.train(signal) resds = ifs(signal) self.assertTrue((resds.samples[:, 0] == signal.samples[:, 0]).all())
def test_cached_kernel_different_datasets(self): skip_if_no_external('shogun', ver_dep='shogun:rev', min_version=4455) # Inspired by the problem Swaroop ran into k = LinearSGKernel(normalizer_cls=False) k_ = LinearSGKernel(normalizer_cls=False) # to be cached ck = CachedKernel(k_) clf = sgSVM(svm_impl='libsvm', kernel=k, C=-1) clf_ = sgSVM(svm_impl='libsvm', kernel=ck, C=-1) cvte = CrossValidation(clf, NFoldPartitioner()) cvte_ = CrossValidation(clf_, NFoldPartitioner()) postproc=BinaryFxNode(mean_mismatch_error, 'targets') te = ProxyMeasure(clf, postproc=postproc) te_ = ProxyMeasure(clf_, postproc=postproc) for r in xrange(2): ds1 = datasets['uni2medium'] errs1 = cvte(ds1) ck.compute(ds1) ok_(ck._recomputed) errs1_ = cvte_(ds1) ok_(~ck._recomputed) assert_array_equal(errs1, errs1_) ds2 = datasets['uni3small'] errs2 = cvte(ds2) ck.compute(ds2) ok_(ck._recomputed) errs2_ = cvte_(ds2) ok_(~ck._recomputed) assert_array_equal(errs2, errs2_) ssel = np.round(datasets['uni2large'].samples[:5, 0]).astype(int) te.train(datasets['uni3small'][::2]) terr = np.asscalar(te(datasets['uni3small'][ssel])) te_.train(datasets['uni3small'][::2]) terr_ = np.asscalar(te_(datasets['uni3small'][ssel])) ok_(~ck._recomputed) ok_(terr == terr_)
def __init__(self, learner, generator=None, errorfx=mean_mismatch_error, splitter=None, **kwargs): """ Parameters ---------- learner : Learner Any trainable node that shall be run on the dataset folds. generator : Node, optional Generator used to resample the input dataset into multiple instances (i.e. partitioning it). The number of datasets yielded by this generator determines the number of cross-validation folds. IMPORTANT: The ``space`` of this generator determines the attribute that will be used to split all generated datasets into training and testing sets. If None provided, a single original dataset will be passed to the ``splitter`` as is errorfx : Node or callable Custom implementation of an error function. The callable needs to accept two arguments (1. predicted values, 2. target values). If not a Node, it gets wrapped into a `BinaryFxNode`. splitter : Splitter or None A Splitter instance to split the dataset into training and testing part. The first split will be used for training and the second for testing -- all other splits will be ignored. If None, a default splitter is auto-generated using the ``space`` setting of the ``generator``. If no ``generator`` provided, splitter uses 'partitions' sample attribute. The default splitter is configured to return the ``1``-labeled partition of the input dataset at first, and the ``2``-labeled partition second. This behavior corresponds to most Partitioners that label the taken-out portion ``2`` and the remainder with ``1``. """ # compile the appropriate repeated measure to do cross-validation from # pieces if errorfx is not None: # error node -- postproc of transfer measure if isinstance(errorfx, Node): enode = errorfx else: # wrap into BinaryFxNode enode = BinaryFxNode(errorfx, learner.get_space()) else: enode = None if splitter is None: # default splitter splits into "1" and "2" partition. # that will effectively ignore 'deselected' samples (e.g. by # Balancer). It is done this way (and not by ignoring '0' samples # because it is guaranteed to yield two splits) and is more likely # to fail in visible ways if the attribute does not have 0,1,2 # values at all (i.e. a literal train/test/spareforlater attribute) splitter = Splitter( generator.get_space() if generator else 'partitions', attr_values=(1, 2)) # transfer measure to wrap the learner # splitter used the output space of the generator to know what to split tm = TransferMeasure(learner, splitter, postproc=enode) space = kwargs.pop('space', 'sa.cvfolds') # and finally the repeated measure to perform the x-val RepeatedMeasure.__init__(self, tm, generator=generator, space=space, **kwargs) for ca in ['stats', 'training_stats']: if self.ca.is_enabled(ca): # enforce ca if requested tm.ca.enable(ca) if self.ca.is_enabled('training_stats'): # also enable training stats in the learner learner.ca.enable('training_stats')
def test_rfe(self, clf): # sensitivity analyser and transfer error quantifier use the SAME clf! sens_ana = clf.get_sensitivity_analyzer(postproc=maxofabs_sample()) pmeasure = ProxyMeasure(clf, postproc=BinaryFxNode(mean_mismatch_error, 'targets')) cvmeasure = CrossValidation(clf, NFoldPartitioner(), errorfx=mean_mismatch_error, postproc=mean_sample()) rfesvm_split = SplitClassifier(clf, OddEvenPartitioner()) # explore few recipes for rfe, data in [ # because the clf is already trained when computing the sensitivity # map, prevent retraining for transfer error calculation # Use absolute of the svm weights as sensitivity (RFE(sens_ana, pmeasure, Splitter('train'), fselector=FixedNElementTailSelector(1), train_pmeasure=False), self.get_data()), # use cross-validation within training to get error for the stopping point # but use full training data to derive sensitivity ( RFE( sens_ana, cvmeasure, Repeater( 2 ), # give the same full dataset to sens_ana and cvmeasure fselector=FractionTailSelector(0.70, mode='select', tail='upper'), train_pmeasure=True), normal_feature_dataset(perlabel=20, nchunks=5, nfeatures=200, nonbogus_features=[0, 1], snr=1.5)), # use cross-validation (via SplitClassifier) and get mean # of normed sensitivities across those splits ( RFE( rfesvm_split.get_sensitivity_analyzer( postproc=ChainMapper([ FxMapper('features', l2_normed), FxMapper('samples', np.mean), FxMapper('samples', np.abs) ])), ConfusionBasedError(rfesvm_split, confusion_state='stats'), Repeater( 2), # we will use the same full cv-training dataset fselector=FractionTailSelector(0.50, mode='select', tail='upper'), stopping_criterion=NBackHistoryStopCrit( BestDetector(), 10), train_pmeasure= False, # we just extract it from existing confusion update_sensitivity=True), normal_feature_dataset(perlabel=28, nchunks=7, nfeatures=200, nonbogus_features=[0, 1], snr=1.5)) ]: # prep data # data = datasets['uni2medium'] data_nfeatures = data.nfeatures rfe.train(data) resds = rfe(data) # fail if orig datasets are changed self.assertTrue(data.nfeatures == data_nfeatures) # check that the features set with the least error is selected if len(rfe.ca.errors): e = np.array(rfe.ca.errors) if isinstance(rfe._fselector, FixedNElementTailSelector): self.assertTrue(resds.nfeatures == data_nfeatures - e.argmin()) else: imin = np.argmin(e) if 'does_feature_selection' in clf.__tags__: # if clf is smart it might figure it out right away assert_array_less(imin, len(e)) else: # in this case we can even check if we had actual # going down/up trend... although -- why up??? self.assertTrue(1 < imin < len(e) - 1) else: self.assertTrue(resds.nfeatures == data_nfeatures) # silly check if nfeatures is in decreasing order nfeatures = np.array(rfe.ca.nfeatures).copy() nfeatures.sort() self.assertTrue((nfeatures[::-1] == rfe.ca.nfeatures).all()) # check if history has elements for every step self.assertTrue( set(rfe.ca.history) == set(range(len(np.array( rfe.ca.errors))))) # Last (the largest number) can be present multiple times even # if we remove 1 feature at a time -- just need to stop well # in advance when we have more than 1 feature left ;) self.assertTrue(rfe.ca.nfeatures[-1] == len( np.where(rfe.ca.history == max(rfe.ca.history))[0]))
raise AssertionError, \ "Failed to load due to %r" % (e,) ok_(isinstance(lrn_, Classifier)) # Verify that we have the same ca enabled # XXX FAILS atm! #ok_(set(lrn.ca.enabled) == set(lrn_.ca.enabled)) # lets choose a dataset dsname, errorfx = \ {False: ('uni2large', mean_mismatch_error), True: ('sin_modulated', corr_error)}\ ['regression' in lrn.__tags__] ds = datasets[dsname] splitter = Splitter('train') postproc = BinaryFxNode(errorfx, 'targets') te = TransferMeasure(lrn, splitter, postproc=postproc) te_ = TransferMeasure(lrn_, splitter, postproc=postproc) error = te(ds) error_ = te_(ds) if len(set(['swig', 'rpy2']).intersection(lrn.__tags__)): raise SkipTest("Trained swigged and R-interfaced classifiers can't " "be stored/reloaded yet") # now lets store/reload the trained one try: h5save(fname, lrn_) except Exception, e: raise AssertionError, \
def _train(self, dataset): pmeasure = ProxyMeasure( self.lrn, postproc=BinaryFxNode(self.errorfx, self.lrn.space), skip_train=not self. train_pmeasure # do not train since fmeasure will ) # First we need to replicate our RFE construct but this time # with pmeasure for the classifier rfe = RFE( self.fmeasure, pmeasure, Splitter('partitions'), fselector=self.fselector, bestdetector=None, train_pmeasure=self.train_pmeasure, stopping_criterion=None, # full "track" update_sensitivity=self.update_sensitivity, enable_ca=['errors', 'nfeatures']) errors, nfeatures = [], [] if __debug__: debug("RFEC", "Stage 1: initial nested CV/RFE for %s", (dataset, )) if self.nproc != 1 and externals.exists('joblib'): nested_results = jl.Parallel(self.nproc)( jl.delayed(_process_partition)(rfe, partition) for partition in self.partitioner.generate(dataset)) else: nested_results = [ _process_partition(rfe, partition) for partition in self.partitioner.generate(dataset) ] # unzip errors = [x[0] for x in nested_results] nfeatures = [x[1] for x in nested_results] self.ca.nested_nfeatures = nfeatures self.ca.nested_errors = errors # mean errors across splits and find optimal number errors_mean = np.mean(errors, axis=0) nfeatures_mean = np.mean(nfeatures, axis=0) # we will take the "mean location" of the min to stay # within the most 'stable' choice mins_idx = np.where(errors_mean == np.min(errors_mean))[0] min_idx = mins_idx[int(len(mins_idx) / 2)] min_error = errors_mean[min_idx] assert (min_error == np.min(errors_mean)) nfeatures_min = nfeatures_mean[min_idx] if __debug__: debug( "RFEC", "Choosing among %d choices to have %d features with " "mean error=%.2g (initial mean error %.2g)", (len(mins_idx), nfeatures_min, min_error, errors_mean[0])) self.nfeatures_min = nfeatures_min if __debug__: debug( "RFEC", "Stage 2: running RFE on full training dataset to " "obtain the best %d features" % nfeatures_min) super(SplitRFE, self)._train(dataset)
def test_retrainables(self, clf): # XXX we agreed to not worry about this for the initial 0.6 release raise SkipTest # we need a copy since will tune its internals later on clf = clf.clone() clf.ca.change_temporarily( enable_ca=['estimates'], # ensure that it does do predictions # while training disable_ca=['training_stats']) clf_re = clf.clone() # TODO: .retrainable must have a callback to call smth like # _set_retrainable clf_re._set_retrainable(True) # need to have high snr so we don't 'cope' with problematic # datasets since otherwise unittests would fail. dsargs = { 'perlabel': 50, 'nlabels': 2, 'nfeatures': 5, 'nchunks': 1, 'nonbogus_features': [2, 4], 'snr': 5.0 } ## !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # NB datasets will be changed by the end of testing, so if # are to change to use generic datasets - make sure to copy # them here ds = deepcopy(datasets['uni2large']) clf.untrain() clf_re.untrain() trerr = TransferMeasure(clf, Splitter('train'), postproc=BinaryFxNode(mean_mismatch_error, 'targets')) trerr_re = TransferMeasure(clf_re, Splitter('train'), disable_ca=['training_stats'], postproc=BinaryFxNode( mean_mismatch_error, 'targets')) # Just check for correctness of retraining err_1 = np.asscalar(trerr(ds)) self.assertTrue( err_1 < 0.3, msg="We should test here on easy dataset. Got error of %s" % err_1) values_1 = clf.ca.estimates[:] # some times retraining gets into deeper optimization ;-) eps = 0.05 corrcoef_eps = 0.85 # just to get no failures... usually > 0.95 def batch_test(retrain=True, retest=True, closer=True): err = np.asscalar(trerr(ds)) err_re = np.asscalar(trerr_re(ds)) corr = np.corrcoef(clf.ca.estimates, clf_re.ca.estimates)[0, 1] corr_old = np.corrcoef(values_1, clf_re.ca.estimates)[0, 1] if __debug__: debug( 'TEST', "Retraining stats: errors %g %g corr %g " "with old error %g corr %g" % (err, err_re, corr, err_1, corr_old)) self.assertTrue(clf_re.ca.retrained == retrain, ("Must fully train", "Must retrain instead of full training")[retrain]) self.assertTrue(clf_re.ca.repredicted == retest, ("Must fully test", "Must retest instead of full testing")[retest]) self.assertTrue( corr > corrcoef_eps, msg="Result must be close to the one without retraining." " Got corrcoef=%s" % (corr)) if closer: self.assertTrue( corr >= corr_old, msg="Result must be closer to current without retraining" " than to old one. Got corrcoef=%s" % (corr_old)) # Check sequential retraining/retesting for i in xrange(3): flag = bool(i != 0) # ok - on 1st call we should train/test, then retrain/retest # and we can't compare for closinest to old result since # we are working on the same data/classifier batch_test(retrain=flag, retest=flag, closer=False) # should retrain nicely if we change a parameter if 'C' in clf.params: clf.params.C *= 0.1 clf_re.params.C *= 0.1 batch_test() elif 'sigma_noise' in clf.params: clf.params.sigma_noise *= 100 clf_re.params.sigma_noise *= 100 batch_test() else: raise RuntimeError, \ 'Please implement testing while changing some of the ' \ 'params for clf %s' % clf # should retrain nicely if we change kernel parameter if hasattr(clf, 'kernel_params') and len(clf.kernel_params): clf.kernel_params.gamma = 0.1 clf_re.kernel_params.gamma = 0.1 # retest is false since kernel got recomputed thus # can't expect to use the same kernel batch_test(retest=not ('gamma' in clf.kernel_params)) # should retrain nicely if we change labels permute = AttributePermutator('targets', assure=True) oldlabels = dstrain.targets[:] dstrain = permute(dstrain) self.assertTrue( (oldlabels != dstrain.targets).any(), msg="We should succeed at permutting -- now got the same targets") ds = vstack((dstrain, dstest)) batch_test() # Change labels in testing oldlabels = dstest.targets[:] dstest = permute(dstest) self.assertTrue( (oldlabels != dstest.targets).any(), msg="We should succeed at permutting -- now got the same targets") ds = vstack((dstrain, dstest)) batch_test() # should re-train if we change data # reuse trained SVM and its 'final' optimization point if not clf.__class__.__name__ in [ 'GPR' ]: # on GPR everything depends on the data ;-) oldsamples = dstrain.samples.copy() dstrain.samples[:] += dstrain.samples * 0.05 self.assertTrue((oldsamples != dstrain.samples).any()) ds = vstack((dstrain, dstest)) batch_test(retest=False) clf.ca.reset_changed_temporarily() # test retrain() # TODO XXX -- check validity clf_re.retrain(dstrain) self.assertTrue(clf_re.ca.retrained) clf_re.retrain(dstrain, labels=True) self.assertTrue(clf_re.ca.retrained) clf_re.retrain(dstrain, traindataset=True) self.assertTrue(clf_re.ca.retrained) # test repredict() clf_re.repredict(dstest.samples) self.assertTrue(clf_re.ca.repredicted) self.assertRaises(RuntimeError, clf_re.repredict, dstest.samples, labels=True) """for now retesting with anything changed makes no sense""" clf_re._set_retrainable(False)
def __init__(self, **kwargs): BinaryFxNode.__init__(self, fx=mean_mismatch_error, space='targets', **kwargs)
def __init__(self, fx, space, subj_space, **kwargs): self.subj_space = subj_space BinaryFxNode.__init__(self, fx, space, **kwargs)
def test_multiclass_ties(clf): if 'lars' in clf.__tags__: raise SkipTest("Known to crash while running this test") ds = _dsties1 # reassign data between ties, so we know that decision is data, not order driven ds_ = ds.copy(deep=True) ds_.samples[ds.a.ties_idx[1]] = ds.samples[ds.a.ties_idx[0]] ds_.samples[ds.a.ties_idx[0]] = ds.samples[ds.a.ties_idx[1]] ok_(np.any(ds_.samples != ds.samples)) clf_ = clf.clone() clf = clf.clone() clf.ca.enable(['estimates', 'predictions']) clf_.ca.enable(['estimates', 'predictions']) te = TransferMeasure(clf, Splitter('train'), postproc=BinaryFxNode(mean_mismatch_error, 'targets'), enable_ca=['stats']) te_ = TransferMeasure(clf_, Splitter('train'), postproc=BinaryFxNode(mean_mismatch_error, 'targets'), enable_ca=['stats']) te = CrossValidation(clf, NFoldPartitioner(), postproc=mean_sample(), enable_ca=['stats']) te_ = CrossValidation(clf_, NFoldPartitioner(), postproc=mean_sample(), enable_ca=['stats']) error = te(ds) matrix = te.ca.stats.matrix # if ties were broken randomly we should have got nearly the same # number of hits for tied targets ties_indices = [te.ca.stats.labels.index(c) for c in ds.a.ties] hits = np.diag(te.ca.stats.matrix)[ties_indices] # First check is to see if we swap data between tied labels we # are getting the same results if we permute labels accordingly, # i.e. that tie resolution is not dependent on the labels order # but rather on the data te_(ds_) matrix_swapped = te_.ca.stats.matrix if False: #0 in hits: print clf, matrix, matrix_swapped print clf.ca.estimates[:, 2] - clf.ca.estimates[:, 0] #print clf.ca.estimates # TODO: for now disabled all the non-compliant ones to pass the # tests. For visibility decided to skip them instead of just # exclusion and skipping only here to possibly catch crashes # which might happen before if len( set(('libsvm', 'sg', 'skl', 'gpr', 'blr')).intersection(clf.__tags__)): raise SkipTest("Skipped %s because it is known to fail") ok_(not (np.array_equal(matrix, matrix_swapped) and 0 in hits)) # this check is valid only if ties are not broken randomly # like it is the case with SMLR if not ('random_tie_breaking' in clf.__tags__ or # since __tags__ would not go that high up e.g. in # <knn on SMLR non-0> 'SMLR' in str(clf)): assert_array_equal(hits, np.diag(matrix_swapped)[ties_indices[::-1]]) # Second check is to just see if we didn't get an obvious bias and # got 0 in one of the hits, although it is labile if cfg.getboolean('tests', 'labile', default='yes'): ok_(not 0 in hits)
def test_gideon_weird_case(self): """Test if MappedClassifier could handle a mapper altering number of samples 'The utter collapse' -- communicated by Peter J. Kohler Desire to collapse all samples per each category in training and testing sets, thus resulting only in a single sample/category per training and per testing. It is a peculiar scenario which pin points the problem that so far mappers assumed not to change number of samples """ from mvpa2.mappers.fx import mean_group_sample from mvpa2.clfs.knn import kNN from mvpa2.mappers.base import ChainMapper ds = datasets['uni2large'].copy() #ds = ds[ds.sa.chunks < 9] accs = [] k = 1 # for kNN nf = 1 # for NFoldPartitioner for i in xrange(1): # # of random runs ds.samples = np.random.randn(*ds.shape) # # There are 3 ways to accomplish needed goal # # 0. Hard way: overcome the problem by manually # pre-splitting/meaning in a loop from mvpa2.clfs.transerror import ConfusionMatrix partitioner = NFoldPartitioner(nf) meaner = mean_group_sample(['targets', 'partitions']) cm = ConfusionMatrix() te = TransferMeasure(kNN(k), Splitter('partitions'), postproc=BinaryFxNode(mean_mismatch_error, 'targets'), enable_ca=['stats']) errors = [] for part in partitioner.generate(ds): ds_meaned = meaner(part) errors.append(np.asscalar(te(ds_meaned))) cm += te.ca.stats #print i, cm.stats['ACC'] accs.append(cm.stats['ACC']) if False: # not yet working -- see _tent/allow_ch_nsamples # branch for attempt to make it work # 1. This is a "native way" IF we allow change of number # of samples via _call to be done by MappedClassifier # while operating solely on the mapped dataset clf2 = MappedClassifier( clf=kNN(k), #clf, mapper=mean_group_sample(['targets', 'partitions'])) cv = CrossValidation(clf2, NFoldPartitioner(nf), postproc=None, enable_ca=['stats']) # meaning all should be ok since we should have ballanced # sets across all chunks here errors_native = cv(ds) self.assertEqual( np.max(np.abs(errors_native.samples[:, 0] - errors)), 0) # 2. Work without fixes to MappedClassifier allowing # change of # of samples # # CrossValidation will operate on a chain mapper which # would perform necessary meaning first before dealing with # kNN cons: .stats would not be exposed since ChainMapper # doesn't expose them from ChainMapper (yet) if __debug__ and 'ENFORCE_CA_ENABLED' in debug.active: raise SkipTest("Known to fail while trying to enable " "training_stats for the ChainMapper") cv2 = CrossValidation(ChainMapper( [mean_group_sample(['targets', 'partitions']), kNN(k)], space='targets'), NFoldPartitioner(nf), postproc=None) errors_native2 = cv2(ds) self.assertEqual( np.max(np.abs(errors_native2.samples[:, 0] - errors)), 0) # All of the ways should provide the same results #print i, np.max(np.abs(errors_native.samples[:,0] - errors)), \ # np.max(np.abs(errors_native2.samples[:,0] - errors)) if False: # just to investigate the distribution if we have enough iterations import pylab as pl uaccs = np.unique(accs) step = np.asscalar(np.unique(np.round(uaccs[1:] - uaccs[:-1], 4))) bins = np.linspace(0., 1., np.round(1. / step + 1)) xx = pl.hist(accs, bins=bins, align='left') pl.xlim((0. - step / 2, 1. + step / 2))