def __test_matthias_question(self): rfe_clf = LinearCSVMC(C=1) rfesvm_split = SplitClassifier(rfe_clf) clf = \ FeatureSelectionClassifier( clf = LinearCSVMC(C=1), feature_selection = RFE( sensitivity_analyzer = rfesvm_split.get_sensitivity_analyzer( combiner=first_axis_mean, transformer=np.abs), transfer_error=ConfusionBasedError( rfesvm_split, confusion_state="confusion"), stopping_criterion=FixedErrorThresholdStopCrit(0.20), feature_selector=FractionTailSelector( 0.2, mode='discard', tail='lower'), update_sensitivity=True)) no_permutations = 1000 permutator = AttributePermutator('targets', count=no_permutations) cv = CrossValidation(clf, NFoldPartitioner(), null_dist=MCNullDist(permutator, tail='left'), enable_ca=['stats']) error = cv(datasets['uni2small']) self.assertTrue(error < 0.4) self.assertTrue(cv.ca.null_prob < 0.05)
def test_fixed_error_threshold_stop_crit(self): """Test stopping criterion""" stopcrit = FixedErrorThresholdStopCrit(0.5) self.assertTrue(stopcrit([]) == False) self.assertTrue(stopcrit([0.8, 0.9, 0.5]) == False) self.assertTrue(stopcrit([0.8, 0.9, 0.4]) == True) # only last error has to be below to stop self.assertTrue(stopcrit([0.8, 0.4, 0.6]) == False)
def test_multi_stop_crit(self): """Test multiple stop criteria""" stopcrit = MultiStopCrit( [FixedErrorThresholdStopCrit(0.5), NBackHistoryStopCrit(steps=4)]) # default 'or' mode # nback triggers self.assertTrue(stopcrit([1, 0.9, 0.8] + [0.9] * 4) == True) # threshold triggers self.assertTrue(stopcrit([1, 0.9, 0.2]) == True) # alternative 'and' mode stopcrit = MultiStopCrit( [FixedErrorThresholdStopCrit(0.5), NBackHistoryStopCrit(steps=4)], mode='and') # nback triggers not self.assertTrue(stopcrit([1, 0.9, 0.8] + [0.9] * 4) == False) # threshold triggers not self.assertTrue(stopcrit([1, 0.9, 0.2]) == False) # only both satisfy self.assertTrue(stopcrit([1, 0.9, 0.4] + [0.4] * 4) == True)