def __test_matthias_question(self):
rfe_clf = LinearCSVMC(C=1)
rfesvm_split = SplitClassifier(rfe_clf)
clf = \
FeatureSelectionClassifier(
clf = LinearCSVMC(C=1),
feature_selection = RFE(
sensitivity_analyzer = rfesvm_split.get_sensitivity_analyzer(
combiner=first_axis_mean,
transformer=np.abs),
transfer_error=ConfusionBasedError(
rfesvm_split,
confusion_state="confusion"),
stopping_criterion=FixedErrorThresholdStopCrit(0.20),
feature_selector=FractionTailSelector(
0.2, mode='discard', tail='lower'),
update_sensitivity=True))
no_permutations = 1000
permutator = AttributePermutator('targets', count=no_permutations)
cv = CrossValidation(clf,
NFoldPartitioner(),
null_dist=MCNullDist(permutator, tail='left'),
enable_ca=['stats'])
error = cv(datasets['uni2small'])
self.assertTrue(error < 0.4)
self.assertTrue(cv.ca.null_prob < 0.05)
def test_fixed_error_threshold_stop_crit(self):
"""Test stopping criterion"""
stopcrit = FixedErrorThresholdStopCrit(0.5)
self.assertTrue(stopcrit([]) == False)
self.assertTrue(stopcrit([0.8, 0.9, 0.5]) == False)
self.assertTrue(stopcrit([0.8, 0.9, 0.4]) == True)
# only last error has to be below to stop
self.assertTrue(stopcrit([0.8, 0.4, 0.6]) == False)
def test_multi_stop_crit(self):
"""Test multiple stop criteria"""
stopcrit = MultiStopCrit(
[FixedErrorThresholdStopCrit(0.5),
NBackHistoryStopCrit(steps=4)])
# default 'or' mode
# nback triggers
self.assertTrue(stopcrit([1, 0.9, 0.8] + [0.9] * 4) == True)
# threshold triggers
self.assertTrue(stopcrit([1, 0.9, 0.2]) == True)
# alternative 'and' mode
stopcrit = MultiStopCrit(
[FixedErrorThresholdStopCrit(0.5),
NBackHistoryStopCrit(steps=4)],
mode='and')
# nback triggers not
self.assertTrue(stopcrit([1, 0.9, 0.8] + [0.9] * 4) == False)
# threshold triggers not
self.assertTrue(stopcrit([1, 0.9, 0.2]) == False)
# only both satisfy
self.assertTrue(stopcrit([1, 0.9, 0.4] + [0.4] * 4) == True)