def test_with_confusion_matrices_on_biclass(self): learner = random_learner ds = data.Table("monks-1") pt = testing.proportion_test([learner], ds, times=1) cm = scoring.confusion_matrices(pt, class_index=1) scores = self.score(cm) self.assertIsInstance(scores, list)
def test_with_confusion_matrix_on_multiclass(self): learner = random_learner ds = data.Table("iris") pt = testing.proportion_test([learner], ds, times=1) cm = scoring.confusion_matrices(pt, class_index=1) scores = self.score(cm[0]) self.assertIsInstance(scores, float)
def test_with_test_results_on_multiclass(self): learner = random_learner ds = data.Table("iris") pt = testing.proportion_test([learner], ds, times=1) scores = self.score(pt) self.assertIsInstance(scores, list)
def test_construct_confusion_matrix_from_multiclass(self): learner = random_learner ds = data.Table("iris") pt = testing.proportion_test([learner], ds, times=1) cm = scoring.confusion_matrices(pt) self.assertTrue(isinstance(cm[0], list))
def test_construct_confusion_matrix_from_biclass(self): learner = random_learner ds = data.Table("monks-1") pt = testing.proportion_test([learner], ds, times=1) cm = scoring.confusion_matrices(pt, class_index=1) self.assertTrue(hasattr(cm[0], "TP"))
def test_auc_on_monks(self): ds = data.Table("monks-1") cv = testing.cross_validation([self.learner], ds, folds=5) pt = testing.proportion_test([self.learner], ds, times=1) auc = scoring.AUC(cv) self.assertEqual(len(auc), 1) auc = scoring.AUC(pt) self.assertEqual(len(auc), 1)
def test_ca_from_confusion_matrix_for_classification_on_iris_se(self): ds = data.Table("iris") pt = testing.proportion_test([self.learner], ds, times=1) self.assertEqual(pt.number_of_iterations, 1) ca = scoring.CA(pt, report_se=True) self.assertEqual(len(ca), 1)
def run_tests(datasets, measures, tests, iterations=10): for ds, ds_name in datasets: for t, t_name in tests: print "Testing %s on %s" % (t_name, ds_name) test_results = [t(random_learner, ds) for _ in xrange(iterations)] test(measures, test_results) datasets = ( (data.Table("iris"), "Iris"), (data.Table("monks-1"), "Monks") ) measures = ( (lambda x:ca(x), "CA"), (lambda x:ca(x, report_se=False, ignore_weights=False), "CA-SE-W"), (lambda x:ca(x, report_se=True, ignore_weights=False), "CA+SE-W"), (lambda x:[lambda x:[None]], ""), (lambda x:ca(x, report_se=False, ignore_weights=True), "CA-SE+W"), (lambda x:ca(x, report_se=True, ignore_weights=True), "CA+SE+W"), (lambda x:[lambda x:[None]], ""), ) tests = ( (lambda l, ds: testing.cross_validation([l],ds), "CV"), (lambda l, ds: testing.proportion_test([l], ds, .7, 1), "Proportion test"), (lambda l, ds: scoring.confusion_matrices(testing.proportion_test([l], ds, .7, 1)), "Confusion matrix"), ) run_tests(datasets, measures, tests)
from Orange import data from Orange.classification import svm vehicle = data.Table("vehicle.tab") svm_easy = svm.SVMLearnerEasy(name="svm easy", folds=3) svm_normal = svm.SVMLearner(name="svm") learners = [svm_easy, svm_normal] from Orange.evaluation import testing, scoring results = testing.proportion_test(learners, vehicle, times=1) print "Name CA AUC" for learner,CA,AUC in zip(learners, scoring.CA(results), scoring.AUC(results)): print "%-8s %.2f %.2f" % (learner.name, CA, AUC)
from Orange import data from Orange.classification import svm vehicle = data.Table("vehicle.tab") svm_easy = svm.SVMLearnerEasy(name="svm easy", folds=3) svm_normal = svm.SVMLearner(name="svm") learners = [svm_easy, svm_normal] from Orange.evaluation import testing, scoring results = testing.proportion_test(learners, vehicle, times=1) print "Name CA AUC" for learner, CA, AUC in zip(learners, scoring.CA(results), scoring.AUC(results)): print "%-8s %.2f %.2f" % (learner.name, CA, AUC)
print "%8.4f"*n % tuple(m[0](tr)[0] for m in measures) print def run_tests(datasets, measures, tests, iterations=10): for ds, ds_name in datasets: for t, t_name in tests: print "Testing %s on %s" % (t_name, ds_name) test_results = [t(random_learner, ds) for _ in xrange(iterations)] test(measures, test_results) datasets = ( (data.Table("iris"), "Iris"), (data.Table("monks-1"), "Monks") ) measures = ( (lambda x:auc(x), "AUC"), (lambda x:auc(x, multiclass=0), "AUC+M0"), (lambda x:auc(x, multiclass=1), "AUC+M1"), (lambda x:auc(x, multiclass=2), "AUC+M2"), (lambda x:auc(x, multiclass=3), "AUC+M3"), ) tests = ( (lambda l, ds: testing.cross_validation([l], ds), "CV"), (lambda l, ds: testing.proportion_test([l], ds, .7, 1), "Proportion test"), ) run_tests(datasets, measures, tests)
print def run_tests(datasets, measures, tests, iterations=10): for ds, ds_name in datasets: for t, t_name in tests: print "Testing %s on %s" % (t_name, ds_name) test_results = [t(random_learner, ds) for _ in xrange(iterations)] test(measures, test_results) datasets = ((data.Table("iris"), "Iris"), (data.Table("monks-1"), "Monks")) measures = ( (lambda x: ca(x), "CA"), (lambda x: ca(x, report_se=False, ignore_weights=False), "CA-SE-W"), (lambda x: ca(x, report_se=True, ignore_weights=False), "CA+SE-W"), (lambda x: [lambda x: [None]], ""), (lambda x: ca(x, report_se=False, ignore_weights=True), "CA-SE+W"), (lambda x: ca(x, report_se=True, ignore_weights=True), "CA+SE+W"), (lambda x: [lambda x: [None]], ""), ) tests = ( (lambda l, ds: testing.cross_validation([l], ds), "CV"), (lambda l, ds: testing.proportion_test([l], ds, .7, 1), "Proportion test"), (lambda l, ds: scoring.confusion_matrices( testing.proportion_test([l], ds, .7, 1)), "Confusion matrix"), ) run_tests(datasets, measures, tests)