def test_store_models(self): nrows, ncols = 1000, 10 t = random_data(nrows, ncols) fitters = [naive_bayes.BayesLearner(), majority.MajorityFitter()] cv = testing.CrossValidation(k=5) res = cv(t, fitters) self.assertIsNone(res.models) cv = testing.CrossValidation(k=5, store_models=True) res = cv(t, fitters) self.assertEqual(len(res.models), 5) for models in res.models: self.assertEqual(len(models), 2) self.assertIsInstance(models[0], naive_bayes.BayesClassifier) self.assertIsInstance(models[1], majority.ConstantClassifier) cv = testing.CrossValidation(k=5) res = cv(t, fitters) self.assertIsNone(res.models) res = testing.CrossValidation(t, fitters, k=5, store_models=True) self.assertEqual(len(res.models), 5) for models in res.models: self.assertEqual(len(models), 2) self.assertIsInstance(models[0], naive_bayes.BayesClassifier) self.assertIsInstance(models[1], majority.ConstantClassifier)
def test_miss_majority(self): x = np.zeros((50, 3)) y = x[:, -1] x[49] = 1 data = Table(x, y) res = testing.CrossValidation(data, [majority.MajorityFitter()]) np.testing.assert_equal(res.predicted[0][:49], 0) x[49] = 0 res = testing.CrossValidation(data, [majority.MajorityFitter()]) np.testing.assert_equal(res.predicted[0][:49], 0)
def main(): import sip from PyQt4.QtGui import QApplication from Orange.classification import logistic_regression, svm from Orange.evaluation import testing app = QApplication([]) w = OWLiftCurve() w.show() w.raise_() data = Orange.data.Table("ionosphere") results = testing.CrossValidation( data, [logistic_regression.LogisticRegressionLearner(penalty="l2"), logistic_regression.LogisticRegressionLearner(penalty="l1"), svm.SVMLearner(probability=True), svm.NuSVMLearner(probability=True) ], store_data=True ) results.fitter_names = ["LR l2", "LR l1", "SVM", "Nu SVM"] w.set_results(results) rval = app.exec_() sip.delete(w) del w app.processEvents() del app return rval
def test_NaiveBayes(self): table = Orange.data.Table('titanic') bayes = nb.BayesLearner() results = testing.CrossValidation(table[::20], [bayes], k=10) ca = scoring.CA(results) self.assertGreater(ca, 0.7) self.assertLess(ca, 0.9)
def test_RandomForest(self): table = Orange.data.Table('iris') forest = rf.RandomForestLearner() results = testing.CrossValidation(table, [forest], k=10) ca = scoring.CA(results) self.assertGreater(ca, 0.9) self.assertLess(ca, 0.99)
def test_folds(self): nrows, ncols = 1000, 10 t = random_data(nrows, ncols) res = testing.CrossValidation(t, [naive_bayes.BayesLearner()], k=5) self.assertEqual(len(res.folds), 5) for i, fold in enumerate(res.folds): self.assertEqual(fold.start, i * 200) self.assertEqual(fold.stop, (i + 1) * 200)
def test_store_data(self): nrows, ncols = 1000, 10 t = random_data(nrows, ncols) fitters = [naive_bayes.BayesLearner()] cv = testing.CrossValidation() res = cv(t, fitters) self.assertIsNone(res.data) cv = testing.CrossValidation(store_data=True) res = cv(t, fitters) self.assertIs(res.data, t) res = testing.CrossValidation(t, fitters) self.assertIsNone(res.data) res = testing.CrossValidation(t, fitters, store_data=True) self.assertIs(res.data, t)
def test_10_fold_probs(self): data = Table('iris')[30:130] fitters = [majority.MajorityFitter(), majority.MajorityFitter()] results = testing.CrossValidation(k=10)(data, fitters) self.assertEqual(results.predicted.shape, (2, len(data))) np.testing.assert_equal(results.predicted, np.ones((2, 100))) probs = results.probabilities self.assertTrue((probs[:, :, 0] < probs[:, :, 2]).all()) self.assertTrue((probs[:, :, 2] < probs[:, :, 1]).all())
def update_results(self): # items in need of an update items = [(key, input) for key, input in self.learners.items() if input.results is None] learners = [input.learner for _, input in items] self.setStatusMessage("Running") # TODO: Test each learner individually if self.resampling == OWTestLearners.KFold: results = testing.CrossValidation(self.train_data, learners, k=self.k_folds, store_data=True) elif self.resampling == OWTestLearners.LeaveOneOut: results = testing.LeaveOneOut(self.train_data, learners, store_data=True) elif self.resampling == OWTestLearners.Bootstrap: p = self.sample_p / 100.0 results = testing.Bootstrap(self.train_data, learners, n_resamples=self.n_repeat, p=p, store_data=True) elif self.resampling == OWTestLearners.TestOnTrain: results = testing.TestOnTrainingData(self.train_data, learners, store_data=True) elif self.resampling == OWTestLearners.TestOnTest: assert self.test_data is not None results = testing.TestOnTestData(self.train_data, self.test_data, learners, store_data=True) else: assert False results = list(split_by_model(results)) class_var = self.train_data.domain.class_var if is_discrete(class_var): test_stats = classification_stats else: test_stats = regression_stats stats = [test_stats(res) for res in results] for (key, input), res, stat in zip(items, results, stats): self.learners[key] = input._replace(results=res, stats=stat) self.setStatusMessage("") self._update_stats_model()
def test_results(self): nrows, ncols = 1000, 10 t = random_data(nrows, ncols) res = testing.CrossValidation(t, [naive_bayes.BayesLearner()]) y = t.Y np.testing.assert_equal(res.actual, y[res.row_indices].reshape(nrows)) np.testing.assert_equal(res.predicted[0], y[res.row_indices].reshape(nrows)) np.testing.assert_equal(np.argmax(res.probabilities[0], axis=1), y[res.row_indices].reshape(nrows)) self.assertEqual(len(res.folds), 10) for i, fold in enumerate(res.folds): self.assertEqual(fold.start, i * 100) self.assertEqual(fold.stop, (i + 1) * 100)
def update_results(self): self.warning([1, 2]) self.error(2) if self.train_data is None: return if self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: self.warning(2, "Missing separate test data input") return elif self.test_data.domain.class_var != \ self.train_data.domain.class_var: self.error(2, ("Inconsistent class variable between test " + "and train data sets")) return # items in need of an update items = [(key, input) for key, input in self.learners.items() if input.results is None] learners = [input.learner for _, input in items] self.setStatusMessage("Running") if self.test_data is not None and \ self.resampling != OWTestLearners.TestOnTest: self.warning(1, "Test data is present but unused. " "Select 'Test on test data' to use it.") # TODO: Test each learner individually if self.resampling == OWTestLearners.KFold: results = testing.CrossValidation( self.train_data, learners, k=self.k_folds, store_data=True ) elif self.resampling == OWTestLearners.LeaveOneOut: results = testing.LeaveOneOut( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.Bootstrap: p = self.sample_p / 100.0 results = testing.Bootstrap( self.train_data, learners, n_resamples=self.n_repeat, p=p, store_data=True ) elif self.resampling == OWTestLearners.TestOnTrain: results = testing.TestOnTrainingData( self.train_data, learners, store_data=True ) elif self.resampling == OWTestLearners.TestOnTest: if self.test_data is None: return results = testing.TestOnTestData( self.train_data, self.test_data, learners, store_data=True ) else: assert False results = list(split_by_model(results)) class_var = self.train_data.domain.class_var if is_discrete(class_var): test_stats = classification_stats else: test_stats = regression_stats self._update_header() stats = [test_stats(res) for res in results] for (key, input), res, stat in zip(items, results, stats): self.learners[key] = input._replace(results=res, stats=stat) self.setStatusMessage("") self._update_stats_model()
precompute=False, max_iter=1000, copy_X=True, tol=0.0001, warm_start=False, positive=False, preprocessors=None) ElasticNet.name = "Elactic net Regression" learners = [Linear, Ridge, Lasso, ElasticNet] results = testing.CrossValidation( data, learners, k=10, #random_state=1, #store_data=False, #store_models=False, #preprocessor=None, #callback=None, #warnings=None ) #data information print("------Data Information------") print("File: %a" % data.name, "Data Instances: %a" % len(data), "Attributes: %a" % len(data.domain.attributes), "Class:", data.domain.class_var.name) #regression part for i in range(len(learners)): print("Regularization: %a" % learners[i].name) print("------Testing&Score------") print("MSE: %.3f" % scoring.MSE(results)[i],
def setGeometry(self, rect): super().setGeometry(rect) def paintEvent(self, event): painter = QPainter(self) rect = self.geometry() textRect = QRect(0, 0, rect.width(), rect.height()) painter.translate(textRect.bottomLeft()) painter.rotate(-90) painter.drawText( QRect(QPoint(0, 0), QSize(rect.height(), rect.width())), Qt.AlignCenter, self.text()) painter.end() if __name__ == "__main__": from PyQt4.QtGui import QApplication from Orange.evaluation import testing from Orange.classification import naive_bayes app = QApplication([]) w = OWConfusionMatrix() w.show() data = Orange.data.Table("iris") res = testing.CrossValidation(data, [naive_bayes.BayesLearner()], store_data=True) w.set_results(res) app.exec_()
def setGeometry(self, rect): super().setGeometry(rect) def paintEvent(self, event): painter = QPainter(self) rect = self.geometry() textRect = QRect(0, 0, rect.width(), rect.height()) painter.translate(textRect.bottomLeft()) painter.rotate(-90) painter.drawText( QRect(QPoint(0, 0), QSize(rect.height(), rect.width())), Qt.AlignCenter, self.text()) painter.end() if __name__ == "__main__": from PyQt4.QtGui import QApplication from Orange.evaluation import testing from Orange.classification import tree app = QApplication([]) w = OWConfusionMatrix() w.show() data = Orange.data.Table("iris") res = testing.CrossValidation(data, [tree.ClassificationTreeLearner()], store_data=True) w.set_results(res) app.exec_()