def test_F1_multiclass(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1], [0, 4, 4, 1, 2, 0, 1, 2, 3, 2]]) res = F1(results) self.assertAlmostEqual(res[0], 0.61) self.assertEqual(res[1], 1.)
def compute_auc(self, actual, predicted): predicted = np.array(predicted).reshape(1, -1) probabilities = np.zeros((1, predicted.shape[1], 2)) probabilities[0,:,1] = predicted[0] probabilities[0,:,0] = 1 - predicted[0] results = Results( nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]), actual=actual, predicted=predicted) results.probabilities = probabilities return AUC(results)[0]
def test_F1_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 1]]) res = F1(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 5 / 6) res_target = F1(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = F1(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 3 / 4)
def test_F1_target(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1], [0, 4, 4, 1, 2, 0, 1, 2, 3, 2]]) for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.), (4, 2 / 3)): res = F1(results, target=target) self.assertEqual(res[0], prob) self.assertEqual(res[1], 1.)
def test_precision_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]]) res = self.score(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 3 / 5) res_target = self.score(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = self.score(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 4 / 5)
def test_precision_multiclass(self): results = Results(domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]]) res = self.score(results, average='weighted') self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 0.9, 5) for target, prob in ((0, 7 / 8), (1, 5 / 8), (2, 1), (3, 1), (4, 1)): res = self.score(results, target=target, average=None) self.assertEqual(res[0], 1.) self.assertEqual(res[1], prob)
def test_F1_multiclass(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], ) results.predicted = np.array( [[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]] ) res = self.score(results, average="weighted") self.assertEqual(res[0], 1.0) self.assertAlmostEqual(res[1], 0.61) for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.0), (4, 2 / 3)): res = self.score(results, target=target) self.assertEqual(res[0], 1.0) self.assertEqual(res[1], prob)
def compute_auc(self, actual, predicted): predicted = np.array(predicted).reshape(1, -1) results = Results(nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]), actual=actual, predicted=predicted) return AUC(results)[0]
def test_precision_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]]) res = self.score(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 4 / 6) res_target = self.score(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = self.score(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 3 / 4) res_target = self.score(results, average='macro') self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], (4 / 6 + 3 / 4) / 2)
def test_precision_multiclass(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]]) res = self.score(results, average='weighted') self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 0.78333, 5) for target, prob in ((0, 2 / 3), (1, 1 / 4), (2, 1 / 1), (3, 1 / 1), (4, 1 / 1)): res = self.score(results, target=target, average=None) self.assertEqual(res[0], 1.) self.assertEqual(res[1], prob)
def test_init(self): res = Results(nmethods=2, nrows=100) res.actual[:50] = 0 res.actual[50:] = 1 res.predicted = np.vstack((res.actual, res.actual)) np.testing.assert_almost_equal(CA(res), [1, 1]) res.predicted[0][0] = 1 np.testing.assert_almost_equal(CA(res), [0.99, 1]) res.predicted[1] = 1 - res.predicted[1] np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_store_attributes(self): res = Results( self.data, row_indices=self.row_indices, folds=self.folds, score_by_folds=False, learners=self.learners, models=self.models, failed=self.failed, actual=self.actual, predicted=self.predicted, probabilities=self.probabilities, store_data=42, store_models=43) self.assertIs(res.data, self.data) self.assertEqual(res.nrows, 100) self.assertIs(res.row_indices, self.row_indices) self.assertIs(res.folds, self.folds) self.assertFalse(res.score_by_folds) self.assertIs(res.learners, self.learners) self.assertIs(res.models, self.models) self.assertIs(res.failed, self.failed) self.assertIs(res.actual, self.actual) self.assertIs(res.predicted, self.predicted) self.assertIs(res.probabilities, self.probabilities)
def _commit_evaluation_results(self): slots = [p for p in self._non_errored_predictors() if p.results.predicted is not None] if not slots: self.Outputs.evaluation_results.send(None) return nanmask = numpy.isnan(self.data.get_column_view(self.class_var)[0]) data = self.data[~nanmask] results = Results(data, store_data=True) results.folds = None results.row_indices = numpy.arange(len(data)) results.actual = data.Y.ravel() results.predicted = numpy.vstack( tuple(p.results.predicted[0][~nanmask] for p in slots)) if self.class_var and self.class_var.is_discrete: results.probabilities = numpy.array( [p.results.probabilities[0][~nanmask] for p in slots]) results.learner_names = [p.name for p in slots] self.Outputs.evaluation_results.send(results)
def setUp(self): super().setUp() n, p = (0, 1) actual, probs = np.array([(p, .8), (n, .7), (p, .6), (p, .55), (p, .54), (n, .53), (n, .52), (p, .51), (n, .505), (p, .4), (n, .39), (p, .38), (n, .37), (n, .36), (n, .35), (p, .34), (n, .33), (p, .30), (n, .1)]).T self.curves = Curves(actual, probs) probs2 = (probs + 0.5) / 2 + 1 self.curves2 = Curves(actual, probs2) pred = probs > 0.5 pred2 = probs2 > 0.5 probs = np.vstack((1 - probs, probs)).T probs2 = np.vstack((1 - probs2, probs2)).T domain = Domain([], DiscreteVariable("y", values=("a", "b"))) self.results = Results(domain=domain, actual=actual, folds=np.array([Ellipsis]), models=np.array([[Mock(), Mock()]]), row_indices=np.arange(19), predicted=np.array((pred, pred2)), probabilities=np.array([probs, probs2])) self.lenses = data = Table(test_filename("datasets/lenses.tab")) majority = Orange.classification.MajorityLearner() majority.name = "majority" knn3 = Orange.classification.KNNLearner(n_neighbors=3) knn3.name = "knn-3" knn1 = Orange.classification.KNNLearner(n_neighbors=1) knn1.name = "knn-1" self.lenses_results = Orange.evaluation.TestOnTestData( store_data=True, store_models=True)(data=data[::2], test_data=data[1::2], learners=[majority, knn3, knn1]) self.lenses_results.learner_names = ["majority", "knn-3", "knn-1"] self.widget = self.create_widget( OWCalibrationPlot) # type: OWCalibrationPlot warnings.filterwarnings("ignore", ".*", ConvergenceWarning)
def eval_result_matrix(self, predicted_results, domain_results): self.eval_results = Results(self.data, nrows=len(self.data), row_indices = np.arange(len(self.data)), actual=self.data.Y, predicted=np.array([predicted_results.ravel()]))
def test_tooltips(self): # See https://people.inf.elte.hu/kiss/11dwhdm/roc.pdf for the curve # representing this data actual = np.array([float(c == "n") for c in "ppnpppnnpnpnpnnnpnpn"]) p = np.array([ .9, .8, .7, .6, .55, .54, .53, .52, .51, .505, .4, .39, .38, .37, .36, .35, .34, .33, .30, .1 ]) n = 1 - p predicted = (p > .5).astype(float) # The second curve is like the first except for the first three points: # it goes to the right and then up p2 = p.copy() p2[:4] = [0.7, 0.8, 0.9, 0.59] n2 = 1 - p2 predicted2 = (p2 < .5).astype(float) data = Orange.data.Table( Orange.data.Domain( [], [Orange.data.DiscreteVariable("y", values=tuple("pn"))]), np.empty((len(p), 0), dtype=float), actual) res = Results(data=data, actual=actual, predicted=np.array([list(predicted), list(predicted2)]), probabilities=np.array( [list(zip(p, n)), list(zip(p2, n2))])) self.send_signal(self.widget.Inputs.evaluation_results, res) self.widget.roc_averaging = OWROCAnalysis.Merge self.widget.target_index = 0 self.widget.selected_classifiers = [0, 1] vb = self.widget.plot.getViewBox() vb.childTransform() # Force pyqtgraph to update transforms curve = self.widget.plot_curves(self.widget.target_index, 0) curve_merge = curve.merge() view = self.widget.plotview item = curve_merge.curve_item # type: pg.PlotCurveItem with patch.object(QToolTip, "showText") as show_text: # no tooltips to be shown pos = item.mapToScene(0.0, 1.0) pos = view.mapFromScene(pos) mouseMove(view.viewport(), pos) show_text.assert_not_called() # test single point pos = item.mapToScene(0, 0.1) pos = view.mapFromScene(pos) mouseMove(view.viewport(), pos) (_, text), _ = show_text.call_args self.assertIn("(#1) 0.900", text) self.assertNotIn("#2", text) # test overlapping points pos = item.mapToScene(0.0, 0.0) pos = view.mapFromScene(pos) mouseMove(view.viewport(), pos) (_, text), _ = show_text.call_args self.assertIn("(#1) 1.000\n(#2) 1.000", text) pos = item.mapToScene(0.1, 0.3) pos = view.mapFromScene(pos) mouseMove(view.viewport(), pos) (_, text), _ = show_text.call_args self.assertIn("(#1) 0.600\n(#2) 0.590", text) show_text.reset_mock() # test that cache is invalidated when changing averaging mode self.widget.roc_averaging = OWROCAnalysis.Threshold self.widget._replot() mouseMove(view.viewport(), pos) (_, text), _ = show_text.call_args self.assertIn("(#1) 0.600\n(#2) 0.590", text) show_text.reset_mock() # test nan thresholds self.widget.roc_averaging = OWROCAnalysis.Vertical self.widget._replot() mouseMove(view.viewport(), pos) show_text.assert_not_called()
def _call_predictors(self): if not self.data: return if self.class_var: domain = self.data.domain classless_data = self.data.transform( Domain(domain.attributes, None, domain.metas)) else: classless_data = self.data for inputid, slot in self.predictors.items(): if isinstance(slot.results, Results): continue predictor = slot.predictor try: if predictor.domain.class_var.is_discrete: pred, prob = predictor(classless_data, Model.ValueProbs) else: pred = predictor(classless_data, Model.Value) prob = numpy.zeros((len(pred), 0)) except (ValueError, DomainTransformationError) as err: self.predictors[inputid] = \ slot._replace(results=f"{predictor.name}: {err}") continue results = Results() results.data = self.data results.domain = self.data.domain results.row_indices = numpy.arange(len(self.data)) results.folds = (Ellipsis, ) results.actual = self.data.Y results.unmapped_probabilities = prob results.unmapped_predicted = pred results.probabilities = results.predicted = None self.predictors[inputid] = slot._replace(results=results) target = predictor.domain.class_var if target != self.class_var: continue if target is not self.class_var and target.is_discrete: backmappers, n_values = predictor.get_backmappers(self.data) prob = predictor.backmap_probs(prob, n_values, backmappers) pred = predictor.backmap_value(pred, prob, n_values, backmappers) results.predicted = pred.reshape((1, len(self.data))) results.probabilities = prob.reshape((1, ) + prob.shape)