def test_F1_multiclass(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1], [0, 4, 4, 1, 2, 0, 1, 2, 3, 2]]) res = F1(results) self.assertAlmostEqual(res[0], 0.61) self.assertEqual(res[1], 1.)
def test_init(self): res = Results(nmethods=2, nrows=100) res.actual[:50] = 0 res.actual[50:] = 1 res.predicted = np.vstack((res.actual, res.actual)) np.testing.assert_almost_equal(CA(res), [1, 1]) res.predicted[0][0] = 1 np.testing.assert_almost_equal(CA(res), [0.99, 1]) res.predicted[1] = 1 - res.predicted[1] np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_F1_target(self): results = Results(domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1], [0, 4, 4, 1, 2, 0, 1, 2, 3, 2]]) for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.), (4, 2 / 3)): res = F1(results, target=target) self.assertEqual(res[0], prob) self.assertEqual(res[1], 1.)
def test_F1_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 1]]) res = F1(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 5 / 6) res_target = F1(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = F1(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 3 / 4)
def test_F1_target(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1], [0, 4, 4, 1, 2, 0, 1, 2, 3, 2]]) for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.), (4, 2 / 3)): res = F1(results, target=target) self.assertEqual(res[0], prob) self.assertEqual(res[1], 1.)
def test_F1_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 1]]) res = self.score(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 5 / 6) res_target = self.score(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = self.score(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 3 / 4)
def test_precision_multiclass(self): results = Results(domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]]) res = self.score(results, average='weighted') self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 0.9, 5) for target, prob in ((0, 7 / 8), (1, 5 / 8), (2, 1), (3, 1), (4, 1)): res = self.score(results, target=target, average=None) self.assertEqual(res[0], 1.) self.assertEqual(res[1], prob)
def _call_predictors(self): if not self.data: return if self.class_var: domain = self.data.domain classless_data = self.data.transform( Domain(domain.attributes, None, domain.metas)) else: classless_data = self.data for inputid, slot in self.predictors.items(): if isinstance(slot.results, Results): continue predictor = slot.predictor try: if predictor.domain.class_var.is_discrete: pred, prob = predictor(classless_data, Model.ValueProbs) else: pred = predictor(classless_data, Model.Value) prob = numpy.zeros((len(pred), 0)) except (ValueError, DomainTransformationError) as err: self.predictors[inputid] = \ slot._replace(results=f"{predictor.name}: {err}") continue results = Results() results.data = self.data results.domain = self.data.domain results.row_indices = numpy.arange(len(self.data)) results.folds = (Ellipsis, ) results.actual = self.data.Y results.unmapped_probabilities = prob results.unmapped_predicted = pred results.probabilities = results.predicted = None self.predictors[inputid] = slot._replace(results=results) target = predictor.domain.class_var if target != self.class_var: continue if target is not self.class_var and target.is_discrete: backmappers, n_values = predictor.get_backmappers(self.data) prob = predictor.backmap_probs(prob, n_values, backmappers) pred = predictor.backmap_value(pred, prob, n_values, backmappers) results.predicted = pred.reshape((1, len(self.data))) results.probabilities = prob.reshape((1, ) + prob.shape)
def test_F1_multiclass(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], ) results.predicted = np.array( [[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]] ) res = self.score(results, average="weighted") self.assertEqual(res[0], 1.0) self.assertAlmostEqual(res[1], 0.61) for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.0), (4, 2 / 3)): res = self.score(results, target=target) self.assertEqual(res[0], 1.0) self.assertEqual(res[1], prob)
def test_precision_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]]) res = self.score(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 4 / 6) res_target = self.score(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = self.score(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 3 / 4) res_target = self.score(results, average='macro') self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], (4 / 6 + 3 / 4) / 2)
def test_precision_multiclass(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01234")), actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]) results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]]) res = self.score(results, average='weighted') self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 0.78333, 5) for target, prob in ((0, 2 / 3), (1, 1 / 4), (2, 1 / 1), (3, 1 / 1), (4, 1 / 1)): res = self.score(results, target=target, average=None) self.assertEqual(res[0], 1.) self.assertEqual(res[1], prob)
def test_recall_binary(self): results = Results( domain=Domain([], DiscreteVariable(name="y", values="01")), actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1]) results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1], [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]]) res = self.score(results) self.assertEqual(res[0], 1.) self.assertAlmostEqual(res[1], 4 / 5) res_target = self.score(results, target=1) self.assertEqual(res[0], res_target[0]) self.assertEqual(res[1], res_target[1]) res_target = self.score(results, target=0) self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], 3 / 5) res_target = self.score(results, average='macro') self.assertEqual(res_target[0], 1.) self.assertAlmostEqual(res_target[1], (4 / 5 + 3 / 5) / 2)
def _commit_evaluation_results(self): slots = [p for p in self._non_errored_predictors() if p.results.predicted is not None] if not slots: self.Outputs.evaluation_results.send(None) return nanmask = numpy.isnan(self.data.get_column_view(self.class_var)[0]) data = self.data[~nanmask] results = Results(data, store_data=True) results.folds = None results.row_indices = numpy.arange(len(data)) results.actual = data.Y.ravel() results.predicted = numpy.vstack( tuple(p.results.predicted[0][~nanmask] for p in slots)) if self.class_var and self.class_var.is_discrete: results.probabilities = numpy.array( [p.results.probabilities[0][~nanmask] for p in slots]) results.learner_names = [p.name for p in slots] self.Outputs.evaluation_results.send(results)