def test_F1_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
res = F1(results)
self.assertAlmostEqual(res[0], 0.61)
self.assertEqual(res[1], 1.)
def test_F1_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
res = F1(results)
self.assertAlmostEqual(res[0], 0.61)
self.assertEqual(res[1], 1.)
def test_init(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_init(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_F1_target(self):
results = Results(domain=Domain([],
DiscreteVariable(name="y",
values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.),
(4, 2 / 3)):
res = F1(results, target=target)
self.assertEqual(res[0], prob)
self.assertEqual(res[1], 1.)
def test_F1_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 1]])
res = F1(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 5 / 6)
res_target = F1(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = F1(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 3 / 4)
def test_F1_target(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
for target, prob in ((0, 4 / 5),
(1, 1 / 3),
(2, 1 / 2),
(3, 1.),
(4, 2 / 3)):
res = F1(results, target=target)
self.assertEqual(res[0], prob)
self.assertEqual(res[1], 1.)
def test_F1_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 1]])
res = self.score(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 5 / 6)
res_target = self.score(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = self.score(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 3 / 4)
def test_precision_multiclass(self):
results = Results(domain=Domain([],
DiscreteVariable(name="y",
values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
[0, 1, 4, 1, 1, 0, 0, 2, 3, 1]])
res = self.score(results, average='weighted')
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 0.9, 5)
for target, prob in ((0, 7 / 8), (1, 5 / 8), (2, 1), (3, 1), (4, 1)):
res = self.score(results, target=target, average=None)
self.assertEqual(res[0], 1.)
self.assertEqual(res[1], prob)
def _call_predictors(self):
if not self.data:
return
if self.class_var:
domain = self.data.domain
classless_data = self.data.transform(
Domain(domain.attributes, None, domain.metas))
else:
classless_data = self.data
for inputid, slot in self.predictors.items():
if isinstance(slot.results, Results):
continue
predictor = slot.predictor
try:
if predictor.domain.class_var.is_discrete:
pred, prob = predictor(classless_data, Model.ValueProbs)
else:
pred = predictor(classless_data, Model.Value)
prob = numpy.zeros((len(pred), 0))
except (ValueError, DomainTransformationError) as err:
self.predictors[inputid] = \
slot._replace(results=f"{predictor.name}: {err}")
continue
results = Results()
results.data = self.data
results.domain = self.data.domain
results.row_indices = numpy.arange(len(self.data))
results.folds = (Ellipsis, )
results.actual = self.data.Y
results.unmapped_probabilities = prob
results.unmapped_predicted = pred
results.probabilities = results.predicted = None
self.predictors[inputid] = slot._replace(results=results)
target = predictor.domain.class_var
if target != self.class_var:
continue
if target is not self.class_var and target.is_discrete:
backmappers, n_values = predictor.get_backmappers(self.data)
prob = predictor.backmap_probs(prob, n_values, backmappers)
pred = predictor.backmap_value(pred, prob, n_values,
backmappers)
results.predicted = pred.reshape((1, len(self.data)))
results.probabilities = prob.reshape((1, ) + prob.shape)
def test_F1_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
)
results.predicted = np.array(
[[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]]
)
res = self.score(results, average="weighted")
self.assertEqual(res[0], 1.0)
self.assertAlmostEqual(res[1], 0.61)
for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.0), (4, 2 / 3)):
res = self.score(results, target=target)
self.assertEqual(res[0], 1.0)
self.assertEqual(res[1], prob)
def test_precision_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 0]])
res = self.score(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 4 / 6)
res_target = self.score(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = self.score(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 3 / 4)
res_target = self.score(results, average='macro')
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], (4 / 6 + 3 / 4) / 2)
def test_precision_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
[0, 1, 4, 1, 1, 0, 0, 2, 3, 1]])
res = self.score(results, average='weighted')
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 0.78333, 5)
for target, prob in ((0, 2 / 3),
(1, 1 / 4),
(2, 1 / 1),
(3, 1 / 1),
(4, 1 / 1)):
res = self.score(results, target=target, average=None)
self.assertEqual(res[0], 1.)
self.assertEqual(res[1], prob)
def test_recall_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 0]])
res = self.score(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 4 / 5)
res_target = self.score(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = self.score(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 3 / 5)
res_target = self.score(results, average='macro')
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], (4 / 5 + 3 / 5) / 2)
def _commit_evaluation_results(self):
slots = [p for p in self._non_errored_predictors()
if p.results.predicted is not None]
if not slots:
self.Outputs.evaluation_results.send(None)
return
nanmask = numpy.isnan(self.data.get_column_view(self.class_var)[0])
data = self.data[~nanmask]
results = Results(data, store_data=True)
results.folds = None
results.row_indices = numpy.arange(len(data))
results.actual = data.Y.ravel()
results.predicted = numpy.vstack(
tuple(p.results.predicted[0][~nanmask] for p in slots))
if self.class_var and self.class_var.is_discrete:
results.probabilities = numpy.array(
[p.results.probabilities[0][~nanmask] for p in slots])
results.learner_names = [p.name for p in slots]
self.Outputs.evaluation_results.send(results)
