def test_F1_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
res = F1(results)
self.assertAlmostEqual(res[0], 0.61)
self.assertEqual(res[1], 1.)
def compute_auc(self, actual, predicted):
predicted = np.array(predicted).reshape(1, -1)
probabilities = np.zeros((1, predicted.shape[1], 2))
probabilities[0,:,1] = predicted[0]
probabilities[0,:,0] = 1 - predicted[0]
results = Results(
nmethods=1, domain=Domain([], [DiscreteVariable(values='01')]),
actual=actual, predicted=predicted)
results.probabilities = probabilities
return AUC(results)[0]
def test_F1_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 1]])
res = F1(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 5 / 6)
res_target = F1(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = F1(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 3 / 4)
def test_F1_target(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
for target, prob in ((0, 4 / 5),
(1, 1 / 3),
(2, 1 / 2),
(3, 1.),
(4, 2 / 3)):
res = F1(results, target=target)
self.assertEqual(res[0], prob)
self.assertEqual(res[1], 1.)
def test_precision_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 0]])
res = self.score(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 3 / 5)
res_target = self.score(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = self.score(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 4 / 5)
def test_precision_multiclass(self):
results = Results(domain=Domain([],
DiscreteVariable(name="y",
values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
[0, 1, 4, 1, 1, 0, 0, 2, 3, 1]])
res = self.score(results, average='weighted')
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 0.9, 5)
for target, prob in ((0, 7 / 8), (1, 5 / 8), (2, 1), (3, 1), (4, 1)):
res = self.score(results, target=target, average=None)
self.assertEqual(res[0], 1.)
self.assertEqual(res[1], prob)
def test_F1_target(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 1, 4, 1, 1, 0, 0, 2, 3, 1],
[0, 4, 4, 1, 2, 0, 1, 2, 3, 2]])
for target, prob in ((0, 4 / 5),
(1, 1 / 3),
(2, 1 / 2),
(3, 1.),
(4, 2 / 3)):
res = F1(results, target=target)
self.assertEqual(res[0], prob)
self.assertEqual(res[1], 1.)
def test_F1_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
)
results.predicted = np.array(
[[0, 4, 4, 1, 2, 0, 1, 2, 3, 2], [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]]
)
res = self.score(results, average="weighted")
self.assertEqual(res[0], 1.0)
self.assertAlmostEqual(res[1], 0.61)
for target, prob in ((0, 4 / 5), (1, 1 / 3), (2, 1 / 2), (3, 1.0), (4, 2 / 3)):
res = self.score(results, target=target)
self.assertEqual(res[0], 1.0)
self.assertEqual(res[1], prob)
def compute_auc(self, actual, predicted):
predicted = np.array(predicted).reshape(1, -1)
results = Results(nmethods=1,
domain=Domain([], [DiscreteVariable(values='01')]),
actual=actual,
predicted=predicted)
return AUC(results)[0]
def test_precision_binary(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01")),
actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
[0, 1, 1, 1, 0, 0, 1, 1, 1, 0]])
res = self.score(results)
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 4 / 6)
res_target = self.score(results, target=1)
self.assertEqual(res[0], res_target[0])
self.assertEqual(res[1], res_target[1])
res_target = self.score(results, target=0)
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], 3 / 4)
res_target = self.score(results, average='macro')
self.assertEqual(res_target[0], 1.)
self.assertAlmostEqual(res_target[1], (4 / 6 + 3 / 4) / 2)
def test_precision_multiclass(self):
results = Results(
domain=Domain([], DiscreteVariable(name="y", values="01234")),
actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
[0, 1, 4, 1, 1, 0, 0, 2, 3, 1]])
res = self.score(results, average='weighted')
self.assertEqual(res[0], 1.)
self.assertAlmostEqual(res[1], 0.78333, 5)
for target, prob in ((0, 2 / 3),
(1, 1 / 4),
(2, 1 / 1),
(3, 1 / 1),
(4, 1 / 1)):
res = self.score(results, target=target, average=None)
self.assertEqual(res[0], 1.)
self.assertEqual(res[1], prob)
def test_init(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_init(self):
res = Results(nmethods=2, nrows=100)
res.actual[:50] = 0
res.actual[50:] = 1
res.predicted = np.vstack((res.actual, res.actual))
np.testing.assert_almost_equal(CA(res), [1, 1])
res.predicted[0][0] = 1
np.testing.assert_almost_equal(CA(res), [0.99, 1])
res.predicted[1] = 1 - res.predicted[1]
np.testing.assert_almost_equal(CA(res), [0.99, 0])
def test_store_attributes(self):
res = Results(
self.data,
row_indices=self.row_indices, folds=self.folds,
score_by_folds=False, learners=self.learners, models=self.models,
failed=self.failed, actual=self.actual, predicted=self.predicted,
probabilities=self.probabilities, store_data=42, store_models=43)
self.assertIs(res.data, self.data)
self.assertEqual(res.nrows, 100)
self.assertIs(res.row_indices, self.row_indices)
self.assertIs(res.folds, self.folds)
self.assertFalse(res.score_by_folds)
self.assertIs(res.learners, self.learners)
self.assertIs(res.models, self.models)
self.assertIs(res.failed, self.failed)
self.assertIs(res.actual, self.actual)
self.assertIs(res.predicted, self.predicted)
self.assertIs(res.probabilities, self.probabilities)
def _commit_evaluation_results(self):
slots = [p for p in self._non_errored_predictors()
if p.results.predicted is not None]
if not slots:
self.Outputs.evaluation_results.send(None)
return
nanmask = numpy.isnan(self.data.get_column_view(self.class_var)[0])
data = self.data[~nanmask]
results = Results(data, store_data=True)
results.folds = None
results.row_indices = numpy.arange(len(data))
results.actual = data.Y.ravel()
results.predicted = numpy.vstack(
tuple(p.results.predicted[0][~nanmask] for p in slots))
if self.class_var and self.class_var.is_discrete:
results.probabilities = numpy.array(
[p.results.probabilities[0][~nanmask] for p in slots])
results.learner_names = [p.name for p in slots]
self.Outputs.evaluation_results.send(results)
def setUp(self):
super().setUp()
n, p = (0, 1)
actual, probs = np.array([(p, .8), (n, .7), (p, .6), (p, .55),
(p, .54), (n, .53), (n, .52), (p, .51),
(n, .505), (p, .4), (n, .39), (p, .38),
(n, .37), (n, .36), (n, .35), (p, .34),
(n, .33), (p, .30), (n, .1)]).T
self.curves = Curves(actual, probs)
probs2 = (probs + 0.5) / 2 + 1
self.curves2 = Curves(actual, probs2)
pred = probs > 0.5
pred2 = probs2 > 0.5
probs = np.vstack((1 - probs, probs)).T
probs2 = np.vstack((1 - probs2, probs2)).T
domain = Domain([], DiscreteVariable("y", values=("a", "b")))
self.results = Results(domain=domain,
actual=actual,
folds=np.array([Ellipsis]),
models=np.array([[Mock(), Mock()]]),
row_indices=np.arange(19),
predicted=np.array((pred, pred2)),
probabilities=np.array([probs, probs2]))
self.lenses = data = Table(test_filename("datasets/lenses.tab"))
majority = Orange.classification.MajorityLearner()
majority.name = "majority"
knn3 = Orange.classification.KNNLearner(n_neighbors=3)
knn3.name = "knn-3"
knn1 = Orange.classification.KNNLearner(n_neighbors=1)
knn1.name = "knn-1"
self.lenses_results = Orange.evaluation.TestOnTestData(
store_data=True,
store_models=True)(data=data[::2],
test_data=data[1::2],
learners=[majority, knn3, knn1])
self.lenses_results.learner_names = ["majority", "knn-3", "knn-1"]
self.widget = self.create_widget(
OWCalibrationPlot) # type: OWCalibrationPlot
warnings.filterwarnings("ignore", ".*", ConvergenceWarning)
def eval_result_matrix(self, predicted_results, domain_results):
self.eval_results = Results(self.data,
nrows=len(self.data),
row_indices = np.arange(len(self.data)),
actual=self.data.Y,
predicted=np.array([predicted_results.ravel()]))
def test_tooltips(self):
# See https://people.inf.elte.hu/kiss/11dwhdm/roc.pdf for the curve
# representing this data
actual = np.array([float(c == "n") for c in "ppnpppnnpnpnpnnnpnpn"])
p = np.array([
.9, .8, .7, .6, .55, .54, .53, .52, .51, .505, .4, .39, .38, .37,
.36, .35, .34, .33, .30, .1
])
n = 1 - p
predicted = (p > .5).astype(float)
# The second curve is like the first except for the first three points:
# it goes to the right and then up
p2 = p.copy()
p2[:4] = [0.7, 0.8, 0.9, 0.59]
n2 = 1 - p2
predicted2 = (p2 < .5).astype(float)
data = Orange.data.Table(
Orange.data.Domain(
[], [Orange.data.DiscreteVariable("y", values=tuple("pn"))]),
np.empty((len(p), 0), dtype=float), actual)
res = Results(data=data,
actual=actual,
predicted=np.array([list(predicted),
list(predicted2)]),
probabilities=np.array(
[list(zip(p, n)), list(zip(p2, n2))]))
self.send_signal(self.widget.Inputs.evaluation_results, res)
self.widget.roc_averaging = OWROCAnalysis.Merge
self.widget.target_index = 0
self.widget.selected_classifiers = [0, 1]
vb = self.widget.plot.getViewBox()
vb.childTransform() # Force pyqtgraph to update transforms
curve = self.widget.plot_curves(self.widget.target_index, 0)
curve_merge = curve.merge()
view = self.widget.plotview
item = curve_merge.curve_item # type: pg.PlotCurveItem
with patch.object(QToolTip, "showText") as show_text:
# no tooltips to be shown
pos = item.mapToScene(0.0, 1.0)
pos = view.mapFromScene(pos)
mouseMove(view.viewport(), pos)
show_text.assert_not_called()
# test single point
pos = item.mapToScene(0, 0.1)
pos = view.mapFromScene(pos)
mouseMove(view.viewport(), pos)
(_, text), _ = show_text.call_args
self.assertIn("(#1) 0.900", text)
self.assertNotIn("#2", text)
# test overlapping points
pos = item.mapToScene(0.0, 0.0)
pos = view.mapFromScene(pos)
mouseMove(view.viewport(), pos)
(_, text), _ = show_text.call_args
self.assertIn("(#1) 1.000\n(#2) 1.000", text)
pos = item.mapToScene(0.1, 0.3)
pos = view.mapFromScene(pos)
mouseMove(view.viewport(), pos)
(_, text), _ = show_text.call_args
self.assertIn("(#1) 0.600\n(#2) 0.590", text)
show_text.reset_mock()
# test that cache is invalidated when changing averaging mode
self.widget.roc_averaging = OWROCAnalysis.Threshold
self.widget._replot()
mouseMove(view.viewport(), pos)
(_, text), _ = show_text.call_args
self.assertIn("(#1) 0.600\n(#2) 0.590", text)
show_text.reset_mock()
# test nan thresholds
self.widget.roc_averaging = OWROCAnalysis.Vertical
self.widget._replot()
mouseMove(view.viewport(), pos)
show_text.assert_not_called()
def _call_predictors(self):
if not self.data:
return
if self.class_var:
domain = self.data.domain
classless_data = self.data.transform(
Domain(domain.attributes, None, domain.metas))
else:
classless_data = self.data
for inputid, slot in self.predictors.items():
if isinstance(slot.results, Results):
continue
predictor = slot.predictor
try:
if predictor.domain.class_var.is_discrete:
pred, prob = predictor(classless_data, Model.ValueProbs)
else:
pred = predictor(classless_data, Model.Value)
prob = numpy.zeros((len(pred), 0))
except (ValueError, DomainTransformationError) as err:
self.predictors[inputid] = \
slot._replace(results=f"{predictor.name}: {err}")
continue
results = Results()
results.data = self.data
results.domain = self.data.domain
results.row_indices = numpy.arange(len(self.data))
results.folds = (Ellipsis, )
results.actual = self.data.Y
results.unmapped_probabilities = prob
results.unmapped_predicted = pred
results.probabilities = results.predicted = None
self.predictors[inputid] = slot._replace(results=results)
target = predictor.domain.class_var
if target != self.class_var:
continue
if target is not self.class_var and target.is_discrete:
backmappers, n_values = predictor.get_backmappers(self.data)
prob = predictor.backmap_probs(prob, n_values, backmappers)
pred = predictor.backmap_value(pred, prob, n_values,
backmappers)
results.predicted = pred.reshape((1, len(self.data)))
results.probabilities = prob.reshape((1, ) + prob.shape)
