def test_rank_classification_raise(self):
with self.assertRaisesWithLiteralMatch(
ValueError,
"`targets` should contain 3 elements but has 2."):
metrics.rank_classification(
[
((0, 0), True),
((0, 1), True),
],
[
0.1, 0.5
],
num_classes=2)
with self.assertRaisesWithLiteralMatch(
ValueError,
"The first element of `targets` ('idx') should be 2-dimensional. Got "
"0."):
metrics.rank_classification(
[
(0, True, 1.0),
(0, True, 1.0),
],
[
0.1, 0.5
],
num_classes=2)
def test_rank_classification(self):
self.assertDictClose(
metrics.rank_classification(
[0, 0,
1, 1,
0, 0,
0, 0],
[0.1, 0.5,
1.0, 1.1,
0.3, 0.1,
0.6, 0.5]),
{
"accuracy": 75.,
"f1": 66.6666667,
"auc-roc": 66.6666667,
"auc-pr": 70.8333333
})
self.assertDictClose(
metrics.rank_classification(
[1, 1, 1,
0, 0, 0,
2, 2, 2],
[0.1, 0.5, 0.0,
-2, -1, -3,
3.0, 3.1, 3.2],
num_classes=3),
{
"accuracy": 66.6666667,
"mean_3class_f1": 55.5555556,
"auc-roc": 50.0,
"auc-pr": 61.1111111
})
def test_rank_classification_custom_weights(self):
# num_classes = 2
self.assertDictClose(
metrics.rank_classification(
[(0, True, 0.2), (0, False, 0.2),
(1, False, 1.0), (1, True, 1.0),
(0, True, 0.8), (0, False, 0.8),
(0, True, 0.5), (0, False, 0.5),],
[0.1, 0.5,
1.0, 1.1,
0.3, 0.1,
0.6, 0.5],
num_classes=2),
{
"accuracy": 92.0,
"auc-pr": 89.0,
"auc-roc": 86.6666667,
"f1": 90.9090909,
})
# num_classes = 3
self.assertDictClose(
metrics.rank_classification(
[
# 1
(0, False, 0.2),
(0, True, 0.2),
(0, False, 0.2),
# 0
(1, True, 0.5),
(1, False, 0.5),
(1, False, 0.5),
# 2
(2, False, 1.0),
(2, False, 1.0),
(2, True, 1.0)
],
[0.1, 0.5, 0.0, -2, -1, -3, 3.0, 3.1, 3.2],
num_classes=3),
{
"accuracy": 70.5882353,
"auc-pr": 52.6610644,
"auc-roc": 55.5555556,
"mean_3class_f1": 48.1481481,
})
# num_classes = None, multi-answer
self.assertDictClose(
metrics.rank_classification(
[(0, False, 0.2), (0, True, 0.2), # 1
(1, True, 0.5), (1, False, 0.5), (1, True, 0.5), # 0, 3
(2, True, 1.0)], # 1
[0.1, 0.5,
-2, -1, -3,
3.0],
num_classes=None),
{
"accuracy": 70.5882353,
})
def test_rank_classification_raise(self):
with self.assertRaisesRegex(
ValueError,
"`targets` should contain three elements. Only 2 are provided."):
metrics.rank_classification([
(0, True),
(0, True),
(0, False),
], [0.1, 0.5, 1.0],
num_classes=2)
def test_rank_classification_normalized(self):
# num_classes = 2
self.assertDictClose(
metrics.rank_classification(
[
# 0
((0, 0), True, 1.0, 5),
((0, 1), False, 1.0, 10),
# 1
((1, 0), False, 1.0, 2),
((1, 1), True, 1.0, 3),
# 0
((2, 0), True, 1.0, 5),
((2, 1), False, 1.0, 6),
# 0
((3, 0), True, 1.0, 3),
((3, 1), False, 1.0, 2),
],
[
0.5, 5.0,
2.0, 3.3,
1.5, 0.6,
1.8, 1.0
],
num_classes=2,
normalize_by_target_length=True,),
{
"accuracy": 75.,
"auc-pr": 50.0,
"auc-roc": 66.6666667,
"f1": 66.6666667,
})
def test_rank_classification_shuffled(self):
# num_classes = 2
self.assertDictClose(
metrics.rank_classification(
[
((3, 0), True, 0.5),
((0, 0), True, 0.2),
((1, 0), False, 1.0),
((1, 1), True, 1.0),
((2, 0), True, 0.8),
((2, 1), False, 0.8),
((3, 1), False, 0.5),
((0, 1), False, 0.2),
],
[
0.6,
0.1,
1.0,
1.1,
0.3,
0.1,
0.5,
0.5,
],
num_classes=2),
{
"accuracy": 92.0,
"auc-pr": 83.3333333,
"auc-roc": 86.6666667,
"f1": 90.9090909,
})
# num_classes = 3
self.assertDictClose(
metrics.rank_classification(
[
((0, 0), False, 0.2),
((2, 1), False, 1.0),
((0, 1), True, 0.2),
((1, 0), True, 0.5),
((1, 1), False, 0.5),
((1, 2), False, 0.5),
((0, 2), False, 0.2),
((2, 0), False, 1.0),
((2, 2), True, 1.0)
],
[
0.1,
3.1,
0.5,
-2,
-1,
-3,
0.0,
3.0,
3.2
],
num_classes=3),
{
"accuracy": 70.5882353,
"mean_3class_f1": 48.1481481,
})
# num_classes = None, multi-answer
self.assertDictClose(
metrics.rank_classification(
[
((0, 0), False, 0.2),
((2, 0), True, 1.0),
((0, 1), True, 0.2),
((1, 2), True, 0.5),
((1, 0), True, 0.5),
((1, 1), False, 0.5),
],
[
0.1,
3.0,
0.5,
-3,
-2,
-1,
],
num_classes=None),
{
"accuracy": 70.5882353,
})
def test_rank_classification_default_weights(self):
# num_classes = 2
self.assertDictClose(
metrics.rank_classification(
[
# 0
((0, 0), True, 1.0),
((0, 1), False, 1.0),
# 1
((1, 0), False, 1.0),
((1, 1), True, 1.0),
# 0
((2, 0), True, 1.0),
((2, 1), False, 1.0),
# 0
((3, 0), True, 1.0),
((3, 1), False, 1.0),
],
[
0.1, 0.5,
1.0, 1.1,
0.3, 0.1,
0.6, 0.5
],
num_classes=2),
{
"accuracy": 75.,
"auc-pr": 50.0,
"auc-roc": 66.6666667,
"f1": 66.6666667,
})
# num_classes = 3
self.assertDictClose(
metrics.rank_classification(
[
# 1
((0, 0), False, 1.0),
((0, 1), True, 1.0),
((0, 2), False, 1.0),
# 0
((1, 0), True, 1.0),
((1, 1), False, 1.0),
((1, 2), False, 1.0),
# 2
((2, 0), False, 1.0),
((2, 1), False, 1.0),
((2, 2), True, 1.0)
],
[
0.1, 0.5, 0.0,
-2, -1, -3,
3.0, 3.1, 3.2
],
num_classes=3),
{
"accuracy": 66.6666667,
"mean_3class_f1": 55.5555556,
})
# num_classes = 3, multi-label
self.assertDictClose(
metrics.rank_classification(
[
# 1
((0, 0), False, 1.0),
((0, 1), True, 1.0),
((0, 2), False, 1.0),
# 0, 2
((1, 0), True, 1.0),
((1, 1), False, 1.0),
((1, 2), True, 1.0),
# 1, 2
((2, 0), False, 1.0),
((2, 1), True, 1.0),
((2, 2), True, 1.0)
],
[
0.1, 0.5, 0.0,
-2, -1, -3,
3.0, 3.1, 3.2
],
num_classes=3),
{
"accuracy": 66.6666667,
})
# num_classes = None, multi-answer
self.assertDictClose(
metrics.rank_classification(
[
# 1
((0, 0), False, 1.0),
((0, 1), True, 1.0),
# 0, 3
((1, 0), True, 1.0),
((1, 1), False, 1.0),
((1, 2), True, 1.0),
# 0
((2, 0), True, 1.0)
],
[
0.1, 0.5,
-2, -1, -3,
3.0
],
num_classes=None),
{
"accuracy": 66.6666667,
})
def test_rank_classification(self):
# num_classes = 2
self.assertDictClose(
metrics.rank_classification([
(0, True),
(0, False),
(1, False),
(1, True),
(0, True),
(0, False),
(0, True),
(0, False),
], [0.1, 0.5, 1.0, 1.1, 0.3, 0.1, 0.6, 0.5],
num_classes=2), {
"accuracy": 75.,
"auc-pr": 70.8333333,
"auc-roc": 66.6666667,
"f1": 66.6666667,
})
# num_classes = 3
self.assertDictClose(
metrics.rank_classification(
[
# 1
(0, False),
(0, True),
(0, False),
# 0
(1, True),
(1, False),
(1, False),
# 2
(2, False),
(2, False),
(2, True)
],
[0.1, 0.5, 0.0, -2, -1, -3, 3.0, 3.1, 3.2],
num_classes=3),
{
"accuracy": 66.6666667,
"auc-pr": 61.1111111,
"auc-roc": 50.0,
"mean_3class_f1": 55.5555556,
})
# num_classes = 3, multi-label
self.assertDictClose(
metrics.rank_classification(
[
(0, False),
(0, True),
(0, False), # 1
(1, True),
(1, False),
(1, True), # 0, 2
(2, False),
(2, True),
(2, True)
], # 1, 2
[0.1, 0.5, 0.0, -2, -1, -3, 3.0, 3.1, 3.2],
num_classes=3),
{
"accuracy": 66.6666667,
})
# num_classes = None, multi-answer
self.assertDictClose(
metrics.rank_classification(
[
(0, False),
(0, True), # 1
(1, True),
(1, False),
(1, True), # 0, 3
(2, True)
], # 1
[0.1, 0.5, -2, -1, -3, 3.0],
num_classes=None),
{
"accuracy": 66.6666667,
})
