def test_mean_avg_precision():
"""
Tests for catalyst.mean_avg_precision metric.
"""
# check 1
# Stanford Introdcution to information retrieval primer
y_pred1 = np.arange(9, -1, -1)
y_true1 = [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0]
y_pred2 = np.arange(9, -1, -1)
y_true2 = [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0]
y_pred_torch = torch.Tensor([y_pred1, y_pred2])
y_true_torch = torch.Tensor([y_true1, y_true2])
top_k = [10]
map_at10 = mean_average_precision(y_pred_torch, y_true_torch, top_k)[0]
assert np.allclose(map_at10, 0.5325, atol=1e-3)
# check 2
# map_at1: (1.0 + 0.0) / 2 = 0.5
# map_at3: ((1 + 0.67)/2 + 0.5) / 2 = 0.6675
# map_at5: ((1 + 0.67)/2 + (0.5 + 0.4)/2) / 2 = 0.6425
# map_at10: ((1 + 0.67 + 0.5 + 0.44 + 0.5)/5 + (0.5 + 0.4 + 0.43)/3 ) / 2 = 0.53
y_pred1 = [9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
y_pred2 = [9, 8, 7, 6, 5, 4, 3, 2, 1, 0]
y_true1 = [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0]
y_true2 = [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0]
y_pred_torch = torch.Tensor([y_pred1, y_pred2])
y_true_torch = torch.Tensor([y_true1, y_true2])
top_k = [1, 3, 5, 10]
map_k = mean_average_precision(y_pred_torch, y_true_torch, top_k)
map_at1 = map_k[0]
map_at3 = map_k[1]
map_at5 = map_k[2]
map_at10 = map_k[3]
assert np.allclose(map_at1, 0.5, atol=1e-3)
assert np.allclose(map_at3, 0.6675, atol=1e-3)
assert np.allclose(map_at5, 0.6425, atol=1e-3)
assert np.allclose(map_at10, 0.5325, atol=1e-3)
def update(self, logits: torch.Tensor,
targets: torch.Tensor) -> List[float]:
"""
Update metric value with map for new data and return intermediate metrics values.
Args:
logits (torch.Tensor): tensor of logits
targets (torch.Tensor): tensor of targets
Returns:
list of map@k values
"""
values = mean_average_precision(logits, targets, topk=self.topk_args)
values = [v.item() for v in values]
for value, metric in zip(values, self.additive_metrics):
metric.update(value, len(targets))
return values
def test_mean_avg_precision():
"""
Tests for catalyst.mean_avg_precision metric.
"""
# check 1
# Stanford Introdcution to information retrieval primer
y_pred1 = np.arange(9, -1, -1)
y_true1 = [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0]
y_pred2 = np.arange(9, -1, -1)
y_true2 = [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0]
y_pred_torch = torch.Tensor([y_pred1, y_pred2])
y_true_torch = torch.Tensor([y_true1, y_true2])
top_k = [10]
map_at10 = mean_average_precision(y_pred_torch, y_true_torch, top_k)[0]
assert np.allclose(map_at10, 0.5325, atol=1e-3)