def __init__(
self,
input_key: str = "targets",
output_key: str = "logits",
prefix: str = "accuracy",
topk_args: List[int] = None,
num_classes: int = None,
accuracy_args: List[int] = None,
**kwargs,
):
"""
Args:
input_key: input key to use for accuracy calculation;
specifies our `y_true`
output_key: output key to use for accuracy calculation;
specifies our `y_pred`
prefix: key for the metric's name
topk_args: specifies which accuracy@K to log:
[1] - accuracy
[1, 3] - accuracy at 1 and 3
[1, 3, 5] - accuracy at 1, 3 and 5
num_classes: number of classes to calculate ``topk_args``
if ``accuracy_args`` is None
"""
topk_args = (topk_args or accuracy_args
or get_default_topk_args(num_classes))
super().__init__(
prefix=prefix,
metric_fn=wrap_topk_metric2dict(accuracy, topk_args=topk_args),
input_key=input_key,
output_key=output_key,
**kwargs,
)
def test_wrapper_metrics():
"""
Tests for wrapper for metrics
"""
y_pred1 = np.arange(9, -1, -1)
y_true1 = [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0]
y_pred2 = np.arange(9, -1, -1)
y_true2 = [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0]
outputs = torch.Tensor([y_pred1, y_pred2])
targets = torch.Tensor([y_true1, y_true2])
topk_args = [10]
map_wrapper = wrap_topk_metric2dict(metrics.mean_avg_precision, topk_args)
map_dict = map_wrapper(outputs, targets)
map_at10 = map_dict["10"]
assert np.allclose(map_at10, 0.53, atol=1e3)
def test_wrapper_metrics():
"""
Tests for wrapper for metrics
"""
y_pred = [0.5, 0.2]
y_true = [1.0, 0.0]
topk_args = [1, 2]
outputs = torch.Tensor([y_pred])
targets = torch.Tensor([y_true])
hitrate_wrapper = wrap_topk_metric2dict(metrics.hitrate, topk_args)
hitrate_dict = hitrate_wrapper(outputs, targets)
hitrate_at1 = hitrate_dict["01"]
hitrate_at2 = hitrate_dict["02"]
true_hitrate_at1 = 1.0
true_hitrate_at2 = 0.5
assert np.isclose(true_hitrate_at1, hitrate_at1)
assert np.isclose(true_hitrate_at2, hitrate_at2)
def __init__(
self,
input_key: str = "targets",
output_key: str = "logits",
prefix: str = "accuracy",
topk_args: List[int] = None,
num_classes: int = None,
accuracy_args: List[int] = None,
**kwargs,
):
"""
Args:
input_key: input key to use for accuracy calculation;
specifies our `y_true`
output_key: output key to use for accuracy calculation;
specifies our `y_pred`
prefix: key for the metric's name
topk_args: specifies which accuracy@K to log:
[1] - accuracy
[1, 3] - accuracy at 1 and 3
[1, 3, 5] - accuracy at 1, 3 and 5
num_classes: number of classes to calculate ``topk_args``
if ``accuracy_args`` is None
**kwargs: key-value params to pass to the metric
.. note::
For ``**kwargs`` info, please follow
``catalyst.callbacks.metric.BatchMetricCallback`` and
``catalyst.metrics.accuracy.accuracy`` docs
"""
topk_args = (topk_args or accuracy_args
or get_default_topk_args(num_classes))
super().__init__(
prefix=prefix,
metric_fn=wrap_topk_metric2dict(accuracy, topk_args=topk_args),
input_key=input_key,
output_key=output_key,
**kwargs,
)
def wrapper_mrr():
"""
Tets wrapper
"""
y_pred1 = [4.0, 2.0, 3.0, 1.0]
y_pred2 = [1.0, 2.0, 3.0, 4.0]
y_true1 = [0, 0, 1.0, 1.0]
y_true2 = [0, 0, 1.0, 1.0]
topk_args = [1, 3]
output = torch.Tensor([y_pred1, y_pred2])
target = torch.Tensor([y_true1, y_true2])
mrr_wrapper = wrap_topk_metric2dict(metrics.mrr, topk_args)
mrr_dict = mrr_wrapper(output, target)
mrr_at1 = mrr_dict["01"]
mrr_at3 = mrr_dict["03"]
assert mrr_at1 == 0.5
assert mrr_at3 == 0.75
def __init__(
self,
input_key: str = "targets",
output_key: str = "logits",
prefix: str = "accuracy",
multiplier: float = 1.0,
topk_args: List[int] = None,
num_classes: int = None,
accuracy_args: List[int] = None,
**kwargs,
):
"""
Args:
input_key: input key to use for accuracy calculation;
specifies our `y_true`
output_key: output key to use for accuracy calculation;
specifies our `y_pred`
prefix: key for the metric's name
topk_args: specifies which accuracy@K to log:
[1] - accuracy
[1, 3] - accuracy at 1 and 3
[1, 3, 5] - accuracy at 1, 3 and 5
num_classes: number of classes to calculate ``topk_args``
if ``accuracy_args`` is None
activation: An torch.nn activation applied to the outputs.
Must be one of ``"none"``, ``"Sigmoid"``, or ``"Softmax"``
"""
topk_args = (topk_args or accuracy_args
or get_default_topk_args(num_classes))
super().__init__(
prefix=prefix,
metric_fn=wrap_topk_metric2dict(accuracy, topk_args=topk_args),
input_key=input_key,
output_key=output_key,
multiplier=multiplier,
**kwargs,
)
def test_wrapper_metrics():
"""
Tests for wrapper for metrics
"""
y_pred1 = [0.5, 0.2, 0.1]
y_pred2 = [0.5, 0.2, 0.1]
y_true1 = [1.0, 0.0, 1.0]
y_true2 = [1.0, 0.0, 1.0]
outputs = torch.Tensor([y_pred1, y_pred2])
targets = torch.Tensor([y_true1, y_true2])
topk_args = [1, 2]
ndcg_wrapper = wrap_topk_metric2dict(metrics.ndcg, topk_args)
ndcg_dict = ndcg_wrapper(outputs, targets)
ndcg_at1 = ndcg_dict["01"]
ndcg_at2 = ndcg_dict["02"]
true_ndcg_at1 = 1.0
true_ndcg_at2 = 1.0 / (1.0 + 1 / math.log2(3))
assert np.isclose(true_ndcg_at1, ndcg_at1)
assert np.isclose(true_ndcg_at2, ndcg_at2)
