def __init__( self, input_key: str = "targets", output_key: str = "logits", prefix: str = "accuracy", topk_args: List[int] = None, num_classes: int = None, accuracy_args: List[int] = None, **kwargs, ): """ Args: input_key: input key to use for accuracy calculation; specifies our `y_true` output_key: output key to use for accuracy calculation; specifies our `y_pred` prefix: key for the metric's name topk_args: specifies which accuracy@K to log: [1] - accuracy [1, 3] - accuracy at 1 and 3 [1, 3, 5] - accuracy at 1, 3 and 5 num_classes: number of classes to calculate ``topk_args`` if ``accuracy_args`` is None """ topk_args = (topk_args or accuracy_args or get_default_topk_args(num_classes)) super().__init__( prefix=prefix, metric_fn=wrap_topk_metric2dict(accuracy, topk_args=topk_args), input_key=input_key, output_key=output_key, **kwargs, )
def test_wrapper_metrics(): """ Tests for wrapper for metrics """ y_pred1 = np.arange(9, -1, -1) y_true1 = [1.0, 0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0] y_pred2 = np.arange(9, -1, -1) y_true2 = [0.0, 1.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0] outputs = torch.Tensor([y_pred1, y_pred2]) targets = torch.Tensor([y_true1, y_true2]) topk_args = [10] map_wrapper = wrap_topk_metric2dict(metrics.mean_avg_precision, topk_args) map_dict = map_wrapper(outputs, targets) map_at10 = map_dict["10"] assert np.allclose(map_at10, 0.53, atol=1e3)
def test_wrapper_metrics(): """ Tests for wrapper for metrics """ y_pred = [0.5, 0.2] y_true = [1.0, 0.0] topk_args = [1, 2] outputs = torch.Tensor([y_pred]) targets = torch.Tensor([y_true]) hitrate_wrapper = wrap_topk_metric2dict(metrics.hitrate, topk_args) hitrate_dict = hitrate_wrapper(outputs, targets) hitrate_at1 = hitrate_dict["01"] hitrate_at2 = hitrate_dict["02"] true_hitrate_at1 = 1.0 true_hitrate_at2 = 0.5 assert np.isclose(true_hitrate_at1, hitrate_at1) assert np.isclose(true_hitrate_at2, hitrate_at2)
def __init__( self, input_key: str = "targets", output_key: str = "logits", prefix: str = "accuracy", topk_args: List[int] = None, num_classes: int = None, accuracy_args: List[int] = None, **kwargs, ): """ Args: input_key: input key to use for accuracy calculation; specifies our `y_true` output_key: output key to use for accuracy calculation; specifies our `y_pred` prefix: key for the metric's name topk_args: specifies which accuracy@K to log: [1] - accuracy [1, 3] - accuracy at 1 and 3 [1, 3, 5] - accuracy at 1, 3 and 5 num_classes: number of classes to calculate ``topk_args`` if ``accuracy_args`` is None **kwargs: key-value params to pass to the metric .. note:: For ``**kwargs`` info, please follow ``catalyst.callbacks.metric.BatchMetricCallback`` and ``catalyst.metrics.accuracy.accuracy`` docs """ topk_args = (topk_args or accuracy_args or get_default_topk_args(num_classes)) super().__init__( prefix=prefix, metric_fn=wrap_topk_metric2dict(accuracy, topk_args=topk_args), input_key=input_key, output_key=output_key, **kwargs, )
def wrapper_mrr(): """ Tets wrapper """ y_pred1 = [4.0, 2.0, 3.0, 1.0] y_pred2 = [1.0, 2.0, 3.0, 4.0] y_true1 = [0, 0, 1.0, 1.0] y_true2 = [0, 0, 1.0, 1.0] topk_args = [1, 3] output = torch.Tensor([y_pred1, y_pred2]) target = torch.Tensor([y_true1, y_true2]) mrr_wrapper = wrap_topk_metric2dict(metrics.mrr, topk_args) mrr_dict = mrr_wrapper(output, target) mrr_at1 = mrr_dict["01"] mrr_at3 = mrr_dict["03"] assert mrr_at1 == 0.5 assert mrr_at3 == 0.75
def __init__( self, input_key: str = "targets", output_key: str = "logits", prefix: str = "accuracy", multiplier: float = 1.0, topk_args: List[int] = None, num_classes: int = None, accuracy_args: List[int] = None, **kwargs, ): """ Args: input_key: input key to use for accuracy calculation; specifies our `y_true` output_key: output key to use for accuracy calculation; specifies our `y_pred` prefix: key for the metric's name topk_args: specifies which accuracy@K to log: [1] - accuracy [1, 3] - accuracy at 1 and 3 [1, 3, 5] - accuracy at 1, 3 and 5 num_classes: number of classes to calculate ``topk_args`` if ``accuracy_args`` is None activation: An torch.nn activation applied to the outputs. Must be one of ``"none"``, ``"Sigmoid"``, or ``"Softmax"`` """ topk_args = (topk_args or accuracy_args or get_default_topk_args(num_classes)) super().__init__( prefix=prefix, metric_fn=wrap_topk_metric2dict(accuracy, topk_args=topk_args), input_key=input_key, output_key=output_key, multiplier=multiplier, **kwargs, )
def test_wrapper_metrics(): """ Tests for wrapper for metrics """ y_pred1 = [0.5, 0.2, 0.1] y_pred2 = [0.5, 0.2, 0.1] y_true1 = [1.0, 0.0, 1.0] y_true2 = [1.0, 0.0, 1.0] outputs = torch.Tensor([y_pred1, y_pred2]) targets = torch.Tensor([y_true1, y_true2]) topk_args = [1, 2] ndcg_wrapper = wrap_topk_metric2dict(metrics.ndcg, topk_args) ndcg_dict = ndcg_wrapper(outputs, targets) ndcg_at1 = ndcg_dict["01"] ndcg_at2 = ndcg_dict["02"] true_ndcg_at1 = 1.0 true_ndcg_at2 = 1.0 / (1.0 + 1 / math.log2(3)) assert np.isclose(true_ndcg_at1, ndcg_at1) assert np.isclose(true_ndcg_at2, ndcg_at2)