Exemplo n.º 1
0
    def test_reset(self):
        spearman_correlation = SpearmanCorrelation()
        batch_size = 10
        num_labels = 10
        predictions = np.random.randn(batch_size, num_labels).astype("float32")
        labels = 0.5 * predictions + np.random.randn(
            batch_size, num_labels).astype("float32")

        # 1.test spearman_correlation.reset()
        spearman_correlation.reset()
        spearman_correlation(torch.FloatTensor(predictions),
                             torch.FloatTensor(labels))
        temp = spearman_correlation.get_metric()
        spearman_correlation.reset()
        spearman_correlation(torch.FloatTensor(predictions),
                             torch.FloatTensor(labels))
        assert spearman_correlation.get_metric() == temp

        # 2.test spearman_correlation.reset()
        spearman_correlation.reset()
        spearman_correlation(torch.FloatTensor(predictions),
                             torch.FloatTensor(labels))

        spearman_correlation.get_metric(reset=False)
        assert spearman_correlation.get_metric() != np.nan
        spearman_correlation.get_metric(reset=True)
        assert math.isnan(spearman_correlation.get_metric())
Exemplo n.º 2
0
    def test_unmasked_computation(self, device: str):
        spearman_correlation = SpearmanCorrelation()
        batch_size = 10
        num_labels = 10
        predictions1 = torch.randn(batch_size, num_labels, device=device)
        labels1 = 0.5 * predictions1 + torch.randn(
            batch_size, num_labels, device=device)

        predictions2 = torch.randn(1, device=device).repeat(num_labels)
        predictions2 = predictions2.unsqueeze(0).expand(batch_size, -1)
        labels2 = torch.randn(1, device=device).expand(num_labels)
        labels2 = 0.5 * predictions2 + labels2.unsqueeze(0).expand(
            batch_size, -1)

        # in most cases, the data is constructed like predictions_1, the data of such a batch different.
        # but in a few cases, for example, predictions_2, the data of such a batch is exactly the same.
        predictions_labels_ = [(predictions1, labels1),
                               (predictions2, labels2)]

        for predictions, labels in predictions_labels_:
            spearman_correlation.reset()
            spearman_correlation(predictions, labels)
            assert_allclose(
                spearman_formula(predictions.reshape(-1), labels.reshape(-1)),
                spearman_correlation.get_metric(),
            )
Exemplo n.º 3
0
    def test_unmasked_computation(self):
        spearman_correlation = SpearmanCorrelation()
        batch_size = 10
        num_labels = 10
        predictions1 = np.random.randn(batch_size,
                                       num_labels).astype("float32")
        labels1 = 0.5 * predictions1 + np.random.randn(
            batch_size, num_labels).astype("float32")

        predictions2 = np.random.randn(1).repeat(num_labels).astype("float32")
        predictions2 = predictions2[np.newaxis, :].repeat(batch_size, axis=0)
        labels2 = np.random.randn(1).repeat(num_labels).astype("float32")
        labels2 = 0.5 * predictions2 + labels2[np.newaxis, :].repeat(
            batch_size, axis=0)

        # in most cases, the data is constructed like predictions_1, the data of such a batch different.
        # but in a few cases, for example, predictions_2, the data of such a batch is exactly the same.
        predictions_labels_ = [(predictions1, labels1),
                               (predictions2, labels2)]

        for predictions, labels in predictions_labels_:
            spearman_correlation.reset()
            spearman_correlation(torch.FloatTensor(predictions),
                                 torch.FloatTensor(labels))
            assert_allclose(
                spearman_formula(predictions.reshape(-1), labels.reshape(-1)),
                spearman_correlation.get_metric(),
                rtol=1e-5,
            )
Exemplo n.º 4
0
    def test_masked_computation(self):
        spearman_correlation = SpearmanCorrelation()
        batch_size = 10
        num_labels = 10
        predictions1 = np.random.randn(batch_size, num_labels).astype("float32")
        labels1 = 0.5 * predictions1 + np.random.randn(batch_size, num_labels).astype("float32")

        predictions2 = np.random.randn(1).repeat(num_labels).astype("float32")
        predictions2 = predictions2[np.newaxis, :].repeat(batch_size, axis=0)
        labels2 = np.random.randn(1).repeat(num_labels).astype("float32")
        labels2 = 0.5 * predictions2 + labels2[np.newaxis, :].repeat(batch_size, axis=0)

        # in most cases, the data is constructed like predictions_1, the data of such a batch different.
        # but in a few cases, for example, predictions_2, the data of such a batch is exactly the same.
        predictions_labels_ = [(predictions1, labels1), (predictions2, labels2)]

        # Random binary mask
        mask = np.random.randint(0, 2, size=(batch_size, num_labels)).astype("float32")

        for predictions, labels in predictions_labels_:
            spearman_correlation.reset()
            spearman_correlation(
                torch.FloatTensor(predictions), torch.FloatTensor(labels), torch.FloatTensor(mask)
            )
            expected_spearman_correlation = spearman_formula(
                predictions.reshape(-1), labels.reshape(-1), mask=mask.reshape(-1)
            )

            # because add mask, a batch of predictions or labels will have many 0,
            # spearman correlation algorithm will dependence the sorting position of a set of numbers,
            # too many identical numbers will result in different calculation results each time
            # but the positive and negative results are the same,
            # so here we only test the positive and negative results of the results.
            assert (expected_spearman_correlation * spearman_correlation.get_metric()) > 0
Exemplo n.º 5
0
    def test_masked_computation(self, device: str):
        spearman_correlation = SpearmanCorrelation()
        batch_size = 10
        num_labels = 10
        predictions1 = torch.randn(batch_size, num_labels, device=device)
        labels1 = 0.5 * predictions1 + torch.randn(batch_size, num_labels, device=device)

        predictions2 = torch.randn(1, device=device).expand(num_labels)
        predictions2 = predictions2.unsqueeze(0).expand(batch_size, -1)
        labels2 = torch.randn(1, device=device).expand(num_labels)
        labels2 = 0.5 * predictions2 + labels2.unsqueeze(0).expand(batch_size, -1)

        # in most cases, the data is constructed like predictions_1, the data of such a batch different.
        # but in a few cases, for example, predictions_2, the data of such a batch is exactly the same.
        predictions_labels_ = [(predictions1, labels1), (predictions2, labels2)]

        # Random binary mask
        mask = torch.randint(0, 2, size=(batch_size, num_labels), device=device).bool()

        for predictions, labels in predictions_labels_:
            spearman_correlation.reset()
            spearman_correlation(predictions, labels, mask)
            expected_spearman_correlation = spearman_formula(
                predictions.view(-1), labels.view(-1), mask=mask.view(-1)
            )

            # because add mask, a batch of predictions or labels will have many 0,
            # spearman correlation algorithm will dependence the sorting position of a set of numbers,
            # too many identical numbers will result in different calculation results each time
            # but the positive and negative results are the same,
            # so here we only test the positive and negative results of the results.
            assert (expected_spearman_correlation * spearman_correlation.get_metric()) > 0
Exemplo n.º 6
0
    def test_reset(self, device: str):
        spearman_correlation = SpearmanCorrelation()
        batch_size = 10
        num_labels = 10
        predictions = torch.randn(batch_size, num_labels, device=device)
        labels = 0.5 * predictions + torch.randn(batch_size, num_labels, device=device)

        # 1.test spearman_correlation.reset()
        spearman_correlation.reset()
        spearman_correlation(predictions, labels)
        temp = spearman_correlation.get_metric()
        spearman_correlation.reset()
        spearman_correlation(predictions, labels)
        assert spearman_correlation.get_metric() == temp

        # 2.test spearman_correlation.reset()
        spearman_correlation.reset()
        spearman_correlation(predictions, labels)

        spearman_correlation.get_metric(reset=False)
        assert spearman_correlation.get_metric() != float("NaN")
        spearman_correlation.get_metric(reset=True)
        assert math.isnan(spearman_correlation.get_metric())