def test_unmasked_computation(self, device: str): spearman_correlation = SpearmanCorrelation() batch_size = 10 num_labels = 10 predictions1 = torch.randn(batch_size, num_labels, device=device) labels1 = 0.5 * predictions1 + torch.randn(batch_size, num_labels, device=device) predictions2 = torch.randn(1, device=device).repeat(num_labels) predictions2 = predictions2.unsqueeze(0).expand(batch_size, -1) labels2 = torch.randn(1, device=device).expand(num_labels) labels2 = 0.5 * predictions2 + labels2.unsqueeze(0).expand(batch_size, -1) # in most cases, the data is constructed like predictions_1, the data of such a batch different. # but in a few cases, for example, predictions_2, the data of such a batch is exactly the same. predictions_labels_ = [(predictions1, labels1), (predictions2, labels2)] for predictions, labels in predictions_labels_: spearman_correlation.reset() spearman_correlation(predictions, labels) assert_allclose( spearman_formula(predictions.reshape(-1), labels.reshape(-1)), spearman_correlation.get_metric(), )
def test_unmasked_computation(self): spearman_correlation = SpearmanCorrelation() batch_size = 10 num_labels = 10 predictions1 = np.random.randn(batch_size, num_labels).astype("float32") labels1 = 0.5 * predictions1 + np.random.randn(batch_size, num_labels).astype("float32") predictions2 = np.random.randn(1).repeat(num_labels).astype("float32") predictions2 = predictions2[np.newaxis, :].repeat(batch_size, axis=0) labels2 = np.random.randn(1).repeat(num_labels).astype("float32") labels2 = 0.5 * predictions2 + labels2[np.newaxis, :].repeat(batch_size, axis=0) # in most cases, the data is constructed like predictions_1, the data of such a batch different. # but in a few cases, for example, predictions_2, the data of such a batch is exactly the same. predictions_labels_ = [(predictions1, labels1), (predictions2, labels2)] for predictions, labels in predictions_labels_: spearman_correlation.reset() spearman_correlation(torch.FloatTensor(predictions), torch.FloatTensor(labels)) assert_allclose( spearman_formula(predictions.reshape(-1), labels.reshape(-1)), spearman_correlation.get_metric(), rtol=1e-5, )
def test_distributed_spearman(self): batch_size = 10 num_labels = 10 predictions = torch.randn(batch_size, num_labels) labels = 0.5 * predictions + torch.randn(batch_size, num_labels) desired_spearman = spearman_formula(predictions.reshape(-1), labels.reshape(-1)) predictions = [predictions[:5], predictions[5:]] labels = [labels[:5], labels[5:]] metric_kwargs = {"predictions": predictions, "gold_labels": labels} run_distributed_test( [-1, -1], global_distributed_metric, SpearmanCorrelation(), metric_kwargs, desired_spearman, exact=False, )
def test_reset(self, device: str): spearman_correlation = SpearmanCorrelation() batch_size = 10 num_labels = 10 predictions = torch.randn(batch_size, num_labels, device=device) labels = 0.5 * predictions + torch.randn( batch_size, num_labels, device=device) # 1.test spearman_correlation.reset() spearman_correlation.reset() spearman_correlation(predictions, labels) temp = spearman_correlation.get_metric() spearman_correlation.reset() spearman_correlation(predictions, labels) assert spearman_correlation.get_metric() == temp # 2.test spearman_correlation.reset() spearman_correlation.reset() spearman_correlation(predictions, labels) spearman_correlation.get_metric(reset=False) assert spearman_correlation.get_metric() != float("NaN") spearman_correlation.get_metric(reset=True) assert math.isnan(spearman_correlation.get_metric())
def test_reset(self): spearman_correlation = SpearmanCorrelation() batch_size = 10 num_labels = 10 predictions = np.random.randn(batch_size, num_labels).astype("float32") labels = 0.5 * predictions + np.random.randn( batch_size, num_labels).astype("float32") # 1.test spearman_correlation.reset() spearman_correlation.reset() spearman_correlation(torch.FloatTensor(predictions), torch.FloatTensor(labels)) temp = spearman_correlation.get_metric() spearman_correlation.reset() spearman_correlation(torch.FloatTensor(predictions), torch.FloatTensor(labels)) assert spearman_correlation.get_metric() == temp # 2.test spearman_correlation.reset() spearman_correlation.reset() spearman_correlation(torch.FloatTensor(predictions), torch.FloatTensor(labels)) spearman_correlation.get_metric(reset=False) assert spearman_correlation.get_metric() != np.nan spearman_correlation.get_metric(reset=True) assert math.isnan(spearman_correlation.get_metric())