def test_covariance_masked_computation(self):
covariance = Covariance()
batch_size = 100
num_labels = 10
predictions = np.random.randn(batch_size, num_labels).astype("float32")
labels = 0.5 * predictions + np.random.randn(batch_size, num_labels).astype("float32")
# Random binary mask
mask = np.random.randint(0, 2, size=(batch_size, num_labels)).astype("float32")
stride = 10
for i in range(batch_size // stride):
timestep_predictions = torch.FloatTensor(predictions[stride * i:stride * (i+1), :])
timestep_labels = torch.FloatTensor(labels[stride * i:stride * (i+1), :])
timestep_mask = torch.FloatTensor(mask[stride * i:stride * (i+1), :])
# Flatten the predictions, labels, and mask thus far, so numpy treats them as
# independent observations.
expected_covariance = np.cov(predictions[:stride * (i + 1), :].reshape(-1),
labels[:stride * (i + 1), :].reshape(-1),
fweights=mask[:stride * (i + 1), :].reshape(-1))[0, 1]
covariance(timestep_predictions, timestep_labels, timestep_mask)
assert_allclose(expected_covariance, covariance.get_metric(), rtol=1e-5)
# Test reset
covariance.reset()
covariance(torch.FloatTensor(predictions), torch.FloatTensor(labels), torch.FloatTensor(mask))
assert_allclose(np.cov(predictions.reshape(-1), labels.reshape(-1), fweights=mask.reshape(-1))[0, 1],
covariance.get_metric(), rtol=1e-5)
def __init__(self, word_embeddings: TextFieldEmbedder,
encoder: Seq2VecEncoder, vocab: Vocabulary) -> None:
super().__init__(vocab)
self.word_embeddings = word_embeddings
self.encoder = encoder
hidden_dim = 128
self.mlp = torch.nn.Sequential(
torch.nn.Linear(in_features=encoder.get_output_dim() * 4,
out_features=hidden_dim), torch.nn.Tanh(),
torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim),
torch.nn.Tanh(),
torch.nn.Linear(in_features=hidden_dim, out_features=1))
self.covar = Covariance()
self.pearson = PearsonCorrelation()
def test_distributed_covariance(self):
batch_size = 10
num_labels = 10
predictions = torch.randn(batch_size, num_labels)
labels = 0.5 * predictions + torch.randn(batch_size, num_labels)
# Random binary mask
mask = torch.randint(0, 2, size=(batch_size, num_labels)).bool()
expected_covariance = np.cov(
predictions.view(-1).cpu().numpy(),
labels.view(-1).cpu().numpy(),
fweights=mask.view(-1).cpu().numpy(),
)[0, 1]
predictions = [predictions[:5], predictions[5:]]
labels = [labels[:5], labels[5:]]
mask = [mask[:5], mask[5:]]
metric_kwargs = {
"predictions": predictions,
"gold_labels": labels,
"mask": mask
}
run_distributed_test(
[-1, -1],
global_distributed_metric,
Covariance(),
metric_kwargs,
expected_covariance,
exact=(0.0001, 1e-01),
)
def test_covariance_masked_computation(self, device: str):
covariance = Covariance()
batch_size = 100
num_labels = 10
predictions = torch.randn(batch_size, num_labels, device=device)
labels = 0.5 * predictions + torch.randn(batch_size, num_labels, device=device)
# Random binary mask
mask = torch.randint(0, 2, size=(batch_size, num_labels), device=device)
stride = 10
for i in range(batch_size // stride):
timestep_predictions = predictions[stride * i : stride * (i + 1), :]
timestep_labels = labels[stride * i : stride * (i + 1), :]
timestep_mask = mask[stride * i : stride * (i + 1), :]
# Flatten the predictions, labels, and mask thus far, so numpy treats them as
# independent observations.
expected_covariance = np.cov(
predictions[: stride * (i + 1), :].view(-1).cpu().numpy(),
labels[: stride * (i + 1), :].view(-1).cpu().numpy(),
fweights=mask[: stride * (i + 1), :].view(-1).cpu().numpy(),
)[0, 1]
covariance(timestep_predictions, timestep_labels, timestep_mask)
assert_allclose(expected_covariance, covariance.get_metric())
# Test reset
covariance.reset()
covariance(predictions, labels, mask)
assert_allclose(
np.cov(
predictions.view(-1).cpu().numpy(),
labels.view(-1).cpu().numpy(),
fweights=mask.view(-1).cpu().numpy(),
)[0, 1],
covariance.get_metric(),
)
def test_covariance_unmasked_computation(self):
covariance = Covariance()
batch_size = 100
num_labels = 10
predictions = np.random.randn(batch_size, num_labels).astype("float32")
labels = 0.5 * predictions + np.random.randn(
batch_size, num_labels).astype("float32")
stride = 10
for i in range(batch_size // stride):
timestep_predictions = torch.FloatTensor(
predictions[stride * i:stride * (i + 1), :])
timestep_labels = torch.FloatTensor(labels[stride * i:stride *
(i + 1), :])
# Flatten the predictions and labels thus far, so numpy treats them as
# independent observations.
expected_covariance = np.cov(
predictions[:stride * (i + 1), :].reshape(-1),
labels[:stride * (i + 1), :].reshape(-1),
)[0, 1]
covariance(timestep_predictions, timestep_labels)
assert_allclose(expected_covariance,
covariance.get_metric(),
rtol=1e-5)
# Test reset
covariance.reset()
covariance(torch.FloatTensor(predictions), torch.FloatTensor(labels))
assert_allclose(
np.cov(predictions.reshape(-1), labels.reshape(-1))[0, 1],
covariance.get_metric(),
rtol=1e-5,
)
class RuseModel(Model):
def __init__(self, word_embeddings: TextFieldEmbedder,
encoder: Seq2VecEncoder, vocab: Vocabulary) -> None:
super().__init__(vocab)
self.word_embeddings = word_embeddings
self.encoder = encoder
hidden_dim = 128
self.mlp = torch.nn.Sequential(
torch.nn.Linear(in_features=encoder.get_output_dim() * 4,
out_features=hidden_dim), torch.nn.Tanh(),
torch.nn.Linear(in_features=hidden_dim, out_features=hidden_dim),
torch.nn.Tanh(),
torch.nn.Linear(in_features=hidden_dim, out_features=1))
self.covar = Covariance()
self.pearson = PearsonCorrelation()
def forward(self, mt_sent: Dict[str, torch.Tensor],
ref_sent: Dict[str, torch.Tensor], human_score: np.ndarray,
origin: str) -> Dict[str, torch.Tensor]:
mt_mask = get_text_field_mask(mt_sent)
ref_mask = get_text_field_mask(ref_sent)
mt_embeddings = self.word_embeddings(mt_sent)
ref_embeddings = self.word_embeddings(ref_sent)
mt_encoder_out = self.encoder(mt_embeddings, mt_mask)
ref_encoder_out = self.encoder(ref_embeddings, ref_mask)
input = torch.cat((mt_encoder_out, ref_encoder_out,
torch.mul(mt_encoder_out, ref_encoder_out),
torch.abs(mt_encoder_out - ref_encoder_out)), 1)
reg = self.mlp(input)
output = {"reg": reg}
if human_score is not None:
# run metric calculation
self.covar(reg, human_score)
self.pearson(reg, human_score)
# calculate mean squared error
delta = reg - human_score
output["loss"] = torch.mul(delta, delta).sum()
return output
def get_metrics(self, reset: bool = False) -> Dict[str, float]:
return {
"covar": self.covar.get_metric(reset),
"pearson": self.pearson.get_metric(reset)
}