def eval_bleu(self, source_test_file="", target_test_file=""):
self.eval()
print("Evaluating BLEU")
references = []
with open(target_test_file, "r") as f:
for line in f.readlines():
references.append([line.strip().replace('@@ ', "").split(" ")])
source_sentences = []
with open(source_test_file, "r") as f:
for line in f.readlines():
source_sentences.append(line.strip())
source_sentences = source_sentences
translations = []
for sentence in source_sentences:
translation, _, _ = self.translate(sentence, max_length=d.MAX_LEN)
translations.append(" ".join(translation[:-1]).replace('@@ ',
"").split())
# import nltk
# bleu = nltk.translate.bleu_score.corpus_bleu(references, translations)
bleu = bleu_score(translations, references)
return bleu
def validation_epoch_end(self, validation_step_outputs):
pred_corpus = []
tgt_corpus = []
for out in validation_step_outputs:
pred_corpus += out['pred_corpus']
tgt_corpus += out['tgt_corpus']
score = bleu_score(pred_corpus, tgt_corpus)
self.log('val_bleu', score)
def test_v1_5_metric_others():
translate_corpus = ['the cat is on the mat'.split()]
reference_corpus = [['there is a cat on the mat'.split(), 'a cat is on the mat'.split()]]
bleu_score._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
res = bleu_score(translate_corpus, reference_corpus)
assert torch.allclose(res, torch.tensor(0.7598), atol=1e-4)
embeddings = torch.tensor([[1., 2., 3., 4.], [1., 2., 3., 4.], [4., 5., 6., 7.]])
embedding_similarity._warned = False
with pytest.deprecated_call(match='It will be removed in v1.5.0'):
res = embedding_similarity(embeddings)
assert torch.allclose(
res, torch.tensor([[0.0000, 1.0000, 0.9759], [1.0000, 0.0000, 0.9759], [0.9759, 0.9759, 0.0000]]), atol=1e-4
)