def run_pymteval(data_ref, data_sys):
"""Run document-level BLEU and NIST in their Python implementation (should give the
same results as Perl)."""
print('Running Py-MTEval metrics...', file=sys.stderr)
bleu = BLEUScore()
nist = NISTScore()
# collect statistics
for sents_ref, sent_sys in zip(data_ref, data_sys):
bleu.append(sent_sys, sents_ref)
nist.append(sent_sys, sents_ref)
# return the computed scores
return {'NIST': nist.score(), 'BLEU': bleu.score()}
def sent_level_scores(data_src, data_ref, data_sys, out_fname):
"""Collect segment-level scores for the given data and write them out to a TSV file."""
res_data = []
headers = ['src', 'sys_out', 'BLEU', 'sentBLEU', 'NIST']
coco_scorers = ['METEOR', 'ROUGE_L', 'CIDEr']
mteval_scorers = [BLEUScore(), BLEUScore(smoothing=1.0), NISTScore()]
headers.extend(coco_scorers)
# prepare COCO scores
coco_eval = run_coco_eval(data_ref, data_sys)
# go through the segments
for inst_no, (sent_src, sents_ref,
sent_sys) in enumerate(zip(data_src, data_ref, data_sys)):
res_line = [sent_src, sent_sys]
# run the PyMTEval scorers for the given segment
for scorer in mteval_scorers:
scorer.reset()
scorer.append(sent_sys, sents_ref)
res_line.append('%.4f' % scorer.score())
# extract the segment-level scores from the COCO object
for coco_scorer in coco_scorers:
res_line.append(
'%.4f' % coco_eval.imgToEval['inst-%d' % inst_no][coco_scorer])
# collect the results
res_data.append(res_line)
# write the output file
write_tsv(out_fname, headers, res_data)