def _get_sent_bp(
hypothesis: List[str], references: List[List[str]],
extra_args: Optional[Dict[str, str]] = None
):
tokenizer = get_optional_dict(extra_args, 'bp_tokenizer', 'none')
data = [hypothesis] + references
return [
sb.corpus_bleu(
[h], [[rr] for rr in r], smooth_method='floor',
use_effective_order=True, force=True, tokenize=tokenizer
).bp
for h, *r in zip(*data)
]
def _compute_score(self):
"""Computes sacreBLEU scores for current submission."""
tokenize = '13a'
target_language_code = self.test_set.target_language.code
if target_language_code == 'ja':
# We use char-based tokenizer as MeCab was slow/unstable
tokenize = 'char-based'
elif target_language_code == 'km':
tokenize = 'char-based'
elif target_language_code == 'zh':
tokenize = 'zh'
# _msg = 'language: {0}, tokenize: {1}'.format(
# target_language_code, tokenize
# )
# print(_msg)
hyp_text_path = self.get_hyp_text(path_only=True)
ref_text_path = self.get_ref_text(path_only=True)
try:
hyp_stream = [x for x in open(hyp_text_path, encoding='utf-8')]
ref_stream = [r for r in open(ref_text_path, encoding='utf-8')]
bleu = corpus_bleu(hyp_stream, [ref_stream], tokenize=tokenize)
self.score = bleu.score
chrf = corpus_chrf(hyp_stream, ref_stream)
self.score_chrf = chrf.score
except EOFError:
# Don't set score to None, as that would trigger infinite loop
# TODO: this should provide an error message to the user
self.score = -1
self.score_chrf = None
finally:
self.save()
def score_corpus_multiprocess(
self, hypothesis: List[str], references: List[List[str]]
) -> float:
tokenizer = get_optional_dict(self.extra_args, 'bp_tokenizer', 'none')
if self.n_workers == 1:
corpus_score = sb.corpus_bleu(
hypothesis, references, force=True, tokenize=tokenizer
).bp
else:
batches = list(
self._batch(hypothesis, references, n_batches=self.n_workers)
)
ref_len, sys_len = 0, 0
correct = [0 for _ in range(sb.NGRAM_ORDER)]
total = [0 for _ in range(sb.NGRAM_ORDER)]
with ProcessPoolExecutor(max_workers=self.n_workers) as executor:
futures = [
executor.submit(
sb.corpus_bleu, b[0], b[1], force=True,
tokenize=tokenizer
)
for b in batches
]
progress = as_completed(futures)
if self.verbose:
progress = tqdm(progress)
for future in progress:
s = future.result()
ref_len += s.ref_len
sys_len += s.sys_len
for n in range(sb.NGRAM_ORDER):
correct[n] += s.counts[n]
total[n] += s.totals[n]
corpus_score = sb.compute_bleu(
correct, total, sys_len, ref_len, smooth_method='exp'
).bp
return corpus_score