def _get_sent_cider( hypothesis: List[str], references: List[List[str]], extra_args: Optional[Dict[str, str]] = None) -> List[float]: n_workers = get_optional_dict(extra_args, 'n_workers', 1) verbose = get_optional_dict(extra_args, 'verbose', False) return _CIDErScorer(n_workers=n_workers, verbose=verbose).get_sent_scores( hypothesis, references)
def _get_sent_bleu(hypothesis: List[str], references: List[List[str]], extra_args: Optional[Dict[str, str]] = None) -> List[float]: tokenizer = get_optional_dict(extra_args, 'bleu_tokenizer', 'none') data = [hypothesis] + references return [ sb.corpus_bleu([h], [[rr] for rr in r], smooth_method='floor', use_effective_order=True, force=True, tokenize=tokenizer).score for h, *r in zip(*data) ]
def _get_sent_bleu( hypothesis: List[str], references: List[List[str]], extra_args: Optional[Dict[str, str]] = None, score='score' ) -> List[float]: tokenizer = get_optional_dict(extra_args, 'tokenizer', 'none') data = [hypothesis] + references args = get_default_args(smooth_method='floor', tokenize=tokenizer, num_refs=len(references)) scorer = BLEU(args) scores = [ scorer.corpus_score([h], [[rr] for rr in r], use_effective_order=True) for h, *r in zip(*data) ] proj = {'score': lambda s: s.score, 'bp': lambda s: s.bp}.get(score) return [proj(s) for s in scores]
def score_corpus_multiprocess( self, hypothesis: List[str], references: List[List[str]], score='score' ) -> float: tokenizer = get_optional_dict(self.extra_args, 'tokenizer', 'none') args = get_default_args(tokenize=tokenizer, num_refs=len(references)) scorer = BLEU(args) if self.n_workers == 1: corpus_score = scorer.corpus_score( hypothesis, references, use_effective_order=False ) else: batches = list( self._batch(hypothesis, references, n_batches=self.n_workers) ) ref_len, sys_len = 0, 0 correct = [0 for _ in range(BLEU.NGRAM_ORDER)] total = [0 for _ in range(BLEU.NGRAM_ORDER)] with ProcessPoolExecutor(max_workers=self.n_workers) as executor: futures = [ executor.submit( scorer.corpus_score, b[0], b[1], use_effective_order=False ) for b in batches ] progress = as_completed(futures) if self.verbose: progress = tqdm(progress) for future in progress: s = future.result() ref_len += s.ref_len sys_len += s.sys_len for n in range(BLEU.NGRAM_ORDER): correct[n] += s.counts[n] total[n] += s.totals[n] corpus_score = scorer.compute_bleu( correct, total, sys_len, ref_len, smooth_method='exp' ) proj = {'score': lambda s: s.score, 'bp': lambda s: s.bp}.get(score) return proj(corpus_score)
def score_corpus_multiprocess(self, hypothesis: List[str], references: List[List[str]]) -> float: tokenizer = get_optional_dict(self.extra_args, 'bleu_tokenizer', 'none') if self.n_workers == 1: corpus_score = sb.corpus_bleu(hypothesis, references, force=True, tokenize=tokenizer).score else: batches = list( self._batch(hypothesis, references, n_batches=self.n_workers)) ref_len, sys_len = 0, 0 correct = [0 for _ in range(sb.NGRAM_ORDER)] total = [0 for _ in range(sb.NGRAM_ORDER)] with ProcessPoolExecutor(max_workers=self.n_workers) as executor: futures = [ executor.submit(sb.corpus_bleu, b[0], b[1], force=True, tokenize=tokenizer) for b in batches ] progress = as_completed(futures) if self.verbose: progress = tqdm(progress) for future in progress: s = future.result() ref_len += s.ref_len sys_len += s.sys_len for n in range(sb.NGRAM_ORDER): correct[n] += s.counts[n] total[n] += s.totals[n] corpus_score = sb.compute_bleu(correct, total, sys_len, ref_len, smooth_method='exp').score return corpus_score