def _compute( self, predictions, references, normalized: bool = False, no_punct: bool = False, asian_support: bool = False, case_sensitive: bool = False, ): references_per_prediction = len(references[0]) if any(len(refs) != references_per_prediction for refs in references): raise ValueError( "Sacrebleu requires the same number of references for each prediction" ) transformed_references = [[refs[i] for refs in references] for i in range(references_per_prediction)] sb_ter = TER(normalized, no_punct, asian_support, case_sensitive) output = sb_ter.corpus_score(predictions, transformed_references) return { "score": output.score, "num_edits": output.num_edits, "ref_length": output.ref_length }
pages = "223--231", } @inproceedings{post-2018-call, title = "A Call for Clarity in Reporting {BLEU} Scores", author = "Post, Matt", booktitle = "Proceedings of the Third Conference on Machine Translation: Research Papers", month = oct, year = "2018", address = "Belgium, Brussels", publisher = "Association for Computational Linguistics", url = "https://www.aclweb.org/anthology/W18-6319", pages = "186--191", } """ _DESCRIPTION = """\ TER (Translation Edit Rate, also called Translation Error Rate) is a metric to quantify the edit operations that a hypothesis requires to match a reference translation. We use the implementation that is already present in sacrebleu (https://github.com/mjpost/sacreBLEU#ter), which in turn is inspired by the TERCOM implementation, which can be found here: https://github.com/jhclark/tercom. The implementation here is slightly different from sacrebleu in terms of the required input format. The length of the references and hypotheses lists need to be the same, so you may need to transpose your references compared to sacrebleu's required input format. See https://github.com/huggingface/datasets/issues/3154#issuecomment-950746534 See the README.md file at https://github.com/mjpost/sacreBLEU#ter for more information. """ _KWARGS_DESCRIPTION = """ Produces TER scores alongside the number of edits and reference length.
def create_scorer(self) -> None: """ :return: """ if self.utility_function_name == UTILITY_SENTENCE_CHRF_2_PRECISION: chrf_beta = 2 self.args = argparse.Namespace(chrf_order=6, chrf_beta=chrf_beta, chrf_whitespace=False, short=False) self.scorer = cached_metrics.CachedPrecisionCHRF(self.args) self.cached_scorer = True elif "chrf" in self.utility_function_name: if self.utility_function_name.endswith("balanced"): chrf_beta = 1 else: last_part = self.utility_function_name.split("-")[-1] chrf_beta = int(last_part) self.args = argparse.Namespace(chrf_order=6, chrf_beta=chrf_beta, chrf_whitespace=False, short=False) self.scorer = cached_metrics.CachedCHRF(self.args) self.cached_scorer = True elif "bleu" in self.utility_function_name: if self.utility_function_name.endswith("floor"): smooth_method = "floor" smooth_value = 0.01 elif self.utility_function_name.endswith("exp"): smooth_method = "exp" smooth_value = None elif self.utility_function_name.endswith("add-k"): smooth_method = "add-k" smooth_value = 1 else: smooth_method = "none" smooth_value = None self.args = argparse.Namespace(smooth_method=smooth_method, smooth_value=smooth_value, force=False, short=False, lc=False, tokenize=DEFAULT_TOKENIZER) self.scorer = cached_metrics.CachedBLEU(self.args) self.cached_scorer = True elif self.utility_function_name == "sentence-ter": self.args = argparse.Namespace(normalized=False, no_punct=False, asian_support=False, case_sensitive=False) self.scorer = TER(self.args) self.cached_scorer = False else: if self.utility_function_name.endswith("balanced"): meteor_alpha = 0.5 else: meteor_alpha = 0.85 self.scorer = eval_meteor.MeteorScorer(meteor_alpha=meteor_alpha) self.cached_scorer = False