def build_model(self, args): model = super().build_model(args) if args['task']['eval_bleu']: assert args['task']['eval_bleu_detok'] is not None, ( '--eval-bleu-detok is required if using --eval-bleu; ' 'try --eval-bleu-detok=moses (or --eval-bleu-detok=space ' 'to disable detokenization, e.g., when using sentencepiece)') detok_args = json.loads( args['task']['eval_bleu_detok_args'] if args['task']['eval_bleu_detok_args'] else '{}') self.tokenizer = tokenizers.build_tokenizer( dict(tokenizer=args['task'].get('eval_bleu_detok', '{}'), **detok_args)) self.sequence_generator = self.build_generator([model], args) return model
def build_model(self, args): model = super().build_model(args) if args['task']['eval_bleu']: assert args['task']['eval_bleu_detok'] is not None, ( '--eval-bleu-detok is required if using --eval-bleu; ' 'try --eval-bleu-detok=moses (or --eval-bleu-detok=space ' 'to disable detokenization, e.g., when using sentencepiece)') detok_args = json.loads( args['task']['eval_bleu_detok_args'] if args['task']['eval_bleu_detok_args'] else '{}') self.tokenizer = tokenizers.build_tokenizer( dict( tokenizer=args['task']['eval_bleu_detok'] if args['task']['eval_bleu_detok'] else None, # getattr(args, 'eval_bleu_detok', None), **detok_args)) # The gen_args parameters have been set in the yml file self.sequence_generator = self.build_generator(args) return model