def build_model(self, args):
model = super().build_model(args)
if args['task']['eval_bleu']:
assert args['task']['eval_bleu_detok'] is not None, (
'--eval-bleu-detok is required if using --eval-bleu; '
'try --eval-bleu-detok=moses (or --eval-bleu-detok=space '
'to disable detokenization, e.g., when using sentencepiece)')
detok_args = json.loads(
args['task']['eval_bleu_detok_args']
if args['task']['eval_bleu_detok_args'] else '{}')
self.tokenizer = tokenizers.build_tokenizer(
dict(tokenizer=args['task'].get('eval_bleu_detok', '{}'),
**detok_args))
self.sequence_generator = self.build_generator([model], args)
return model
def build_model(self, args):
model = super().build_model(args)
if args['task']['eval_bleu']:
assert args['task']['eval_bleu_detok'] is not None, (
'--eval-bleu-detok is required if using --eval-bleu; '
'try --eval-bleu-detok=moses (or --eval-bleu-detok=space '
'to disable detokenization, e.g., when using sentencepiece)')
detok_args = json.loads(
args['task']['eval_bleu_detok_args']
if args['task']['eval_bleu_detok_args'] else '{}')
self.tokenizer = tokenizers.build_tokenizer(
dict(
tokenizer=args['task']['eval_bleu_detok']
if args['task']['eval_bleu_detok'] else None,
# getattr(args, 'eval_bleu_detok', None),
**detok_args))
# The gen_args parameters have been set in the yml file
self.sequence_generator = self.build_generator(args)
return model
