def _postprocess_file(self, config, source, target):
if 'tokenization' in config:
tok_config = config['tokenization']
tgt_tokenizer = tokenizer.build_tokenizer(tok_config['target'])
output = "%s.detok" % target
tokenizer.detokenize_file(tgt_tokenizer, target, output)
return output
return target
def _postprocess_file(self, config, input):
if 'tokenization' in config:
tok_config = config['tokenization']
tgt_tokenizer = tokenizer.build_tokenizer(
tok_config['target'] if 'target' in tok_config else tok_config)
output = "%s.detok" % input
tokenizer.detokenize_file(tgt_tokenizer, input, output)
return output
return input