def _postprocess_file(self, config, source, target): if 'tokenization' in config: tok_config = config['tokenization'] tgt_tokenizer = tokenizer.build_tokenizer(tok_config['target']) output = "%s.detok" % target tokenizer.detokenize_file(tgt_tokenizer, target, output) return output return target
def _postprocess_file(self, config, input): if 'tokenization' in config: tok_config = config['tokenization'] tgt_tokenizer = tokenizer.build_tokenizer( tok_config['target'] if 'target' in tok_config else tok_config) output = "%s.detok" % input tokenizer.detokenize_file(tgt_tokenizer, input, output) return output return input