def generate(args): assert_test_corpus_and_vocab_files_specified(args) assert args.path is not None, "--path required for generation!" print(args) if args.source_lang is None: args.source_lang = "src" if args.target_lang is None: args.target_lang = "tgt" src_dict = pytorch_translate_dictionary.Dictionary.load( args.source_vocab_file) dst_dict = pytorch_translate_dictionary.Dictionary.load( args.target_vocab_file) dataset = data.LanguageDatasets(src=args.source_lang, dst=args.target_lang, src_dict=src_dict, dst_dict=dst_dict) models, model_args = load_diverse_ensemble_for_inference( args.path, dataset.src_dict, dataset.dst_dict) append_eos_to_source = model_args[0].append_eos_to_source reverse_source = model_args[0].reverse_source assert all(a.append_eos_to_source == append_eos_to_source and a.reverse_source == reverse_source for a in model_args) dataset.splits[ args. gen_subset] = pytorch_translate_data.make_language_pair_dataset_from_text( source_text_file=args.source_text_file, target_text_file=args.target_text_file, source_dict=src_dict, target_dict=dst_dict, append_eos=append_eos_to_source, reverse_source=reverse_source, ) if args.source_lang is None or args.target_lang is None: # record inferred languages in args args.source_lang, args.target_lang = dataset.src, dataset.dst print(f"| [{dataset.src}] dictionary: {len(dataset.src_dict)} types") print(f"| [{dataset.dst}] dictionary: {len(dataset.dst_dict)} types") print( f"| {args.gen_subset} {len(dataset.splits[args.gen_subset])} examples") scorer, num_sentences, gen_timer = _generate_score( models=models, args=args, dataset=dataset, dataset_split=args.gen_subset) print(f"| Translated {num_sentences} sentences ({gen_timer.n} tokens) " f"in {gen_timer.sum:.1f}s ({1. / gen_timer.avg:.2f} tokens/s)") print(f"| Generate {args.gen_subset} with beam={args.beam}: " f"{scorer.result_string()}") return scorer.score()
def generate(args): assert_test_corpus_and_vocab_files_specified(args) assert args.path is not None, "--path required for generation!" print(args) # Benchmarking should be language-agnostic args.source_lang = "src" args.target_lang = "tgt" src_dict = pytorch_translate_dictionary.Dictionary.load( args.source_vocab_file) dst_dict = pytorch_translate_dictionary.Dictionary.load( args.target_vocab_file) # Generate synthetic raw text files source_text_file = generate_synthetic_text(args.source_lang, src_dict.symbols, args) target_text_file = generate_synthetic_text(args.target_lang, src_dict.symbols, args) dataset = data.LanguageDatasets(src=args.source_lang, dst=args.target_lang, src_dict=src_dict, dst_dict=dst_dict) models, model_args = pytorch_translate_utils.load_diverse_ensemble_for_inference( args.path, dataset.src_dict, dataset.dst_dict) append_eos_to_source = model_args[0].append_eos_to_source reverse_source = model_args[0].reverse_source assert all(a.append_eos_to_source == append_eos_to_source and a.reverse_source == reverse_source for a in model_args) dataset.splits[ args. gen_subset] = pytorch_translate_data.make_language_pair_dataset_from_text( source_text_file=source_text_file, target_text_file=target_text_file, source_dict=src_dict, target_dict=dst_dict, append_eos=append_eos_to_source, reverse_source=reverse_source, ) # Remove temporary text files os.remove(source_text_file) os.remove(target_text_file) if args.source_lang is None or args.target_lang is None: # record inferred languages in args args.source_lang, args.target_lang = dataset.src, dataset.dst print(f"| [{dataset.src}] dictionary: {len(dataset.src_dict)} types") print(f"| [{dataset.dst}] dictionary: {len(dataset.dst_dict)} types") print( f"| {args.gen_subset} {len(dataset.splits[args.gen_subset])} examples") args.keep_detailed_timing = True scorer, num_sentences, gen_timer = pytorch_translate_generate._generate_score( models=models, args=args, dataset=dataset, dataset_split=args.gen_subset) # Remove contribution of primer sentence gen_timer.reset_bucket(0) print( f"| Translated {num_sentences} sentences ({sum(gen_timer.n)} tokens) " f"in {sum(gen_timer.sum):.3f}s ({1. / gen_timer.avg:.2f} tokens/s)") for bucket_id in range(gen_timer.n_buckets): if gen_timer.n[bucket_id] != 0: print( " | Length {}: {} sentences ({} tok) in {:.3f}s ({:.3f} tok/s, avg. latency {:4f}s)" .format( bucket_id * args.increment, gen_timer.count[bucket_id], gen_timer.n[bucket_id], gen_timer.sum[bucket_id], 1. / gen_timer.avgs[bucket_id], gen_timer.sum[bucket_id] / gen_timer.count[bucket_id], )) print(f"| Generate {args.gen_subset} with beam={args.beam}: " f"{scorer.result_string()}") return scorer.score()
def generate(args): pytorch_translate_options.print_args(args) src_dict = pytorch_translate_dictionary.Dictionary.load(args.source_vocab_file) dst_dict = pytorch_translate_dictionary.Dictionary.load(args.target_vocab_file) use_char_source = args.char_source_vocab_file != "" if use_char_source: char_source_dict = pytorch_translate_dictionary.Dictionary.load( args.char_source_vocab_file ) # this attribute is used for CharSourceModel construction args.char_source_dict_size = len(char_source_dict) else: char_source_dict = None dataset = data.LanguageDatasets( src=args.source_lang, dst=args.target_lang, src_dict=src_dict, dst_dict=dst_dict ) models, model_args = pytorch_translate_utils.load_diverse_ensemble_for_inference( args.path, dataset.src_dict, dataset.dst_dict ) append_eos_to_source = model_args[0].append_eos_to_source reverse_source = model_args[0].reverse_source assert all( a.append_eos_to_source == append_eos_to_source and a.reverse_source == reverse_source for a in model_args ) if args.source_binary_file != "": assert args.target_binary_file != "" dst_dataset = pytorch_translate_data.InMemoryNumpyDataset.create_from_file( args.target_binary_file ) if use_char_source: src_dataset = char_data.InMemoryNumpyWordCharDataset.create_from_file( args.source_binary_file ) gen_split = char_data.LanguagePairSourceCharDataset( src=src_dataset, dst=dst_dataset, pad_idx=src_dict.pad(), eos_idx=dst_dict.eos(), ) else: src_dataset = pytorch_translate_data.InMemoryNumpyDataset.create_from_file( args.source_binary_file ) gen_split = data.LanguagePairDataset( src=src_dataset, dst=dst_dataset, pad_idx=src_dict.pad(), eos_idx=dst_dict.eos(), ) elif pytorch_translate_data.is_multilingual(args): gen_split = pytorch_translate_data.make_language_pair_dataset_from_text_multilingual( source_text_file=args.source_text_file[0], target_text_file=args.target_text_file, source_lang_id=args.multiling_source_lang_id, target_lang_id=args.multiling_target_lang_id, source_dict=src_dict, target_dict=dst_dict, append_eos=append_eos_to_source, reverse_source=reverse_source, ) elif args.source_ensembling: gen_split = multisource_data.make_multisource_language_pair_dataset_from_text( source_text_files=args.source_text_file, target_text_file=args.target_text_file, source_dict=src_dict, target_dict=dst_dict, append_eos=append_eos_to_source, reverse_source=reverse_source, ) else: gen_split = pytorch_translate_data.make_language_pair_dataset_from_text( source_text_file=args.source_text_file[0], target_text_file=args.target_text_file, source_dict=src_dict, target_dict=dst_dict, append_eos=append_eos_to_source, reverse_source=reverse_source, char_source_dict=char_source_dict, ) dataset.splits[args.gen_subset] = gen_split if args.source_lang is None or args.target_lang is None: # record inferred languages in args args.source_lang, args.target_lang = dataset.src, dataset.dst print(f"| [{dataset.src}] dictionary: {len(dataset.src_dict)} types") print(f"| [{dataset.dst}] dictionary: {len(dataset.dst_dict)} types") print(f"| {args.gen_subset} {len(dataset.splits[args.gen_subset])} examples") scorer, num_sentences, gen_timer, _ = _generate_score( models=models, args=args, dataset=dataset, dataset_split=args.gen_subset ) print( f"| Translated {num_sentences} sentences ({gen_timer.n} tokens) " f"in {gen_timer.sum:.1f}s ({1. / gen_timer.avg:.2f} tokens/s)" ) print( f"| Generate {args.gen_subset} with beam={args.beam}: " f"{scorer.result_string()}" ) return scorer.score()