def load_dataset(self,
split,
src_bin_path,
tgt_bin_path,
weights_file=None):
corpus = pytorch_translate_data.ParallelCorpusConfig(
source=pytorch_translate_data.CorpusConfig(
dialect=self.args.source_lang, data_file=src_bin_path),
target=pytorch_translate_data.CorpusConfig(
dialect=self.args.target_lang, data_file=tgt_bin_path),
weights_file=weights_file,
)
if self.args.log_verbose:
print("Starting to load binarized data files.", flush=True)
if not os.path.exists(corpus.source.data_file):
raise ValueError(
f"{corpus.source.data_file} for {split} not found!")
if not os.path.exists(corpus.target.data_file):
raise ValueError(
f"{corpus.target.data_file} for {split} not found!")
dst_dataset = pytorch_translate_data.InMemoryNumpyDataset.create_from_file(
corpus.target.data_file)
weights_dataset = None
if corpus.weights_file and os.path.exists(corpus.weights_file):
weights_dataset = weighted_data.IndexedWeightsDataset(
corpus.weights_file)
assert len(dst_dataset) == len(weights_dataset)
if self.char_source_dict is not None:
src_dataset = char_data.InMemoryNumpyWordCharDataset.create_from_file(
corpus.source.data_file)
self.datasets[split] = char_data.LanguagePairSourceCharDataset(
src=src_dataset,
src_sizes=src_dataset.sizes,
src_dict=self.source_dictionary,
tgt=dst_dataset,
tgt_sizes=dst_dataset.sizes,
tgt_dict=self.target_dictionary,
weights=weights_dataset,
)
else:
src_dataset = pytorch_translate_data.InMemoryNumpyDataset.create_from_file(
corpus.source.data_file)
self.datasets[split] = weighted_data.WeightedLanguagePairDataset(
src=src_dataset,
src_sizes=src_dataset.sizes,
src_dict=self.source_dictionary,
tgt=dst_dataset,
tgt_sizes=dst_dataset.sizes,
tgt_dict=self.target_dictionary,
weights=weights_dataset,
)
if self.args.log_verbose:
print("Finished loading dataset", flush=True)
print(f"| {split} {len(self.datasets[split])} examples")
def load_binarized_dataset(
train_corpus: ParallelCorpusConfig,
eval_corpus: ParallelCorpusConfig,
train_split: str,
eval_split: str,
args: argparse.Namespace,
use_char_source: bool = False,
) -> data.LanguageDatasets:
source_dict = pytorch_translate_dictionary.Dictionary.load(
args.source_vocab_file)
target_dict = pytorch_translate_dictionary.Dictionary.load(
args.target_vocab_file)
if use_char_source:
char_source_dict = pytorch_translate_dictionary.Dictionary.load(
args.char_source_vocab_file)
# this attribute is used for CharSourceModel construction
args.char_source_dict_size = len(char_source_dict)
dataset = data.LanguageDatasets(
src=train_corpus.source.dialect,
dst=train_corpus.target.dialect,
src_dict=source_dict,
dst_dict=target_dict,
)
for split, corpus in [(train_split, train_corpus),
(eval_split, eval_corpus)]:
if not os.path.exists(corpus.source.data_file):
raise ValueError(
f"{corpus.source.data_file} for {split} not found!")
if not os.path.exists(corpus.target.data_file):
raise ValueError(
f"{corpus.target.data_file} for {split} not found!")
dst_dataset = InMemoryNumpyDataset.create_from_file(
corpus.target.data_file)
if use_char_source:
src_dataset = char_data.InMemoryNumpyWordCharDataset.create_from_file(
corpus.source.data_file)
dataset.splits[split] = char_data.LanguagePairSourceCharDataset(
src=src_dataset,
dst=dst_dataset,
pad_idx=source_dict.pad(),
eos_idx=source_dict.eos(),
)
else:
src_dataset = InMemoryNumpyDataset.create_from_file(
corpus.source.data_file)
dataset.splits[split] = data.LanguagePairDataset(
src=src_dataset,
dst=dst_dataset,
pad_idx=source_dict.pad(),
eos_idx=source_dict.eos(),
)
return dataset
def load_dataset_from_text(
self,
split: str,
source_text_file: str,
target_text_file: str,
append_eos: Optional[bool] = False,
reverse_source: Optional[bool] = True,
):
dst_dataset = data.IndexedRawTextDataset(
path=target_text_file,
dictionary=self.target_dictionary,
# We always append EOS to the target sentence since we still want
# the model to output an indication the sentence has finished, even
# if we don't append the EOS symbol to the source sentence
# (to prevent the model from misaligning UNKs or other words
# to the frequently occurring EOS).
append_eos=True,
# We don't reverse the order of the target sentence, since
# even if the source sentence is fed to the model backwards,
# we still want the model to start outputting from the first word.
reverse_order=False,
)
if self.char_source_dict is not None:
src_dataset = char_data.InMemoryNumpyWordCharDataset()
src_dataset.parse(
path=source_text_file,
word_dict=self.source_dictionary,
char_dict=self.char_source_dict,
reverse_order=reverse_source,
append_eos=append_eos,
)
self.datasets[split] = char_data.LanguagePairSourceCharDataset(
src_dataset,
src_dataset.sizes,
self.source_dictionary,
dst_dataset,
dst_dataset.sizes,
self.target_dictionary,
)
else:
src_dataset = data.IndexedRawTextDataset(
path=source_text_file,
dictionary=self.source_dictionary,
append_eos=append_eos,
reverse_order=reverse_source,
)
self.datasets[split] = data.LanguagePairDataset(
src_dataset,
src_dataset.sizes,
self.source_dictionary,
dst_dataset,
dst_dataset.sizes,
self.target_dictionary,
left_pad_source=False,
)
print(f"| {split} {len(self.datasets[split])} examples")
def make_language_pair_dataset_from_text(
source_text_file: str,
target_text_file: str,
source_dict: pytorch_translate_dictionary.Dictionary,
target_dict: pytorch_translate_dictionary.Dictionary,
append_eos: Optional[bool] = False,
reverse_source: Optional[bool] = True,
char_source_dict: Optional[pytorch_translate_dictionary.Dictionary] = None,
) -> data.LanguagePairDataset:
dst_dataset = indexed_dataset.IndexedRawTextDataset(
path=target_text_file,
dictionary=target_dict,
# We always append EOS to the target sentence since we still want
# the model to output an indication the sentence has finished, even
# if we don't append the EOS symbol to the source sentence
# (to prevent the model from misaligning UNKs or other words
# to the frequently occurring EOS).
append_eos=True,
# We don't reverse the order of the target sentence, since
# even if the source sentence is fed to the model backwards,
# we still want the model to start outputting from the first word.
reverse_order=False,
)
if char_source_dict is not None:
src_dataset = char_data.InMemoryNumpyWordCharDataset()
src_dataset.parse(
path=source_text_file,
word_dict=source_dict,
char_dict=char_source_dict,
reverse_order=reverse_source,
append_eos=append_eos,
)
return char_data.LanguagePairSourceCharDataset(
src=src_dataset,
dst=dst_dataset,
pad_idx=source_dict.pad(),
eos_idx=source_dict.eos(),
)
else:
return data.LanguagePairDataset(
src=indexed_dataset.IndexedRawTextDataset(
path=source_text_file,
dictionary=source_dict,
append_eos=append_eos,
reverse_order=reverse_source,
),
dst=dst_dataset,
pad_idx=source_dict.pad(),
eos_idx=source_dict.eos(),
)
def generate(args):
pytorch_translate_options.print_args(args)
src_dict = pytorch_translate_dictionary.Dictionary.load(args.source_vocab_file)
dst_dict = pytorch_translate_dictionary.Dictionary.load(args.target_vocab_file)
use_char_source = args.char_source_vocab_file != ""
if use_char_source:
char_source_dict = pytorch_translate_dictionary.Dictionary.load(
args.char_source_vocab_file
)
# this attribute is used for CharSourceModel construction
args.char_source_dict_size = len(char_source_dict)
else:
char_source_dict = None
dataset = data.LanguageDatasets(
src=args.source_lang, dst=args.target_lang, src_dict=src_dict, dst_dict=dst_dict
)
models, model_args = pytorch_translate_utils.load_diverse_ensemble_for_inference(
args.path, dataset.src_dict, dataset.dst_dict
)
append_eos_to_source = model_args[0].append_eos_to_source
reverse_source = model_args[0].reverse_source
assert all(
a.append_eos_to_source == append_eos_to_source
and a.reverse_source == reverse_source
for a in model_args
)
if args.source_binary_file != "":
assert args.target_binary_file != ""
dst_dataset = pytorch_translate_data.InMemoryNumpyDataset.create_from_file(
args.target_binary_file
)
if use_char_source:
src_dataset = char_data.InMemoryNumpyWordCharDataset.create_from_file(
args.source_binary_file
)
gen_split = char_data.LanguagePairSourceCharDataset(
src=src_dataset,
dst=dst_dataset,
pad_idx=src_dict.pad(),
eos_idx=dst_dict.eos(),
)
else:
src_dataset = pytorch_translate_data.InMemoryNumpyDataset.create_from_file(
args.source_binary_file
)
gen_split = data.LanguagePairDataset(
src=src_dataset,
dst=dst_dataset,
pad_idx=src_dict.pad(),
eos_idx=dst_dict.eos(),
)
elif pytorch_translate_data.is_multilingual(args):
gen_split = pytorch_translate_data.make_language_pair_dataset_from_text_multilingual(
source_text_file=args.source_text_file[0],
target_text_file=args.target_text_file,
source_lang_id=args.multiling_source_lang_id,
target_lang_id=args.multiling_target_lang_id,
source_dict=src_dict,
target_dict=dst_dict,
append_eos=append_eos_to_source,
reverse_source=reverse_source,
)
elif args.source_ensembling:
gen_split = multisource_data.make_multisource_language_pair_dataset_from_text(
source_text_files=args.source_text_file,
target_text_file=args.target_text_file,
source_dict=src_dict,
target_dict=dst_dict,
append_eos=append_eos_to_source,
reverse_source=reverse_source,
)
else:
gen_split = pytorch_translate_data.make_language_pair_dataset_from_text(
source_text_file=args.source_text_file[0],
target_text_file=args.target_text_file,
source_dict=src_dict,
target_dict=dst_dict,
append_eos=append_eos_to_source,
reverse_source=reverse_source,
char_source_dict=char_source_dict,
)
dataset.splits[args.gen_subset] = gen_split
if args.source_lang is None or args.target_lang is None:
# record inferred languages in args
args.source_lang, args.target_lang = dataset.src, dataset.dst
print(f"| [{dataset.src}] dictionary: {len(dataset.src_dict)} types")
print(f"| [{dataset.dst}] dictionary: {len(dataset.dst_dict)} types")
print(f"| {args.gen_subset} {len(dataset.splits[args.gen_subset])} examples")
scorer, num_sentences, gen_timer, _ = _generate_score(
models=models, args=args, dataset=dataset, dataset_split=args.gen_subset
)
print(
f"| Translated {num_sentences} sentences ({gen_timer.n} tokens) "
f"in {gen_timer.sum:.1f}s ({1. / gen_timer.avg:.2f} tokens/s)"
)
print(
f"| Generate {args.gen_subset} with beam={args.beam}: "
f"{scorer.result_string()}"
)
return scorer.score()