def test_load_data_single_path(self):
test_args = test_utils.ModelParamsDict()
test_args.source_lang = "en"
test_args.target_lang = "fr"
test_args.log_verbose = False
src_dict, tgt_dict = test_utils.create_vocab_dictionaries()
src_text_file, tgt_text_file = test_utils.create_test_text_files()
src_bin_path = preprocess.binarize_text_file(
text_file=src_text_file,
dictionary=src_dict,
output_path=tempfile.NamedTemporaryFile().name,
append_eos=True,
reverse_order=False,
)
tgt_bin_path = preprocess.binarize_text_file(
text_file=tgt_text_file,
dictionary=tgt_dict,
output_path=tempfile.NamedTemporaryFile().name,
append_eos=True,
reverse_order=False,
)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
split = "0"
task.load_dataset(split, src_bin_path, tgt_bin_path)
self.assertEqual(len(task.datasets[split]), 4)
self.assertIsInstance(task.datasets[split], LanguagePairDataset)
def test_combine_weighted_scores(self):
test_args = test_utils.ModelParamsDict()
test_args.enable_rescoring = True
test_args.length_penalty = 1
test_args.original_model_weight = 1
test_args.l2r_model_path = ""
test_args.l2r_model_weight = 1
test_args.r2l_model_weight = 0
test_args.reverse_model_weight = 0.5
test_args.lm_model_weight = 0.75
test_args.length_penalty = 1
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
rescorer = Rescorer(test_args)
scores = torch.tensor([[10, 20, 30, 40]], dtype=torch.float)
src_tokens = torch.tensor([1, 2, 3, 4, 5])
hypos = [{"tokens": torch.tensor([1, 2, 3, 4, 5, 6, 7, 8, 9, 10])}]
rescorer.combine_weighted_scores(scores, src_tokens, hypos)
# 10=1. , 20*0=0. , 30*(0.5/5)=3. , 40*(0.75/5)=6.
expected = torch.tensor([[10.0, 0.0, 3.0, 6.0]], dtype=torch.float)
assert torch.equal(scores, expected)
def test_load_data_multi_path(self):
test_args = test_utils.ModelParamsDict()
test_args.source_lang = "en"
test_args.target_lang = "fr"
test_args.log_verbose = False
src_dict, tgt_dict = test_utils.create_vocab_dictionaries()
num_paths = 4
src_bin_path, tgt_bin_path = {}, {}
for i in range(num_paths):
src_text_file, tgt_text_file = test_utils.create_test_text_files()
src_bin_path[i] = preprocess.binarize_text_file(
text_file=src_text_file,
dictionary=src_dict,
output_path=tempfile.NamedTemporaryFile().name,
append_eos=True,
reverse_order=False,
)
tgt_bin_path[i] = preprocess.binarize_text_file(
text_file=tgt_text_file,
dictionary=tgt_dict,
output_path=tempfile.NamedTemporaryFile().name,
append_eos=True,
reverse_order=False,
)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
split = "1"
task.load_dataset(split, src_bin_path, tgt_bin_path)
self.assertEqual(len(task.datasets[split]), 16)
self.assertIsInstance(task.datasets[split], MultiCorpusSampledDataset)
def test_compute_scores(self):
# TODO(halilakin): Verify behaviour in batch mode
test_args = test_utils.ModelParamsDict()
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
scorer = SimpleModelScorer(test_args, "/tmp/model_path.txt")
tgt_tokens = torch.tensor([[2, 11, 22, 0], [2, 33, 44, 55]])
logprobs = torch.zeros(tgt_tokens.shape[0], tgt_tokens.shape[1],
len(tgt_dict))
logprobs[0, 0, 11] = 0.5
logprobs[0, 1, 22] = 1.5
logprobs[0, 3, :] = 5
logprobs[1, 0, 33] = 0.5
logprobs[1, 1, 44] = 1.5
logprobs[1, 2, 55] = 2.5
hypos_scores = scorer.compute_scores(tgt_tokens, logprobs)
assert hypos_scores[0] == 2.0
assert hypos_scores[1] == 4.5
def test_load_data_multi_path(self):
num_paths = 4
test_args, src_dict, tgt_dict, src_bin_path, tgt_bin_path = self._prepare_data_multi_path(
num_paths)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
split = "1"
task.load_dataset(split, src_bin_path, tgt_bin_path)
self.assertEqual(len(task.datasets[split]), 16)
self.assertIsInstance(task.datasets[split], ConcatDataset)
def test_load_data_noising(self):
num_paths = 4
test_args, src_dict, tgt_dict, src_bin_path, tgt_bin_path = self._prepare_data_multi_path(
num_paths)
test_args.word_dropout_prob_map = str({"en-fr": {0: 0.1}})
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
split = "1"
task.load_dataset(split, src_bin_path, tgt_bin_path)
self.assertEqual(len(task.datasets[split]), 16)
self.assertIsInstance(task.datasets[split].datasets[0].src,
NoisingDataset)
def test_reverse_tgt_tokens(self):
test_args = test_utils.ModelParamsDict()
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
scorer = R2LModelScorer(test_args, "/tmp/model_path.txt")
pad = task.tgt_dict.pad()
tgt_tokens = torch.Tensor([[1, 2, 3], [1, 2, pad], [1, pad, pad]])
expected_tokens = torch.Tensor([[3, 2, 1], [2, 1, pad],
[1, pad, pad]])
reversed_tgt_tokens = scorer.reverse_tgt_tokens(tgt_tokens)
assert torch.equal(reversed_tgt_tokens, expected_tokens)
def test_load_data_single_path_idx_bin(self):
test_args = test_utils.ModelParamsDict()
test_args.source_lang = "en"
test_args.target_lang = "fr"
test_args.log_verbose = False
src_dict, tgt_dict = test_utils.create_vocab_dictionaries()
src_text_file, tgt_text_file = test_utils.create_test_text_files()
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
with tempfile.TemporaryDirectory() as destdir:
preprocess_args = [
"--source-lang",
test_args.source_lang,
"--target-lang",
test_args.target_lang,
"--destdir",
destdir,
]
preproc_parser = preprocess_options.get_preprocessing_parser()
preproc_args = preproc_parser.parse_args(preprocess_args)
preproc_args.dataset_impl = "mmap"
split = "train"
binarize(
preproc_args,
src_text_file,
src_dict,
split,
test_args.source_lang,
offset=0,
end=-1,
)
binarize(
preproc_args,
tgt_text_file,
tgt_dict,
split,
test_args.target_lang,
offset=0,
end=-1,
)
src_path = dataset_dest_prefix(preproc_args, split,
test_args.source_lang)
tgt_path = dataset_dest_prefix(preproc_args, split,
test_args.target_lang)
task.load_dataset(split, src_path, tgt_path, is_npz=False)
self.assertEqual(len(task.datasets[split]), 4)
self.assertIsInstance(task.datasets[split], LanguagePairDataset)
def test_combine_weighted_scores(self):
test_args = test_utils.ModelParamsDict()
test_args.enable_rescoring = True
test_args.length_penalty = 1
test_args.l2r_model_path = ""
test_args.l2r_model_weight = 1.0
test_args.r2l_model_weight = 0.0
test_args.reverse_model_weight = 0.0
test_args.cloze_transformer_weight = 0.0
test_args.lm_model_weight = 1.01
test_args.length_penalty = 1.0
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
scores = torch.tensor([[80, 0, 0, 0, 0], [0, 0, 0, 80, 0]],
dtype=torch.float)
src_tokens = torch.tensor([1, 2, 3, 4, 5])
hypos = [{
"tokens": torch.tensor([1, 2])
}, {
"tokens": torch.tensor([1, 2])
}]
src_len = len(src_tokens)
tgt_len = torch.tensor([len(hypo["tokens"]) for hypo in hypos],
dtype=torch.float)
weights = [
test_args.l2r_model_weight,
test_args.r2l_model_weight,
test_args.reverse_model_weight,
test_args.lm_model_weight,
test_args.cloze_transformer_weight,
]
combined_scores = combine_weighted_scores(scores, weights, src_len,
tgt_len, 1)
# 80/(2^1), 0, 0, 80*1.01/(2^1)
expected = torch.tensor([40.0, 40.4], dtype=torch.float)
assert torch.equal(combined_scores, expected)
def test_reverse_scorer_prepare_inputs(self):
test_args = test_utils.ModelParamsDict()
test_args.append_eos_to_source = True
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
pad = task.tgt_dict.pad()
eos = task.tgt_dict.eos()
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
scorer = ReverseModelScorer(test_args, "/tmp/model_path.txt", None,
task)
src_tokens = torch.tensor([6, 7, 8], dtype=torch.int)
hypos = [
{
"tokens": torch.tensor([12, 13, 14, eos], dtype=torch.int)
},
{
"tokens": torch.tensor([22, 23, eos], dtype=torch.int)
},
]
(encoder_inputs,
tgt_tokens) = scorer.prepare_inputs(src_tokens, hypos)
# Test encoder inputs
assert torch.equal(
encoder_inputs[0],
torch.tensor([[12, 13, 14, eos], [22, 23, eos, pad]],
dtype=torch.int),
), "Encoder inputs are not as expected"
max_tgt_len = max(len(hypo["tokens"]) for hypo in hypos)
assert encoder_inputs[1][
0] == max_tgt_len, " Src length is not as expected"
# Test target tokens
assert torch.equal(
tgt_tokens,
torch.tensor([[eos, 6, 7, 8, eos], [eos, 6, 7, 8, eos]],
dtype=torch.int),
), "Target tokens are not as expected"
def test_load_data_noising(self):
test_args = test_utils.ModelParamsDict()
test_args.source_lang = "en"
test_args.target_lang = "fr"
test_args.log_verbose = False
src_dict, tgt_dict = test_utils.create_vocab_dictionaries()
num_paths = 4
src_bin_path, tgt_bin_path = {}, {}
for i in range(num_paths):
src_text_file, tgt_text_file = test_utils.create_test_text_files()
src_bin_path[i] = preprocess.binarize_text_file(
text_file=src_text_file,
dictionary=src_dict,
output_path=tempfile.NamedTemporaryFile().name,
append_eos=True,
reverse_order=False,
)
tgt_bin_path[i] = preprocess.binarize_text_file(
text_file=tgt_text_file,
dictionary=tgt_dict,
output_path=tempfile.NamedTemporaryFile().name,
append_eos=True,
reverse_order=False,
)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
split = "1"
task.load_dataset(
split,
src_bin_path,
tgt_bin_path,
noiser={
0:
UnsupervisedMTNoising(
dictionary=src_dict,
max_word_shuffle_distance=3,
word_dropout_prob=0.2,
word_blanking_prob=0.2,
)
},
)
self.assertEqual(len(task.datasets[split]), 16)
self.assertIsInstance(task.datasets[split].datasets[0].src,
NoisingDataset)
def test_convert_hypos_to_tgt_tokens(self):
test_args = test_utils.ModelParamsDict()
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
scorer = SimpleModelScorer(test_args, None)
hypos = [
{
"tokens": torch.Tensor([1, 2, 3, 4, 5])
},
{
"tokens": torch.Tensor([1, 2, 3, 4])
},
{
"tokens": torch.Tensor([1, 2, 3])
},
{
"tokens": torch.Tensor([1, 2])
},
{
"tokens": torch.Tensor([1])
},
]
tgt_tokens = scorer.convert_hypos_to_tgt_tokens(hypos)
pad = task.tgt_dict.pad()
eos = task.tgt_dict.eos()
expected_tgt_tokens = torch.Tensor([
[eos, 1, 2, 3, 4, 5],
[eos, 1, 2, 3, 4, pad],
[eos, 1, 2, 3, pad, pad],
[eos, 1, 2, pad, pad, pad],
[eos, 1, pad, pad, pad, pad],
]).type_as(tgt_tokens)
assert torch.equal(tgt_tokens, expected_tgt_tokens)
def test_model_passing_as_parameter(self):
test_args = test_utils.ModelParamsDict("transformer")
test_args.enable_rescoring = True
test_args.length_penalty = 1
test_args.l2r_model_weight = 1.0
test_args.r2l_model_weight = 0.0
test_args.reverse_model_weight = 0.0
test_args.lm_model_weight = 1.01
test_args.cloze_transformer_weight = 1.0
test_args.length_penalty = 1.0
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
src_tokens = torch.tensor([[1, 2, 3, 4, 5]])
hypos = [{"tokens": torch.tensor([1, 2])}, {"tokens": torch.tensor([1, 2])}]
rescorer = Rescorer(
test_args, task, {"l2r_model": {"model": model, "task": task}}
)
scores = rescorer.score(src_tokens, hypos)
assert scores.size()[1] == 5
def test_batch_computation(self):
test_args = test_utils.ModelParamsDict("transformer")
test_args.enable_rescoring = True
test_args.length_penalty = 1
test_args.l2r_model_path = "/tmp/test_rescorer_model.pt"
test_args.l2r_model_weight = 1.0
test_args.r2l_model_weight = 0.0
test_args.reverse_model_weight = 0.0
test_args.cloze_transformer_weight = 1.0
test_args.lm_model_weight = 0.0
test_args.length_penalty = 1.0
_, src_dict, tgt_dict = test_utils.prepare_inputs(test_args)
task = tasks.PytorchTranslateTask(test_args, src_dict, tgt_dict)
model = task.build_model(test_args)
torch.save(model, test_args.l2r_model_path)
with patch(
"pytorch_translate.utils.load_diverse_ensemble_for_inference",
return_value=([model], test_args, task),
):
rescorer = Rescorer(test_args)
src_tokens = torch.tensor([[1, 3, 3, 4, 2], [1, 3, 2, 0, 0]])
hypos = [
{"tokens": torch.tensor([1, 5, 2])},
{"tokens": torch.tensor([6, 3, 5, 2])},
{"tokens": torch.tensor([1, 2])},
{"tokens": torch.tensor([1, 5, 6, 2])},
]
scores = rescorer.score(src_tokens, hypos)
src_tokens = torch.tensor([[1, 3, 3, 4, 2]])
hypos = [
{"tokens": torch.tensor([1, 5, 2])},
{"tokens": torch.tensor([6, 3, 5, 2])},
]
scores_single = rescorer.score(src_tokens, hypos)
assert torch.equal(scores[0], scores_single[0])
def setUp(self):
self.args = test_utils.ModelParamsDict()
_, src_dict, tgt_dict = test_utils.prepare_inputs(self.args)
self.task = tasks.PytorchTranslateTask(self.args, src_dict, tgt_dict)
self.model = self.task.build_model(self.args)
