def test_build_bilingual_vocab(self):
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
vocab_size = bpe_model.build_vocab(
src_txt_path=f1, dst_txt_path=f2, vocab_size=12, num_ibm_iters=3, num_cpus=3
)
assert vocab_size == len(bpe_model.vocab) == 12
shutil.rmtree(tmp_dir)
def test_calc_word_probs(self):
with patch("builtins.open") as mock_open:
bpe_model = bilingual_bpe.BilingualBPE()
mock_open.return_value.__enter__ = mock_open
mock_open.return_value.__iter__ = Mock(return_value=iter(txt_content))
v = bpe_model._calc_word_probs(txt_path="no_exist_file.txt")
assert len(v) == 9
assert v["123"] == 2 / 11
assert v["123456789"] == 1 / 11
def test_best_candidate_bilingual(self):
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
bpe_model._init_params(src_txt_path=f1,
dst_txt_path=f2,
num_ibm_iters=3,
num_cpus=3)
b1 = bpe_model.get_best_candidate()
c1 = bpe_model.get_best_candidate()
# For the best step, it is the same as monolingual.
assert b1 == c1
shutil.rmtree(tmp_dir)
def test_save_load(self):
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
dst2src_ibm_model = char_ibm_model1.Word2CharIBMModel1()
dst2src_ibm_model.learn_ibm_parameters(src_path=f2, dst_path=f1, num_iters=3)
ibm_path = path.join(tmp_dir, "ibm")
dst2src_ibm_model.save(file_path=ibm_path)
vocab_size = bpe_model.build_vocab(
ibm_model_path=ibm_path, src_txt_path=f1, dst_txt_path=f2, vocab_size=12
)
assert vocab_size == len(bpe_model.vocab) == 12
bpe_model.save(file_path=tmp_dir + "/vocab.txt")
loaded_model = bilingual_bpe.BilingualBPE()
loaded_model.load(file_path=tmp_dir + "/vocab.txt")
assert loaded_model.vocab == bpe_model.vocab
assert bpe_model.segment_word("1234") == loaded_model.segment_word("1234")
shutil.rmtree(tmp_dir)
def test_build_bilingual_vocab(self):
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
dst2src_ibm_model = char_ibm_model1.Word2CharIBMModel1()
dst2src_ibm_model.learn_ibm_parameters(src_path=f2, dst_path=f1, num_iters=3)
ibm_path = path.join(tmp_dir, "ibm")
dst2src_ibm_model.save(file_path=ibm_path)
vocab_size = bpe_model.build_vocab(
ibm_model_path=ibm_path, src_txt_path=f1, dst_txt_path=f2, vocab_size=12
)
assert vocab_size == len(bpe_model.vocab) == 12
shutil.rmtree(tmp_dir)
def test_bilingual_bpe_init(self):
"""
This looks more like an integration test because each subpeace is tested
in different places.
"""
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
bpe_model._init_params(
src_txt_path=f1, dst_txt_path=f2, num_ibm_iters=3, num_cpus=3
)
assert len(bpe_model.bpe_probs_from_alignment) == 80
assert bpe_model.eow_symbol in bpe_model.bpe_probs_from_alignment
shutil.rmtree(tmp_dir)
def test_best_candidate_bilingual(self):
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
dst2src_ibm_model = char_ibm_model1.Word2CharIBMModel1()
dst2src_ibm_model.learn_ibm_parameters(src_path=f2, dst_path=f1, num_iters=3)
ibm_path = path.join(tmp_dir, "ibm")
dst2src_ibm_model.save(file_path=ibm_path)
bpe_model._init_params(
ibm_model_path=ibm_path, src_txt_path=f1, dst_txt_path=f2
)
b1 = bpe_model.get_best_candidate()
c1 = bpe_model.get_best_candidate()
# For the best step, it is the same as monolingual.
assert b1 == c1
shutil.rmtree(tmp_dir)
def test_bilingual_bpe_init(self):
"""
This looks more like an integration test because each subpeace is tested
in different places.
"""
bpe_model = bilingual_bpe.BilingualBPE()
tmp_dir, f1, f2 = morph_utils.get_two_different_tmp_files()
dst2src_ibm_model = char_ibm_model1.Word2CharIBMModel1()
dst2src_ibm_model.learn_ibm_parameters(src_path=f2, dst_path=f1, num_iters=3)
ibm_path = path.join(tmp_dir, "ibm")
dst2src_ibm_model.save(file_path=ibm_path)
bpe_model._init_params(
ibm_model_path=ibm_path, src_txt_path=f1, dst_txt_path=f2
)
assert len(bpe_model.bpe_probs_from_alignment) == 80
assert bpe_model.eow_symbol in bpe_model.bpe_probs_from_alignment
shutil.rmtree(tmp_dir)