def generator(self, data_dir, tmp_dir, train):
"""
Generate the vocab and then build train and validation t2t-datagen files.
Four .txt files have to be present in the data_dir directory:
trainSource.txt
trainTarget.txt
devSource.txt
devTarget.txt
Params:
:train: Whether we are in train mode or not.
"""
character_vocab = text_encoder.ByteTextEncoder()
mode = "train" if train else "dev"
print("t2t_csaky_log: " + mode + " data generation activated.")
sourcePath = os.path.join(data_dir, mode + "Source.txt")
targetPath = os.path.join(data_dir, mode + "Target.txt")
# Try to find the txt files.
if os.path.isfile(sourcePath) and os.path.isfile(targetPath):
print("t2t_csaky_log: Generating " + mode + " files in " +
data_dir)
return translate.character_generator(sourcePath, targetPath,
character_vocab, EOS)
else:
print("t2t_csaky_log: " + mode +
" source or target file not found, please check " +
"that the following files exist in your " + data_dir +
" directory and rerun this program:")
print(" trainSource.txt")
print(" trainTarget.txt")
print(" devSource.txt")
print(" devTarget.txt")
def generator(self, data_dir, tmp_dir, train):
'''
Generate the vocab and then build train and validation t2t-datagen files.
Four .txt files have to be present in the data_dir directory:
trainSource.txt
trainTarget.txt
devSource.txt
devTarget.txt
Params:
:train: Whether we are in train mode or not.
'''
character_vocab = text_encoder.ByteTextEncoder()
mode = 'train' if train else 'dev'
print('t2t_csaky_log: ' + mode + ' data generation activated.')
sourcePath = os.path.join(data_dir, mode + 'Source.txt')
targetPath = os.path.join(data_dir, mode + 'Target.txt')
# Try to find the txt files.
if os.path.isfile(sourcePath) and os.path.isfile(targetPath):
print('t2t_csaky_log: Generating ' + mode + ' files in ' + data_dir)
return translate.character_generator(sourcePath,
targetPath,
character_vocab,
EOS)
else:
print('t2t_csaky_log: ' + mode +
' source or target file not found, please check ' +
'that the following files exist in your ' + data_dir +
' directory and rerun this program:')
print(' trainSource.txt')
print(' trainTarget.txt')
print(' devSource.txt')
print(' devTarget.txt')
def generator(self, _, tmp_dir, train):
character_vocab = text_encoder.ByteTextEncoder()
tag = "train" if train else "dev"
sPath = tag + "Source.txt"
tPath = tag + "Target.txt"
return translate.character_generator(sPath, tPath, character_vocab,
EOS)
def generator(self, _, tmp_dir, train):
character_vocab = text_encoder.ByteTextEncoder()
datasets = _ENDE_TRAIN_DATASETS if train else _ENDE_TEST_DATASETS
tag = "train" if train else "dev"
data_path = translate.compile_data(tmp_dir, datasets,
"wmt_ende_chr_%s" % tag)
return translate.character_generator(
data_path + ".lang1", data_path + ".lang2", character_vocab, EOS)
def generator(self, _, tmp_dir, train):
character_vocab = text_encoder.ByteTextEncoder()
datasets = _ENDE_TRAIN_DATASETS if train else _ENDE_TEST_DATASETS
tag = "train" if train else "dev"
data_path = translate.compile_data(tmp_dir, datasets,
"wmt_ende_chr_%s" % tag)
return translate.character_generator(data_path + ".lang1",
data_path + ".lang2",
character_vocab, EOS)
def generator(self, data_dir, tmp_dir, train):
character_vocab = text_encoder.ByteTextEncoder()
if self.use_small_dataset:
datasets = _ENFR_TRAIN_SMALL_DATA if train else _ENFR_TEST_SMALL_DATA
else:
datasets = _ENFR_TRAIN_LARGE_DATA if train else _ENFR_TEST_LARGE_DATA
tag = "train" if train else "dev"
data_path = translate.compile_data(tmp_dir, datasets,
"wmt_enfr_chr_%s" % tag)
return translate.character_generator(
data_path + ".lang1", data_path + ".lang2", character_vocab, EOS)
def testCharacterGenerator(self):
# Generate a trivial source and target file.
tmp_dir = self.get_temp_dir()
(_, tmp_file_path) = tempfile.mkstemp(dir=tmp_dir)
if six.PY2:
enc_f = lambda s: s
else:
enc_f = lambda s: s.encode("utf-8")
with io.open(tmp_file_path + ".src", "wb") as src_file:
src_file.write(enc_f("source1\n"))
src_file.write(enc_f("source2\n"))
with io.open(tmp_file_path + ".tgt", "wb") as tgt_file:
tgt_file.write(enc_f("target1\n"))
tgt_file.write(enc_f("target2\n"))
# Call character generator on the generated files.
results_src, results_tgt = [], []
character_vocab = text_encoder.ByteTextEncoder()
for dictionary in translate.character_generator(
tmp_file_path + ".src", tmp_file_path + ".tgt", character_vocab):
self.assertEqual(sorted(list(dictionary)), ["inputs", "targets"])
results_src.append(dictionary["inputs"])
results_tgt.append(dictionary["targets"])
# Check that the results match the files.
# First check that the results match the encoded original strings;
# this is a comparison of integer arrays.
self.assertEqual(len(results_src), 2)
self.assertEqual(results_src[0],
character_vocab.encode("source1"))
self.assertEqual(results_src[1],
character_vocab.encode("source2"))
self.assertEqual(results_tgt[0],
character_vocab.encode("target1"))
self.assertEqual(results_tgt[1],
character_vocab.encode("target2"))
# Then decode the results and compare with the original strings;
# this is a comparison of strings
self.assertEqual(character_vocab.decode(results_src[0]),
"source1")
self.assertEqual(character_vocab.decode(results_src[1]),
"source2")
self.assertEqual(character_vocab.decode(results_tgt[0]),
"target1")
self.assertEqual(character_vocab.decode(results_tgt[1]),
"target2")
# Clean up.
os.remove(tmp_file_path + ".src")
os.remove(tmp_file_path + ".tgt")
os.remove(tmp_file_path)