def testEvalDataLoader(self):
vocab_src = Vocab(vocab_src_file)
vocab_trg = Vocab(vocab_trg_file)
vocab_srcbpe = Vocab(vocab_srcbpe_file, bpe_codes=codes_src_file)
vocab_trgbpe = Vocab(vocab_trgbpe_file, bpe_codes=codes_trg_file)
data = EvalTextIterator(
train_src_file, train_trg_file,
vocab_src, vocab_trg,
batch_size=1)
for (x, len_x), (y, len_y) in data:
print (x[0])
print (' '.join(vocab_src.convert_to_wordlist(x[0])))
print (y[0])
print (' '.join(vocab_trg.convert_to_wordlist(y[0])))
break
def testTestDataLoader(self):
vocab_src = Vocab(vocab_src_file)
vocab_srcbpe = Vocab(vocab_srcbpe_file, bpe_codes_file=codes_src_file)
data = TestTextIterator(train_src_file, vocab_srcbpe, batch_size=1)
for x_str, (x, len_x) in data:
print(x_str[0])
print(x[0])
print(' '.join(vocab_srcbpe.convert_to_wordlist(x[0])))
break