def testEvalDataLoader(self): vocab_src = Vocab(vocab_src_file) vocab_trg = Vocab(vocab_trg_file) vocab_srcbpe = Vocab(vocab_srcbpe_file, bpe_codes=codes_src_file) vocab_trgbpe = Vocab(vocab_trgbpe_file, bpe_codes=codes_trg_file) data = EvalTextIterator( train_src_file, train_trg_file, vocab_src, vocab_trg, batch_size=1) for (x, len_x), (y, len_y) in data: print (x[0]) print (' '.join(vocab_src.convert_to_wordlist(x[0]))) print (y[0]) print (' '.join(vocab_trg.convert_to_wordlist(y[0]))) break
def testTestDataLoader(self): vocab_src = Vocab(vocab_src_file) vocab_srcbpe = Vocab(vocab_srcbpe_file, bpe_codes_file=codes_src_file) data = TestTextIterator(train_src_file, vocab_srcbpe, batch_size=1) for x_str, (x, len_x) in data: print(x_str[0]) print(x[0]) print(' '.join(vocab_srcbpe.convert_to_wordlist(x[0]))) break