1. データの準備
'''
data_dir = os.path.join(os.path.dirname(__file__), 'data')
en_train_path = os.path.join(data_dir, 'train.en')
en_val_path = os.path.join(data_dir, 'dev.en')
en_test_path = os.path.join(data_dir, 'test.en')
ja_train_path = os.path.join(data_dir, 'train.ja')
ja_val_path = os.path.join(data_dir, 'dev.ja')
ja_test_path = os.path.join(data_dir, 'test.ja')
en_vocab = Vocab()
ja_vocab = Vocab()
en_vocab.fit(en_train_path)
ja_vocab.fit(ja_train_path)
x_train = en_vocab.transform(en_train_path)
x_val = en_vocab.transform(en_val_path)
x_test = en_vocab.transform(en_test_path)
t_train = ja_vocab.transform(ja_train_path, eos=True)
t_val = ja_vocab.transform(ja_val_path, eos=True)
t_test = ja_vocab.transform(ja_test_path, eos=True)
def sort(x, t):
lens = [len(i) for i in x]
indices = sorted(range(len(lens)), key=lambda i: -lens[i])
x = [x[i] for i in indices]
t = [t[i] for i in indices]
