self._idx += self.batch_size
return x, y
def _reorder(self):
if self.shuffle:
self.dataset = shuffle(self.dataset,
random_state=self.random_state)
self._idx = 0
(x_train, y_train), \
(x_test, y_test), \
(num_x, num_y), \
(w2i_x, w2i_y), (i2w_x, i2w_y) = \
load_small_parallel_enja(to_ja=True, add_bos=False)
train_dataloader = ParallelDataLoader((x_train, y_train), shuffle=True)
valid_dataloader = ParallelDataLoader((x_test, y_test))
test_dataloader = ParallelDataLoader((x_test, y_test),
batch_size=1,
shuffle=True)
'''
Build model
'''
input_dim = num_x
hidden_dim = 128
output_dim = num_y
model = EncoderDecoder(input_dim, hidden_dim, output_dim,
device=device).to(device)
self._idx += self.batch_size
return x, y
def _reorder(self):
if self.shuffle:
self.dataset = shuffle(self.dataset,
random_state=self.random_state)
self._idx = 0
(x_train, y_train), \
(x_test, y_test), \
(num_x, num_y), \
(w2i_x, w2i_y), (i2w_x, i2w_y) = \
load_small_parallel_enja(to_ja=True)
train_dataloader = ParallelDataLoader((x_train, y_train),
shuffle=True)
valid_dataloader = ParallelDataLoader((x_test, y_test))
test_dataloader = ParallelDataLoader((x_test, y_test),
batch_size=1,
shuffle=True)
'''
Build model
'''
model = Transformer(num_x,
num_y,
N=3,
h=4,