self._idx += self.batch_size return x, y def _reorder(self): if self.shuffle: self.dataset = shuffle(self.dataset, random_state=self.random_state) self._idx = 0 (x_train, y_train), \ (x_test, y_test), \ (num_x, num_y), \ (w2i_x, w2i_y), (i2w_x, i2w_y) = \ load_small_parallel_enja(to_ja=True, add_bos=False) train_dataloader = ParallelDataLoader((x_train, y_train), shuffle=True) valid_dataloader = ParallelDataLoader((x_test, y_test)) test_dataloader = ParallelDataLoader((x_test, y_test), batch_size=1, shuffle=True) ''' Build model ''' input_dim = num_x hidden_dim = 128 output_dim = num_y model = EncoderDecoder(input_dim, hidden_dim, output_dim, device=device).to(device)
self._idx += self.batch_size return x, y def _reorder(self): if self.shuffle: self.dataset = shuffle(self.dataset, random_state=self.random_state) self._idx = 0 (x_train, y_train), \ (x_test, y_test), \ (num_x, num_y), \ (w2i_x, w2i_y), (i2w_x, i2w_y) = \ load_small_parallel_enja(to_ja=True) train_dataloader = ParallelDataLoader((x_train, y_train), shuffle=True) valid_dataloader = ParallelDataLoader((x_test, y_test)) test_dataloader = ParallelDataLoader((x_test, y_test), batch_size=1, shuffle=True) ''' Build model ''' model = Transformer(num_x, num_y, N=3, h=4,