def to_input_tensor(self, sents: List[List[str]], device: torch.device) -> torch.Tensor:
word_ids = self.words2indices(sents)
sents_t = input_transpose(word_ids, self['<pad>'])
sents_var = torch.tensor(sents_t, dtype=torch.long, device=device)
return sents_var
def to_input_tensor(self, sents: List[List[str]],
device: torch.device) -> torch.Tensor:
word_ids = self.words2indices(sents)
sents_t = input_transpose(word_ids, self['<pad>'])
sents_var = torch.tensor(sents_t, dtype=torch.long, device=device)
return sents_var
def to_input_tensor_src(self, sents_src, device):
word_src_ids = self.words2indices(sents_src)
sents_t_src = input_transpose(word_src_ids, self['<pad>'])
sents_var_src = torch.tensor(sents_t_src,
dtype=torch.long,
device=device)
return sents_var_src
def to_input_tensor_tgt(self, sents_src, sents_tgt, device):
(word_tgt_ids, word_src_complete_ids, word_tgt_complete_ids, word_oovs,
max_oov_num) = self.words2indices_complete(sents_src, sents_tgt)
sents_t_src_complete = input_transpose(word_src_complete_ids,
self['<pad>'])
sents_t_tgt_complete = input_transpose(word_tgt_complete_ids,
self['<pad>'])
sents_var_complete_src = torch.tensor(sents_t_src_complete,
dtype=torch.long,
device=device)
sents_var_complete_tgt = torch.tensor(sents_t_tgt_complete,
dtype=torch.long,
device=device)
sents_t_tgt = input_transpose(word_tgt_ids, self['<pad>'])
sents_var_tgt = torch.tensor(sents_t_tgt,
dtype=torch.long,
device=device)
return sents_var_tgt, sents_var_complete_src, sents_var_complete_tgt, word_oovs, max_oov_num
def encode(self, sents: List[List[str]]) -> Tensor:
sents_t = input_transpose(sents, pad_token)
# sent_t = [sent_len, batch_size]
sent_len = len(sents_t)
batch_size = len(sents_t[0])
indices_list = [[self.vocab.to_idx(w) for w in s] for s in sents_t]
indices = torch.LongTensor(indices_list).to(device)
loss = 0
hidden_init = torch.FloatTensor(1,
batch_size, self.hidden_size).uniform_(
-0.1, 0.1).to(device)
cell_init = torch.FloatTensor(1,
batch_size, self.hidden_size).uniform_(
-0.1, 0.1).to(device)
hidden = (hidden_init, cell_init)
for i in range(sent_len - 1):
word_batch = indices[i, :].unsqueeze(0)
output, hidden = self.encoder(word_batch, hidden)
loss += self.criterion(output.squeeze(0), indices[i + 1])
# print(loss.item())
return loss