def __init__(self,
src,
trg,
pad_index=0,
word_drop=0.0,
unk_indx=0,
use_cuda=False):
src, src_lengths = src
self.src = src
self.src_lengths = src_lengths
self.src_mask = (src != pad_index).unsqueeze(-2)
self.nseqs = src.size(0)
self.trg = None
self.trg_y = None
self.trg_mask = None
self.trg_lengths = None
self.ntokens = None
if trg is not None:
trg, trg_lengths = trg
self.trg = trg[:, :-1]
#word drop out approach proposed in bowman et. al 2016
mask = trg.new_zeros(self.trg.size(0),
self.trg.size(1)).float().fill_(word_drop)
mask = Bernoulli(mask).sample().byte()
try:
mask = mask.bool()
except AttributeError as e:
#just means your using an older pytorch version...
_ = 0
self.trg.masked_fill_(mask, unk_indx)
self.trg_lengths = trg_lengths
self.trg_y = trg[:, 1:]
self.trg_mask = (self.trg_y != pad_index)
self.ntokens = (self.trg_y != pad_index).data.sum().item()
if use_cuda:
self.src = self.src.cuda()
self.src_mask = self.src_mask.cuda()
if trg is not None:
self.trg = self.trg.cuda()
self.trg_y = self.trg_y.cuda()
self.trg_mask = self.trg_mask.cuda()
else:
self.src = self.src.cpu()
self.src_mask = self.src_mask.cpu()
self.src_lengths = self.src_lengths.cpu()
if trg is not None:
self.trg = self.trg.cpu()
self.trg_y = self.trg_y.cpu()
self.trg_mask = self.trg_mask.cpu()
self.trg_lengths = self.trg_lengths.cpu()
def getWordEmbeddingsWithWordDropout(self, embeddings, indexes, pad_mask):
#word drop out approach proposed in bowman et. al 2016
if len(pad_mask.size()) > 2:
pad_mask = pad_mask.squeeze()
indexes = indexes.clone(
) #clone tensor just in case...don't want any mutation
mask = torch.zeros_like(indexes).float().fill_(self.word_drop)
mask = Bernoulli(mask).sample().byte()
mask = mask * pad_mask.byte(
) # don't word drop things passed sentence length
mask[0, :] = 0 #do not mask out sos token
try:
mask = mask.bool()
except:
#do nothing
i = 0
indexes.masked_fill_(mask, self.unk_tok_indx)
return embeddings(indexes)
