def do_build_decoder(cls, args, captions_dict):
decoder_embedding = transformer.Embedding(
num_embeddings=len(captions_dict),
embedding_dim=args.decoder_embed_dim,
padding_idx=captions_dict.pad())
return transformer.TransformerDecoder(args, captions_dict,
decoder_embedding)
def PretrainedEmbedding(num_embeddings, embedding_dim, padding_idx, dictionary,
pretrain_path):
print(f'| Load embedding at {pretrain_path}')
emb = transformer.Embedding(num_embeddings, embedding_dim, padding_idx)
embed_dict = utils.parse_embedding(pretrain_path)
mask_factor = []
for idx in range(len(dictionary)):
token = dictionary[idx]
if token in embed_dict:
emb.weight.data[idx] = embed_dict[token]
mask_factor.append(0)
else:
mask_factor.append(1)
mask_factor = torch.tensor(mask_factor,
dtype=torch.float32).unsqueeze_(-1).expand(
len(mask_factor), embedding_dim)
def hook(grad):
grad *= mask_factor.type_as(grad)
return grad
emb.weight.register_hook(hook)
return emb
def create_new_embed(self,
num_embeds,
embedding_dim,
pad_idx,
pretrained_emb=None):
if self.same_dist:
_mean = pretrained_emb.weight.mean()
_std = pretrained_emb.weight.std()
new_embed = nn.Embedding(num_embeds,
embedding_dim,
padding_idx=pad_idx)
nn.init.normal_(new_embed.weight, mean=_std, std=_std)
nn.init.constant_(new_embed.weight[pad_idx], 0)
else:
new_embed = transformer.Embedding(num_embeds, embedding_dim,
pad_idx)
return new_embed
def build_embed(self, num_embeddings, embedding_dim, padding_idx,
dictionary, pretrain_path, tune_epoch):
print(f'| Load embedding at {pretrain_path}')
emb = transformer.Embedding(num_embeddings, embedding_dim, padding_idx)
embed_dict = utils.parse_embedding(pretrain_path)
self.mask_factor = []
for idx in range(len(dictionary)):
token = dictionary[idx]
if token in embed_dict:
emb.weight.data[idx] = embed_dict[token]
self.mask_factor.append(0)
else:
self.mask_factor.append(1)
self.mask_factor = torch.tensor(
self.mask_factor,
dtype=torch.float32).unsqueeze_(-1).expand(len(self.mask_factor),
embedding_dim)
emb.weight.register_hook(self.hook)
return emb
def FreePretrainedEmbedding(num_embeddings, embedding_dim, padding_idx,
dictionary, pretrain_path):
print(f'| Load FreePretained embedding at {pretrain_path}')
emb = transformer.Embedding(num_embeddings, embedding_dim, padding_idx)
embed_dict = utils.parse_embedding(pretrain_path)
# mask_factor = []
for idx in range(len(dictionary)):
token = dictionary[idx]
if token in embed_dict:
emb.weight.data[idx] = embed_dict[token]
# mask_factor.append(0)
# else:
# mask_factor.append(1)
special_toks = ['-LRB-', '-RRB-']
bind_toks = ['(', ')']
for tok, btok in zip(special_toks, bind_toks):
idx = dictionary.index(tok)
if btok in embed_dict and idx != dictionary.unk_index:
print(f'Replace embed for {tok} {btok} - {idx}')
emb.weight.data[idx] = embed_dict[btok]
return emb
def __build_embedding(dictionary, embed_dim):
num_embeddings = len(dictionary)
padding_idx = dictionary.pad()
emb = transformer.Embedding(num_embeddings, embed_dim, padding_idx)
return emb