class classify_model(nn.Module):
def __init__(self, size_question, path_init):
super(classify_model, self).__init__()
self.w_emb = WordEmbedding(size_question, 300, 0.0, False)
self.w_emb.init_embedding(path_init)
self.q_emb = QuestionEmbedding(300, 1024, 1, False, 0.0, 'GRU')
self.q_final = QuestionAttention(1024)
self.f_fc1 = linear(1024, 256)
self.f_fc2 = linear(256, 64)
self.f_fc3 = linear(64, 2)
def forward(self, question):
w_emb = self.w_emb(question)
q_emb = self.q_emb.forward_all(w_emb) # [batch, q_len, q_dim]
q_final = self.q_final(w_emb, q_emb) #b, 1024
x_f = self.f_fc1(q_final)
x_f = F.relu(x_f)
x_f = self.f_fc2(x_f)
x_f = F.dropout(x_f)
x_f = F.relu(x_f)
x_f = self.f_fc3(x_f)
return x_f
class Model(nn.Module):
def __init__(self, opt):
super(Model, self).__init__()
num_hid = opt.num_hid
activation = opt.activation
dropG = opt.dropG
dropW = opt.dropW
dropout = opt.dropout
dropL = opt.dropL
norm = opt.norm
dropC = opt.dropC
self.opt = opt
self.w_emb = WordEmbedding(opt.ntokens, emb_dim=300, dropout=dropW)
self.w_emb.init_embedding('data/glove6b_init_300d.npy')
self.q_emb = QuestionEmbedding(in_dim=300, num_hid=num_hid, nlayers=1,
bidirect=False, dropout=dropG, rnn_type='GRU')
self.q_net = FCNet([self.q_emb.num_hid, num_hid], dropout=dropL, norm=norm, act=activation)
self.gv_net = FCNet([2048, num_hid], dropout=dropL, norm=norm, act=activation)
self.gv_att_1 = Att_3(v_dim=2048, q_dim=self.q_emb.num_hid, num_hid=num_hid, dropout=dropout, norm=norm,
act=activation)
self.gv_att_2 = Att_3(v_dim=2048, q_dim=self.q_emb.num_hid, num_hid=num_hid, dropout=dropout, norm=norm,
act=activation)
self.classifier = SimpleClassifier(in_dim=num_hid, hid_dim=2 * num_hid, out_dim=3129,
dropout=dropC, norm=norm, act=activation)
def forward(self, q, gv):
"""Forward
q: [batch_size, seq_length]
c: [batch, 5, 20]
return: logits, not probs
"""
w_emb = self.w_emb(q)
q_emb = self.q_emb(w_emb) # run GRU on word embeddings [batch, q_dim]
att_1 = self.gv_att_1(gv, q_emb) # [batch, 1, v_dim]
att_2 = self.gv_att_2(gv, q_emb) # [batch, 1, v_dim]
att_gv = att_1 + att_2
gv_embs = (att_gv * gv) # [batch, v_dim]
gv_emb = gv_embs.sum(1)
q_repr = self.q_net(q_emb)
gv_repr = self.gv_net(gv_emb)
joint_repr = q_repr * gv_repr
logits = self.classifier(joint_repr)
ansidx = torch.argsort(logits, dim=1, descending=True)
return logits, att_gv, ansidx
utils.assert_eq(len(tokens), max_length)
ans_tokens.append(tokens)
return ans_tokens
def create_answer_embedding(ans_list, dictionary, w_emb, ans_emb):
ans_tokens = tokenize(ans_list, dictionary)
ans_tokens = torch.from_numpy(np.array(ans_tokens))
answer_embedding = torch.zeros(3129, 1024)
for idx, ans in enumerate(ans_tokens):
ans = ans.unsqueeze(0)
w = w_emb(ans)
ans = ans_emb(w)
answer_embedding[idx] = ans.squeeze()
with open('data/answer_embedding.pkl', 'wb') as f:
cPickle.dump(answer_embedding, f)
if __name__ == '__main__':
dictionary = Dictionary.load_from_file('data/dictionary.pkl')
w_emb = WordEmbedding(dictionary.ntoken, 300, .0, 'c')
w_emb.init_embedding('data/glove6b_init_300d.npy', None, None)
ans_emb = QuestionEmbedding(600, 1024, 1, False, .0)
ans2label_path = ans2label_path = os.path.join('data', 'cache',
'trainval_ans2label.pkl')
ans2label = cPickle.load(open(ans2label_path, 'rb'))
ans_list = [ans for ans in ans2label]
create_answer_embedding(ans_list, dictionary, w_emb, ans_emb)
class Model(nn.Module):
def __init__(self, opt):
super(Model, self).__init__()
num_hid = opt.num_hid
activation = opt.activation
dropG = opt.dropG
dropW = opt.dropW
dropout = opt.dropout
dropL = opt.dropL
norm = opt.norm
dropC = opt.dropC
self.opt = opt
self.w_emb = WordEmbedding(opt.ntokens, emb_dim=300, dropout=dropW)
self.w_emb.init_embedding(opt.dataroot + 'glove6b_init_300d.npy')
self.q_emb = QuestionEmbedding(in_dim=300, num_hid=num_hid, nlayers=1,
bidirect=False, dropout=dropG, rnn_type='GRU')
self.q_net = FCNet([self.q_emb.num_hid, num_hid], dropout=dropL, norm=norm, act=activation)
self.gv_net = FCNet([opt.v_dim, num_hid], dropout=dropL, norm=norm, act=activation)
self.gv_att_1 = Att_3(v_dim=opt.v_dim, q_dim=self.q_emb.num_hid, num_hid=num_hid, dropout=dropout, norm=norm,
act=activation)
self.gv_att_2 = Att_3(v_dim=opt.v_dim, q_dim=self.q_emb.num_hid, num_hid=num_hid, dropout=dropout, norm=norm,
act=activation)
self.classifier = SimpleClassifier(in_dim=num_hid, hid_dim=2 * num_hid, out_dim=opt.ans_dim,
dropout=dropC, norm=norm, act=activation)
self.normal = nn.BatchNorm1d(num_hid,affine=False)
def forward(self, q, gv_pos, self_sup=True):
"""Forward
q: [batch_size, seq_length]
gv_pos: [batch, K, v_dim]
self_sup: use negative images or not
return: logits, not probs
"""
w_emb = self.w_emb(q)
q_emb = self.q_emb(w_emb) # run GRU on word embeddings [batch, q_dim]
q_repr = self.q_net(q_emb)
batch_size = q.size(0)
logits_pos, att_gv_pos = self.compute_predict(q_repr, q_emb, gv_pos)
if self_sup:
# construct an irrelevant Q-I pair for each instance
index = random.sample(range(0, batch_size), batch_size)
gv_neg = gv_pos[index]
logits_neg, att_gv_neg = \
self.compute_predict(q_repr, q_emb, gv_neg)
return logits_pos, logits_neg, att_gv_pos, att_gv_neg
else:
return logits_pos, att_gv_pos
def compute_predict(self, q_repr, q_emb, v):
att_1 = self.gv_att_1(v, q_emb) # [batch, 1, v_dim]
att_2 = self.gv_att_2(v, q_emb) # [batch, 1, v_dim]
att_gv = att_1 + att_2
gv_embs = (att_gv * v) # [batch, v_dim]
gv_emb = gv_embs.sum(1)
gv_repr = self.gv_net(gv_emb)
joint_repr = q_repr * gv_repr
joint_repr_normal = self.normal(joint_repr)
logits = self.classifier(joint_repr_normal)
return logits, att_gv
class Model(nn.Module):
def __init__(self, opt):
super(Model, self).__init__()
self.dictionary = Dictionary.load_from_file(opt.dataroot +
'dictionary.pkl')
num_hid = 128
activation = opt.activation
dropG = opt.dropG
dropW = opt.dropW
dropout = opt.dropout
dropL = opt.dropL
norm = opt.norm
dropC = opt.dropC
self.opt = opt
self.w_emb = WordEmbedding(opt.ntokens, emb_dim=300, dropout=dropW)
self.w_emb.init_embedding(opt.dataroot + 'glove6b_init_300d.npy')
self.q_emb = QuestionEmbedding(in_dim=300,
num_hid=num_hid,
nlayers=1,
bidirect=False,
dropout=dropG,
rnn_type='GRU')
self.q_net = FCNet([self.q_emb.num_hid, num_hid],
dropout=dropL,
norm=norm,
act=activation)
self.classifier = SimpleClassifier(
in_dim=num_hid,
hid_dim=num_hid // 2,
out_dim=2, #opt.test_candi_ans_num,
dropout=dropC,
norm=norm,
act=activation)
self.normal = nn.BatchNorm1d(num_hid, affine=False)
def forward(self, q):
q = self.tokenize(q)
q = torch.from_numpy(np.array(q))
w_emb = self.w_emb(q.cuda())
q_emb = self.q_emb(w_emb)
q_repr = self.q_net(q_emb)
batch_size = q.size(0)
logits_pos = self.classifier(q_repr)
return logits_pos
def tokenize(self, q_text, max_length=14):
"""Tokenizes the questions.
This will add q_token in each entry of the dataset.
-1 represent nil, and should be treated as padding_idx in embedding
"""
token_list = []
for q_iter in q_text:
tokens = self.dictionary.tokenize(q_iter, False)
tokens = tokens[:max_length]
if len(tokens) < max_length:
padding = [self.dictionary.padding_idx
] * (max_length - len(tokens))
tokens = tokens + padding
assert len(tokens) == max_length
token_list.append(tokens)
return token_list
vals[idx] /= z[row]
return vals
vals = normalize(inds, vals)
tfidf = torch.sparse.FloatTensor(torch.LongTensor(inds), torch.FloatTensor(vals))
tfidf = tfidf.coalesce()
# Latent word embeddings
emb_dim = 300
glove_file = os.path.join(dataroot, 'glove', 'glove.6B.%dd.txt' % emb_dim)
weights, word2emb = utils.create_glove_embedding_init(dictionary.idx2word[N:], glove_file)
print('tf-idf stochastic matrix (%d x %d) is generated.' % (tfidf.size(0), tfidf.size(1)))
return tfidf, weights
if __name__=='__main__':
dictionary = Dictionary.load_from_file('data_RAD/dictionary.pkl')
tfidf, weights = tfidf_from_questions(['train'], None, dictionary)
w_emb = WordEmbedding(dictionary.ntoken, 300, .0, 'c')
w_emb.init_embedding(os.path.join('data_RAD', 'glove6b_init_300d.npy'), tfidf, weights)
with open('data_RAD/embed_tfidf_weights.pkl', 'wb') as f:
torch.save(w_emb, f)
print("Saving embedding with tfidf and weights successfully")
#
# dictionary = Dictionary.load_from_file('data_RAD/dictionary.pkl')
# w_emb = WordEmbedding(dictionary.ntoken, 300, .0, 'c')
# with open('data_RAD/embed_tfidf_weights.pkl', 'rb') as f:
# w_emb = torch.load(f)
# print("Load embedding with tfidf and weights successfully")