def ENCODE(self, words, orig_tree = None):
word_rep = self.Wwrd(makevar(words)).squeeze(0)
word_seq = word_rep.view(word_rep.size(0),1,word_rep.size(1))
output0, (ht0, ct0) = self.rnn0(word_seq, (self.h00, self.c00))
output1, (ht1, ct1) = self.rnn1(output0, (self.h01, self.c01))
outputs = torch.cat( [output0.view(output0.size(0),-1),output1.view(output1.size(0),-1)],1)
scores = self.WscrSUB(self.DROP(outputs)).t()
att_sub = self.SMAX(scores).t()
weighted = att_sub.expand_as(word_rep) * word_rep
qsub = torch.sum(weighted, 0)
scores = self.WscrOBJ(self.DROP(outputs)).t()
att_obj = self.SMAX(scores).t()
weighted = att_obj.expand_as(word_rep) * word_rep
qobj = torch.sum(weighted, 0)
scores = self.WscrREL(self.DROP(outputs)).t()
att_rel = self.SMAX(scores).t()
weighted = att_rel.expand_as(word_rep) * word_rep
qrel = torch.sum(weighted, 0)
if self.debug:
print "ENC>att_sub:",printVec(att_sub.t())
print "ENC>att_obj:",printVec(att_obj.t())
print "ENC>att_rel:",printVec(att_rel.t())
return qsub,qobj,qrel
def ENCODE(self, words, orig_tree=None):
word_rep = self.Wwrd(makevar(words)).squeeze(0)
word_seq = word_rep.view(word_rep.size(0), 1, word_rep.size(1))
output0, (ht0, ct0) = self.rnn0(word_seq, (self.h00, self.c00))
output1, (ht1, ct1) = self.rnn1(output0, (self.h01, self.c01))
outputs = torch.cat([
output0.view(output0.size(0), -1),
output1.view(output1.size(0), -1)
], 1)
# print ""
# print "_"*20
# print "D>>outputs",outputs.size()
# print "D>>outputs",outputs[-1].view(1,outputs[-1].size(0)).size()
qsub = self.WprojSUB(outputs[-1].view(1, outputs[-1].size(0)))
# scores = self.WscrOBJ(self.DROP(outputs)).t()
# att_obj = self.SMAX(scores).t()
# weighted = att_obj.expand_as(word_rep) * word_rep
qobj = self.WprojOBJ(outputs[-1].view(1, outputs[-1].size(0)))
# scores = self.WscrREL(self.DROP(outputs)).t()
# att_rel = self.SMAX(scores).t()
# weighted = att_rel.expand_as(word_rep) * word_rep
qrel = self.WprojREL(outputs[-1].view(1, outputs[-1].size(0)))
if self.debug:
print "ENC>att_sub:", printVec(att_sub.t())
print "ENC>att_obj:", printVec(att_obj.t())
print "ENC>att_rel:", printVec(att_rel.t())
return qsub, qobj, qrel
def REL(self, t_txt, box1, box2):
#conc = torch.cat((box1.expand_as(box2),box2),1)
conc = torch.cat((box1[:, -5:].expand_as(box2[:, -5:]), box2[:, -5:]),
1)
if self.use_outer:
qkern0 = outer(box1[:, -5:].view(1, -1), box2[:, -5:])
conc = torch.cat((conc, qkern0), 1)
spat = self.Wrbox(conc)
if self.fusion == 'mul':
prod = t_txt.expand_as(spat) * spat
norm = prod / torch.norm(prod, 2, 1).expand_as(prod)
elif self.fusion == 'sum':
norm = t_txt.expand_as(spat) + spat
elif self.fusion == 'concat':
norm = torch.cat([t_txt.expand_as(spat), spat], 1)
else:
raise NotImplementedError()
score = self.Wrel1(norm)
# score = self.Wrel1(self.RELU(self.Wrel0(norm)))
if self.debug:
print
print "REL>norm:", printVec(norm.t())
print "REL>score:", printVec(score.t())
return score
def LOC(self, t_txt, box):
t_box = self.Wbox(box)
t_sum = t_txt.expand_as(t_box) * t_box
norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum)
score = self.Wout0(norm)
if self.debug:
print
print "LOC>norm:", printVec(norm.t())
print "LOC>score:", printVec(score.t())
return score
def forward(self, txt, box, orig_tree=None):
qsub = self.ENCODE(txt)
score_sub = self.LOC(qsub, box)
lprob = self.SIGM(score_sub.t().add(EPS))
if self.debug:
print
print "=" * 20
print "D>sub", printVec(score_sub.t())
print "=" * 20,
print "D>lprob", printVec(lprob)
print "*" * 20
return lprob
def SHIFT(self, t_txt, box, p_child):
A = []
for i in range(box.size(0)):
A.append(self.REL(t_txt, box[i].view(1, -1), box))
A[i][i] = 0.0
A = torch.cat(A, 1).t()
prob = self.LSMAX(torch.mm(torch.exp(p_child), A).add(EPS))
if self.debug:
print "SHIFT>child", printVec(p_child, logprob=True)
for i in range(box.size(0)):
print "SHIFTA[{}]:".format(i), printVec(A[i])
print "SHIFT>score:", printVec(prob, logprob=True)
return prob
def forward(self, txt, box, orig_tree = None):
qsub = self.ENCODE(txt)
score_sub = self.LOC(qsub,box)
lprob = self.LSMAX(score_sub.t().add(EPS))
if self.debug:
print
print "="*20
print "D>sub",printVec(score_sub.t())
print "="*20,
print "D>lprob",printVec(lprob, logprob = True)
print "*"*20
# if self.evaluate:
# return lprob, [], [], []
return lprob
def LOC(self, t_txt, box):
t_box1 = self.RELU(self.Wbox1(box))
t_box2 = self.RELU(self.Wbox2(t_box1))
t_box = self.Wbox3(t_box2)
t_sum = t_txt.expand_as(t_box) * t_box
norm = t_sum / torch.norm(t_sum,2,1).expand_as(t_sum)
score0 = self.RELU(self.Wout0(norm))
score1 = self.RELU(self.Wout1(score0))
score = self.Wout2(score1)
# score = self.Wout(norm)
if self.debug:
print
print "LOC>norm:",printVec(norm.t())
print "LOC>score:",printVec(score.t())
return score
def expr_for_txt(self, tree, box, orig_tree = None, decorate = False):
assert(not tree.isleaf())
if len(tree.children) == 1:
if tree.children[0].isleaf():
wlist = [ self.w2i.get(w,0) for w in filter(lambda a: a!= '', tree.children[0].label.split('_'))]
if self.use_phrase_context:
# context = makevar(np.zeros((1,self.hdim*8)), numpy_var = True)
context = torch.cat([self.context0,self.context1],1)
else:
context = self.context0
# context = makevar(np.zeros((1,self.hdim*4)), numpy_var = True)
txt, attn_weights, context = self.ENCODE(wlist, context, orig_tree)
prob = self.LOC(txt, box)
if decorate:
tree._expr = prob
tree._attn = attn_weights
return prob, context
else:
p_child, context = self.expr_for_txt(tree.children[0],box, orig_tree, decorate)
if tree.label == 'loc':
wlist = [ self.w2i.get(w,0) for w in filter(lambda a: a!= '', tree.children[0].label.split('_'))]
plist = wlist
#context = makevar(np.zeros((1,self.hdim*4)), numpy_var = True)
context = self.context0
else:
wlist = [self.w2i.get(w,0) for w in orig_tree.original_text.split()]
plist = [ self.w2i.get(w,0) for w in filter(lambda a: a!= '', tree.label.split('_'))]
if self.use_phrase_context:
p_ctx = self.CONTEXT(plist)
context = torch.cat([context,p_ctx],1)
txt, attn_weights, context = self.ENCODE(wlist, context, orig_tree)
prob = self.SHIFT(txt,box,p_child)
if decorate:
tree._expr = prob
tree._attn = attn_weights
return prob, context
assert(len(tree.children) == 2)
p_l, ctx_l = self.expr_for_txt(tree.children[0], box, orig_tree, decorate)
p_r, ctx_r = self.expr_for_txt(tree.children[1], box, orig_tree, decorate)
prob = (p_l + p_r)
norm = torch.exp(prob)
norm = norm / torch.norm(norm,1,1).expand_as(norm)
prob = torch.log(norm).add(EPS)
if decorate:
tree._attn = None
tree._expr = prob
if self.debug:
print "AND>",printVec(prob, logprob = True)
return prob, torch.mean(torch.cat([ctx_l, ctx_r],0),0)
def LOC(self, t_txt, box):
t_box = self.Wbox(box)
if self.fusion == 'mul':
t_sum = t_txt.expand_as(t_box) * t_box
norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum)
elif self.fusion == 'sum':
norm = t_txt.expand_as(t_box) + t_box
elif self.fusion == 'concat':
norm = torch.cat([t_txt.expand_as(t_box), t_box], 1)
else:
raise NotImplementedError()
score = self.Wout0(norm)
if self.debug:
print
print "LOC>norm:", printVec(norm.t())
print "LOC>score:", printVec(score.t())
return score
def expr_for_txt(self, tree, box, orig_tree=None, decorate=False):
assert (not tree.isleaf())
if len(tree.children) == 1:
if tree.children[0].isleaf():
wlist = [
self.w2i.get(w, 0) for w in filter(
lambda a: a != '', tree.children[0].label.split('_'))
]
txt, attn_weights = self.ENCODE(wlist, orig_tree)
prob = self.LOC(txt, box)
if decorate:
tree._expr = prob
tree._attn = attn_weights
return prob
else:
if tree.label == 'loc':
wlist = [
self.w2i.get(w, 0)
for w in filter(lambda a: a != '',
tree.children[0].label.split('_'))
]
else:
wlist = [
self.w2i.get(w, 0) for w in filter(
lambda a: a != '', tree.label.split('_'))
]
txt, attn_weights = self.ENCODE(wlist)
p_child = self.expr_for_txt(tree.children[0], box, orig_tree,
decorate)
prob = self.SHIFT(txt, box, p_child)
if decorate:
tree._expr = prob
tree._attn = attn_weights
return prob
assert (len(tree.children) == 2)
p_l = self.expr_for_txt(tree.children[0], box, orig_tree, decorate)
p_r = self.expr_for_txt(tree.children[1], box, orig_tree, decorate)
prob = (p_l + p_r)
norm = torch.exp(prob)
norm = norm / torch.norm(norm, 1, 1).expand_as(norm)
prob = torch.log(norm).add(EPS)
if decorate:
tree._attn = None
tree._expr = prob
if self.debug:
print "AND>", printVec(prob, logprob=True)
return prob
def forward(self, txt, box, orig_tree=None):
qsub, qobj, qrel = self.ENCODE(txt)
score_sub = self.LOC(qsub, box)
score_obj = self.LOC(qobj, box)
score_rel = []
# score_rel = makevar(np.zeros((box.size(0),box.size(0))),numpy_var=True)
for i in range(box.size(0)):
# score_rel[i] = self.REL(qrel, box[i].view(1,-1), box)
score_rel.append(self.REL(qrel, box[i].view(1, -1), box))
score_rel = torch.cat(score_rel, 1).t()
score_tot = score_sub.t().expand_as(score_rel) + score_obj.expand_as(
score_rel) + score_rel
score_fin = score_tot.max(1)[0].view(1, -1)
lprob = self.LSMAX(score_fin).add(EPS)
if self.debug:
print
print "=" * 20
print "D>rel", score_rel
print "=" * 20
print "D>obj", printVec(score_obj.t())
print "=" * 20
print "D>sub", printVec(score_sub.t())
print "=" * 20
print "D>tot", score_tot
print "=" * 20
print "D>max", score_tot.max(1)[0].max(0)[0], score_tot.max(
1)[0].max(0)[1]
print "=" * 20,
print "D>fin", printVec(score_fin)
print "=" * 20,
print "D>lprob", printVec(lprob, logprob=True)
print "*" * 20
# if self.evaluate:
# return lprob, score_sub, score_obj, score_rel
return lprob
def LOC(self, t_txt, box):
if self.use_outer:
O = []
for i in range(box.size(0)):
qkern0 = outer(box[i, self.cnn_feat:].contiguous().view(1, -1),
box[i, self.cnn_feat:].view(1, -1))
O.append(qkern0)
box_out = torch.cat((box, torch.cat(O, 0)), 1)
t_box = self.Wbox(box_out)
else:
t_box = self.Wbox(box)
t_sum = t_txt.expand_as(t_box) * t_box
norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum)
score = self.Wout0(norm)
if self.debug:
print
print "LOC>norm:", printVec(norm.t())
print "LOC>score:", printVec(score.t())
return score
def ENCODE(self, words, orig_tree = None):
word_rep = self.Wwrd(makevar(words)).squeeze(0)
word_seq = word_rep.view(word_rep.size(0),1,word_rep.size(1))
h00 = c00 = makevar(np.zeros((2,1,self.hdim)),numpy_var = True)
h01 = c01 = makevar(np.zeros((2,1,self.hdim)),numpy_var = True)
output0, (ht0, ct0) = self.rnn0(word_seq, (h00, c00))
output1, (ht1, ct1) = self.rnn1(output0, (h01, c01))
outputs = torch.cat( [output0.view(output0.size(0),-1),output1.view(output1.size(0),-1)],1)
scores = self.WscrSUB(self.DROP(outputs)).t()
att_sub = self.SMAX(scores).t()
weighted = att_sub.expand_as(word_rep) * word_rep
qsub = torch.sum(weighted, 0)
if self.debug:
print "ENC>att_sub:",printVec(att_sub.t())
return qsub
def LOC(self, t_txt, box):
t_box = self.Wbox(box)
if self.fusion == 'mul':
t_sum = t_txt.expand_as(t_box) * t_box
norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum)
elif self.fusion == 'sum':
norm = t_txt.expand_as(t_box) + t_box
elif self.fusion == 'concat':
norm = torch.cat([t_txt.expand_as(t_box), t_box], 1)
else:
raise NotImplementedError()
if self.layer == 2:
norm = self.Wout0(norm)
score = self.Wout1(norm).t()
prob = self.LSMAX(score).add(EPS)
if self.debug:
print "LOC>prob:", printVec(prob, logprob=True)
return prob
loss = criterion(prediction, gold)
if math.isnan(float(loss.data[0])):
print("")
print("="*20)
print("problem with instance: {}, filename {}".format(ii,tree.fname))
print("Nonterms:"," ".join([n.label for n in tree.nonterms()]))
print("Span:",tree.getSpan())
print("Gold:",Ytrn[ii])
print("*"*20)
net.debug = True
if config['model'] in set(["groundnet", "groundnetflexall", "groundnetflexrel","treernn"]):
pred = net(tree, box_rep, tree)
else:
pred = net([w2i.get(n.label,0) for n in tree.leaves()], box_rep, tree)
print("Prediction:",printVec(pred, logprob = False))
print("="*60)
quit()
closs += float(loss.data[0])
cinst += 1
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm(net.parameters(), config['clip'])
optimizer.step()
if args.verbose:
pbar.set_description("trn_loss {:5.3f} trn_acc {:5.3f}".format(closs/cinst,correct/cinst))
if time.time() - start_time > args.timeout:
timeout = True
break
