def ENCODE(self, words, orig_tree = None): word_rep = self.Wwrd(makevar(words)).squeeze(0) word_seq = word_rep.view(word_rep.size(0),1,word_rep.size(1)) output0, (ht0, ct0) = self.rnn0(word_seq, (self.h00, self.c00)) output1, (ht1, ct1) = self.rnn1(output0, (self.h01, self.c01)) outputs = torch.cat( [output0.view(output0.size(0),-1),output1.view(output1.size(0),-1)],1) scores = self.WscrSUB(self.DROP(outputs)).t() att_sub = self.SMAX(scores).t() weighted = att_sub.expand_as(word_rep) * word_rep qsub = torch.sum(weighted, 0) scores = self.WscrOBJ(self.DROP(outputs)).t() att_obj = self.SMAX(scores).t() weighted = att_obj.expand_as(word_rep) * word_rep qobj = torch.sum(weighted, 0) scores = self.WscrREL(self.DROP(outputs)).t() att_rel = self.SMAX(scores).t() weighted = att_rel.expand_as(word_rep) * word_rep qrel = torch.sum(weighted, 0) if self.debug: print "ENC>att_sub:",printVec(att_sub.t()) print "ENC>att_obj:",printVec(att_obj.t()) print "ENC>att_rel:",printVec(att_rel.t()) return qsub,qobj,qrel
def ENCODE(self, words, orig_tree=None): word_rep = self.Wwrd(makevar(words)).squeeze(0) word_seq = word_rep.view(word_rep.size(0), 1, word_rep.size(1)) output0, (ht0, ct0) = self.rnn0(word_seq, (self.h00, self.c00)) output1, (ht1, ct1) = self.rnn1(output0, (self.h01, self.c01)) outputs = torch.cat([ output0.view(output0.size(0), -1), output1.view(output1.size(0), -1) ], 1) # print "" # print "_"*20 # print "D>>outputs",outputs.size() # print "D>>outputs",outputs[-1].view(1,outputs[-1].size(0)).size() qsub = self.WprojSUB(outputs[-1].view(1, outputs[-1].size(0))) # scores = self.WscrOBJ(self.DROP(outputs)).t() # att_obj = self.SMAX(scores).t() # weighted = att_obj.expand_as(word_rep) * word_rep qobj = self.WprojOBJ(outputs[-1].view(1, outputs[-1].size(0))) # scores = self.WscrREL(self.DROP(outputs)).t() # att_rel = self.SMAX(scores).t() # weighted = att_rel.expand_as(word_rep) * word_rep qrel = self.WprojREL(outputs[-1].view(1, outputs[-1].size(0))) if self.debug: print "ENC>att_sub:", printVec(att_sub.t()) print "ENC>att_obj:", printVec(att_obj.t()) print "ENC>att_rel:", printVec(att_rel.t()) return qsub, qobj, qrel
def REL(self, t_txt, box1, box2): #conc = torch.cat((box1.expand_as(box2),box2),1) conc = torch.cat((box1[:, -5:].expand_as(box2[:, -5:]), box2[:, -5:]), 1) if self.use_outer: qkern0 = outer(box1[:, -5:].view(1, -1), box2[:, -5:]) conc = torch.cat((conc, qkern0), 1) spat = self.Wrbox(conc) if self.fusion == 'mul': prod = t_txt.expand_as(spat) * spat norm = prod / torch.norm(prod, 2, 1).expand_as(prod) elif self.fusion == 'sum': norm = t_txt.expand_as(spat) + spat elif self.fusion == 'concat': norm = torch.cat([t_txt.expand_as(spat), spat], 1) else: raise NotImplementedError() score = self.Wrel1(norm) # score = self.Wrel1(self.RELU(self.Wrel0(norm))) if self.debug: print print "REL>norm:", printVec(norm.t()) print "REL>score:", printVec(score.t()) return score
def LOC(self, t_txt, box): t_box = self.Wbox(box) t_sum = t_txt.expand_as(t_box) * t_box norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum) score = self.Wout0(norm) if self.debug: print print "LOC>norm:", printVec(norm.t()) print "LOC>score:", printVec(score.t()) return score
def forward(self, txt, box, orig_tree=None): qsub = self.ENCODE(txt) score_sub = self.LOC(qsub, box) lprob = self.SIGM(score_sub.t().add(EPS)) if self.debug: print print "=" * 20 print "D>sub", printVec(score_sub.t()) print "=" * 20, print "D>lprob", printVec(lprob) print "*" * 20 return lprob
def SHIFT(self, t_txt, box, p_child): A = [] for i in range(box.size(0)): A.append(self.REL(t_txt, box[i].view(1, -1), box)) A[i][i] = 0.0 A = torch.cat(A, 1).t() prob = self.LSMAX(torch.mm(torch.exp(p_child), A).add(EPS)) if self.debug: print "SHIFT>child", printVec(p_child, logprob=True) for i in range(box.size(0)): print "SHIFTA[{}]:".format(i), printVec(A[i]) print "SHIFT>score:", printVec(prob, logprob=True) return prob
def forward(self, txt, box, orig_tree = None): qsub = self.ENCODE(txt) score_sub = self.LOC(qsub,box) lprob = self.LSMAX(score_sub.t().add(EPS)) if self.debug: print print "="*20 print "D>sub",printVec(score_sub.t()) print "="*20, print "D>lprob",printVec(lprob, logprob = True) print "*"*20 # if self.evaluate: # return lprob, [], [], [] return lprob
def LOC(self, t_txt, box): t_box1 = self.RELU(self.Wbox1(box)) t_box2 = self.RELU(self.Wbox2(t_box1)) t_box = self.Wbox3(t_box2) t_sum = t_txt.expand_as(t_box) * t_box norm = t_sum / torch.norm(t_sum,2,1).expand_as(t_sum) score0 = self.RELU(self.Wout0(norm)) score1 = self.RELU(self.Wout1(score0)) score = self.Wout2(score1) # score = self.Wout(norm) if self.debug: print print "LOC>norm:",printVec(norm.t()) print "LOC>score:",printVec(score.t()) return score
def expr_for_txt(self, tree, box, orig_tree = None, decorate = False): assert(not tree.isleaf()) if len(tree.children) == 1: if tree.children[0].isleaf(): wlist = [ self.w2i.get(w,0) for w in filter(lambda a: a!= '', tree.children[0].label.split('_'))] if self.use_phrase_context: # context = makevar(np.zeros((1,self.hdim*8)), numpy_var = True) context = torch.cat([self.context0,self.context1],1) else: context = self.context0 # context = makevar(np.zeros((1,self.hdim*4)), numpy_var = True) txt, attn_weights, context = self.ENCODE(wlist, context, orig_tree) prob = self.LOC(txt, box) if decorate: tree._expr = prob tree._attn = attn_weights return prob, context else: p_child, context = self.expr_for_txt(tree.children[0],box, orig_tree, decorate) if tree.label == 'loc': wlist = [ self.w2i.get(w,0) for w in filter(lambda a: a!= '', tree.children[0].label.split('_'))] plist = wlist #context = makevar(np.zeros((1,self.hdim*4)), numpy_var = True) context = self.context0 else: wlist = [self.w2i.get(w,0) for w in orig_tree.original_text.split()] plist = [ self.w2i.get(w,0) for w in filter(lambda a: a!= '', tree.label.split('_'))] if self.use_phrase_context: p_ctx = self.CONTEXT(plist) context = torch.cat([context,p_ctx],1) txt, attn_weights, context = self.ENCODE(wlist, context, orig_tree) prob = self.SHIFT(txt,box,p_child) if decorate: tree._expr = prob tree._attn = attn_weights return prob, context assert(len(tree.children) == 2) p_l, ctx_l = self.expr_for_txt(tree.children[0], box, orig_tree, decorate) p_r, ctx_r = self.expr_for_txt(tree.children[1], box, orig_tree, decorate) prob = (p_l + p_r) norm = torch.exp(prob) norm = norm / torch.norm(norm,1,1).expand_as(norm) prob = torch.log(norm).add(EPS) if decorate: tree._attn = None tree._expr = prob if self.debug: print "AND>",printVec(prob, logprob = True) return prob, torch.mean(torch.cat([ctx_l, ctx_r],0),0)
def LOC(self, t_txt, box): t_box = self.Wbox(box) if self.fusion == 'mul': t_sum = t_txt.expand_as(t_box) * t_box norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum) elif self.fusion == 'sum': norm = t_txt.expand_as(t_box) + t_box elif self.fusion == 'concat': norm = torch.cat([t_txt.expand_as(t_box), t_box], 1) else: raise NotImplementedError() score = self.Wout0(norm) if self.debug: print print "LOC>norm:", printVec(norm.t()) print "LOC>score:", printVec(score.t()) return score
def expr_for_txt(self, tree, box, orig_tree=None, decorate=False): assert (not tree.isleaf()) if len(tree.children) == 1: if tree.children[0].isleaf(): wlist = [ self.w2i.get(w, 0) for w in filter( lambda a: a != '', tree.children[0].label.split('_')) ] txt, attn_weights = self.ENCODE(wlist, orig_tree) prob = self.LOC(txt, box) if decorate: tree._expr = prob tree._attn = attn_weights return prob else: if tree.label == 'loc': wlist = [ self.w2i.get(w, 0) for w in filter(lambda a: a != '', tree.children[0].label.split('_')) ] else: wlist = [ self.w2i.get(w, 0) for w in filter( lambda a: a != '', tree.label.split('_')) ] txt, attn_weights = self.ENCODE(wlist) p_child = self.expr_for_txt(tree.children[0], box, orig_tree, decorate) prob = self.SHIFT(txt, box, p_child) if decorate: tree._expr = prob tree._attn = attn_weights return prob assert (len(tree.children) == 2) p_l = self.expr_for_txt(tree.children[0], box, orig_tree, decorate) p_r = self.expr_for_txt(tree.children[1], box, orig_tree, decorate) prob = (p_l + p_r) norm = torch.exp(prob) norm = norm / torch.norm(norm, 1, 1).expand_as(norm) prob = torch.log(norm).add(EPS) if decorate: tree._attn = None tree._expr = prob if self.debug: print "AND>", printVec(prob, logprob=True) return prob
def forward(self, txt, box, orig_tree=None): qsub, qobj, qrel = self.ENCODE(txt) score_sub = self.LOC(qsub, box) score_obj = self.LOC(qobj, box) score_rel = [] # score_rel = makevar(np.zeros((box.size(0),box.size(0))),numpy_var=True) for i in range(box.size(0)): # score_rel[i] = self.REL(qrel, box[i].view(1,-1), box) score_rel.append(self.REL(qrel, box[i].view(1, -1), box)) score_rel = torch.cat(score_rel, 1).t() score_tot = score_sub.t().expand_as(score_rel) + score_obj.expand_as( score_rel) + score_rel score_fin = score_tot.max(1)[0].view(1, -1) lprob = self.LSMAX(score_fin).add(EPS) if self.debug: print print "=" * 20 print "D>rel", score_rel print "=" * 20 print "D>obj", printVec(score_obj.t()) print "=" * 20 print "D>sub", printVec(score_sub.t()) print "=" * 20 print "D>tot", score_tot print "=" * 20 print "D>max", score_tot.max(1)[0].max(0)[0], score_tot.max( 1)[0].max(0)[1] print "=" * 20, print "D>fin", printVec(score_fin) print "=" * 20, print "D>lprob", printVec(lprob, logprob=True) print "*" * 20 # if self.evaluate: # return lprob, score_sub, score_obj, score_rel return lprob
def LOC(self, t_txt, box): if self.use_outer: O = [] for i in range(box.size(0)): qkern0 = outer(box[i, self.cnn_feat:].contiguous().view(1, -1), box[i, self.cnn_feat:].view(1, -1)) O.append(qkern0) box_out = torch.cat((box, torch.cat(O, 0)), 1) t_box = self.Wbox(box_out) else: t_box = self.Wbox(box) t_sum = t_txt.expand_as(t_box) * t_box norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum) score = self.Wout0(norm) if self.debug: print print "LOC>norm:", printVec(norm.t()) print "LOC>score:", printVec(score.t()) return score
def ENCODE(self, words, orig_tree = None): word_rep = self.Wwrd(makevar(words)).squeeze(0) word_seq = word_rep.view(word_rep.size(0),1,word_rep.size(1)) h00 = c00 = makevar(np.zeros((2,1,self.hdim)),numpy_var = True) h01 = c01 = makevar(np.zeros((2,1,self.hdim)),numpy_var = True) output0, (ht0, ct0) = self.rnn0(word_seq, (h00, c00)) output1, (ht1, ct1) = self.rnn1(output0, (h01, c01)) outputs = torch.cat( [output0.view(output0.size(0),-1),output1.view(output1.size(0),-1)],1) scores = self.WscrSUB(self.DROP(outputs)).t() att_sub = self.SMAX(scores).t() weighted = att_sub.expand_as(word_rep) * word_rep qsub = torch.sum(weighted, 0) if self.debug: print "ENC>att_sub:",printVec(att_sub.t()) return qsub
def LOC(self, t_txt, box): t_box = self.Wbox(box) if self.fusion == 'mul': t_sum = t_txt.expand_as(t_box) * t_box norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum) elif self.fusion == 'sum': norm = t_txt.expand_as(t_box) + t_box elif self.fusion == 'concat': norm = torch.cat([t_txt.expand_as(t_box), t_box], 1) else: raise NotImplementedError() if self.layer == 2: norm = self.Wout0(norm) score = self.Wout1(norm).t() prob = self.LSMAX(score).add(EPS) if self.debug: print "LOC>prob:", printVec(prob, logprob=True) return prob
loss = criterion(prediction, gold) if math.isnan(float(loss.data[0])): print("") print("="*20) print("problem with instance: {}, filename {}".format(ii,tree.fname)) print("Nonterms:"," ".join([n.label for n in tree.nonterms()])) print("Span:",tree.getSpan()) print("Gold:",Ytrn[ii]) print("*"*20) net.debug = True if config['model'] in set(["groundnet", "groundnetflexall", "groundnetflexrel","treernn"]): pred = net(tree, box_rep, tree) else: pred = net([w2i.get(n.label,0) for n in tree.leaves()], box_rep, tree) print("Prediction:",printVec(pred, logprob = False)) print("="*60) quit() closs += float(loss.data[0]) cinst += 1 optimizer.zero_grad() loss.backward() torch.nn.utils.clip_grad_norm(net.parameters(), config['clip']) optimizer.step() if args.verbose: pbar.set_description("trn_loss {:5.3f} trn_acc {:5.3f}".format(closs/cinst,correct/cinst)) if time.time() - start_time > args.timeout: timeout = True break