def REL(self, t_txt, box1, box2):
#conc = torch.cat((box1.expand_as(box2),box2),1)
conc = torch.cat((box1[:, -5:].expand_as(box2[:, -5:]), box2[:, -5:]),
1)
if self.use_outer:
qkern0 = outer(box1[:, -5:].view(1, -1), box2[:, -5:])
conc = torch.cat((conc, qkern0), 1)
spat = self.Wrbox(conc)
if self.fusion == 'mul':
prod = t_txt.expand_as(spat) * spat
norm = prod / torch.norm(prod, 2, 1).expand_as(prod)
elif self.fusion == 'sum':
norm = t_txt.expand_as(spat) + spat
elif self.fusion == 'concat':
norm = torch.cat([t_txt.expand_as(spat), spat], 1)
else:
raise NotImplementedError()
score = self.Wrel1(norm)
# score = self.Wrel1(self.RELU(self.Wrel0(norm)))
if self.debug:
print
print "REL>norm:", printVec(norm.t())
print "REL>score:", printVec(score.t())
return score
def REL(self, t_txt, box1, box2):
if self.only_spatial:
conc = torch.cat(
(box1[:, -5:].expand_as(box2[:, -5:]), box2[:, -5:]), 1)
else:
conc = torch.cat((box1.expand_as(box2), box2), 1)
if self.use_outer:
qkern0 = outer(box1[:, -5:].view(1, -1), box2[:, -5:])
conc = torch.cat((conc, qkern0), 1)
spat = self.Wrbox(conc)
if self.fusion == 'mul':
prod = t_txt.expand_as(spat) * spat
norm = prod / torch.norm(prod, 2, 1).expand_as(prod)
elif self.fusion == 'sum':
norm = t_txt.expand_as(spat) + spat
elif self.fusion == "concat":
norm = torch.cat([t_txt.expand_as(spat), spat], 1)
if self.layer == 2:
norm = self.RELU(self.Wrel0(norm))
score = self.Wrel1(norm)
return score
def LOC(self, t_txt, box):
if self.use_outer:
O = []
for i in range(box.size(0)):
qkern0 = outer(box[i, self.cnn_feat:].contiguous().view(1, -1),
box[i, self.cnn_feat:].view(1, -1))
O.append(qkern0)
box_out = torch.cat((box, torch.cat(O, 0)), 1)
t_box = self.Wbox(box_out)
else:
t_box = self.Wbox(box)
t_sum = t_txt.expand_as(t_box) * t_box
norm = t_sum / torch.norm(t_sum, 2, 1).expand_as(t_sum)
score = self.Wout0(norm)
if self.debug:
print
print "LOC>norm:", printVec(norm.t())
print "LOC>score:", printVec(score.t())
return score