def batch_forward(self, inputs, child_h_sum, child_fc_sum):
child_h_sum = torch.squeeze(child_h_sum, 1)
child_fc_sum = torch.squeeze(child_fc_sum, 1)
i = F.sigmoid(self.ix(inputs) + self.ih(child_h_sum))
o = F.sigmoid(self.ox(inputs) + self.oh(child_h_sum))
u = F.tanh(self.ux(inputs) + self.uh(child_h_sum))
c = F.mul(i, u) + child_fc_sum
h = F.mul(o, F.tanh(c))
return c, h
def forward(self, rgb_inp, flow_inp):
x_rgb = self.rgb_rnn(rgb_inp)[0]
x_flow = self.rgb_rnn(flow_inp)[0]
#fused_outputs = torch.cat([x_rgb, x_flow], dim=2)
alpha = F.sigmoid(self.w_att(x_rgb) + self.w_att(x_flow))
fused_outputs = F.mul(alpha, x_rgb) + F.mul((1-alpha), x_flow)
outputs = self.out(fused_outputs)
outputs_mean = Variable(torch.zeros(outputs.size()[0], num_classes)).cuda(DEVICE)
for i in range(outputs.size()[0]):
outputs_mean[i] = outputs[i].mean(dim=0)
return outputs_mean
def forward(self, x, hidden):
hx, cx = hidden
x = x.view(-1, x.size(1))
gates = self.x2h(x) + self.h2h(hx)
gates = gates.squeeze()
# 将gates按照通道数dim=1也就是4*hidden_size切分为四个
ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1)
ingate = F.sigmoid(ingate)
forgetgate = F.sigmoid(forgetgate)
cellgate = F.tanh(cellgate)
outgate = F.sigmoid(outgate)
cy = F.mul(cx, forgetgate) + F.mul(ingate, cellgate)
hy = F.mul(outgate, F.tanh(cy))
return (hy, cy)
def GeneralizedDice(probs, onehot_labels):
'''
:param probs: b2wh, the probs of last CNN layer input to F.log_softmax()
:param onehot_labels: one-hot operation labels
:return:
'''
#assert utils.checkSimplex_(probs) and utils.checkSimplex_(onehot_labels)
idc = [0, 1]
#pc = probs[:, idc, ...].type(torch.float32) #pc: bcwh
pc = probs.type(torch.float32)
#tc = onehot_labels[:, idc, ...].type(torch.float32)
tc = onehot_labels #convert ndarray to Tensor
pc_ = torch.zeros(pc.shape[0], pc.shape[1]).cuda() # bc
tc_ = torch.zeros(tc.shape[0], tc.shape[1]).cuda() #bc
## intersection = torch.einsum('bcwh, bcwh -> bc', [pc, tc])
temp = F.mul(tc, pc) #bcwh
intersection_ = torch.zeros(pc.shape[0], pc.shape[1])
## bellow operation equals as
## pc_= torch.einsum('bcwh', [pc]) tc_ = torch.einsum('bcwh -> bc', [tc])
for vi in range(pc.shape[0]):
for vj in range(pc.shape[1]):
pc_[vi][vj] = torch.sum(pc[vi, vj, ...])
tc_[vi][vj] = torch.sum(tc[vi, vj, :, :])
intersection_ = torch.sum(
temp[vi, vj, :, :]) #intersection of pre mask and GT mask
w = 1 / ((pc_.type(torch.float32) + 1e-10)**2)
intersection = w * intersection_
union = w * (pc_ + tc_) # union of pre mask and GT mask
divided = 1 - 2 * (torch.sum(intersection, 1) +
1e-10) / (torch.sum(union, 1) + 1e-10)
#loss = divided.mean()
loss = torch.mean(divided)
return loss
def repar(self, mu, logvar):
# the infamous reparamaterization trick (aka 4 lines of code)
samp = self.sample()
samp = F.mul((0.5 * logvar).exp(), samp)
samp = samp + mu
return samp
def __call__(self, tensor):
"""
Args:
tensor (Tensor): Tensor image of size (C, H, W) to be normalized.
Returns:
Tensor: Normalized Tensor image.
"""
tensor = F.mul(tensor, self.factor)
return tensor
def forward(self, data_in):
out1 = self.dilated_conv_tanh(data_in)
out2 = self.dilated_conv_sigmoid(data_in)
tanh_out = torch.tanh(out1)
sigm_out = torch.sigmoid(out2)
data = F.mul(tanh_out, sigm_out)
skip = self.skip(data)
res = self.res(data) + data_in
return res, skip
def forward(self, x):
x_mean = torch.mean(torch.abs(x))
x = BinaryFunc.apply(x)
if self.is_inference:
x = self.layer(x)
else:
weight_full = self.layer.weight.data.clone()
self.layer.weight.data.sign_()
x = self.layer(x)
self.layer.weight.data = weight_full
x = F.mul(x, x_mean)
return x
def forward(self, x):
self.clip_recurrent_kernel()
output = self.non_linearity(
F.linear(x, self.input_kernel, self.bias) +
F.mul(self.recurrent_kernel, self.h))
# self.clip_recurrent_kernel()
# recurrent_update = self.h.mul(
# self.recurrent_kernel.expand_as(self.h))
# gate_inputs += recurrent_update.expand_as(gate_inputs)
# gate_inputs += self.bias.expand_as(gate_inputs)
# output = self.non_linearity(gate_inputs)
return output
def SurfaceLoss(probs, dis_map):
#assert utils.checkSimplex_(probs)
assert not utils.one_hot(dis_map) #if false throw exception e
idc = [1]
#pc = probs[:, idc, ...].type(torch.float32) #bcwh
pc = probs[:, 1, :].type(torch.float32)
dc = dis_map[:, 1, ...] #bcwh
multipled = F.mul(
pc, dc) #bcwh, equal to 'torch.einsum('bcwh, bcwh -> bcwh', [pc, dc])'
#loss = multipled.mean()
loss = torch.mean(multipled)
return loss
def forward(self, x, z):
x = self.alexnet(x)
y = F.relu(self.Conv1(x))
y = F.relu(self.Conv2(y))
y = F.relu(self.Conv3(y))
x = F.dropout(F.relu(F.mul(x, y)), 0.5)
x = x.view(x.size(0), -1)
x = self.layer1(x)
x = self.layer2(x)
x = self.fc(x)
return x
def forward(self, rgb_inputs, flow_inputs, lengths):
rgb_inputs = pad_sequence(rgb_inputs, batch_first=False)
rgb_inputs = pack_padded_sequence(rgb_inputs,
lengths,
batch_first=False)
rgb_inputs = rgb_inputs.cuda()
rgb_outputs, rgb_hiddens = self.indrnn(rgb_inputs)
rgb_outputs, _ = pad_packed_sequence(rgb_outputs, batch_first=False)
flow_inputs = pad_sequence(flow_inputs, batch_first=False)
flow_inputs = pack_padded_sequence(flow_inputs,
lengths,
batch_first=False)
flow_inputs = flow_inputs.cuda()
flow_outputs, flow_hiddens = self.indrnn(flow_inputs)
flow_outputs, _ = pad_packed_sequence(flow_outputs, batch_first=False)
alpha = F.sigmoid(self.w_att(rgb_outputs) + self.w_att(flow_outputs))
fused_outputs = F.mul(alpha, rgb_outputs) + F.mul(
(1 - alpha), flow_outputs)
outputs = self.out(fused_outputs)
#outputs = outputs.view(-1, num_classes)
return outputs
def forward(self, word_inputs, word_seq_length, char_inputs,
char_seq_length, char_recover):
"""
word_inputs: (batch_size,seq_len)
word_seq_length:()
"""
batch_size = word_inputs.size(0)
seq_len = word_inputs.size(1)
word_emb = self.word_embedding(word_inputs)
# word_rep = self.drop(word_emb)
word_rep = word_emb
if self.args.use_char:
size = char_inputs.size(0)
char_emb = self.char_embedding(char_inputs)
char_emb = pack(char_emb,
char_seq_length.numpy(),
batch_first=True)
char_lstm_out, char_hidden = self.char_feature(char_emb)
char_lstm_out = pad(char_lstm_out, batch_first=True)
char_hidden = char_hidden[0].transpose(1, 0).contiguous().view(
size, -1)
char_hidden = char_hidden[char_recover]
char_hidden = char_hidden.view(batch_size, seq_len, -1)
if self.args.attention:
word_rep = F.tanh(
self.attn1(word_emb) + self.attn2(char_hidden))
z = F.sigmoid(self.attn3(word_rep))
x = 1 - z
word_rep = F.mul(z, word_emb) + F.mul(x, char_hidden)
else:
word_rep = torch.cat((word_emb, char_hidden), 2)
word_rep = pack(word_rep,
word_seq_length.cpu().numpy(),
batch_first=True)
out, hidden = self.word_feature(word_rep)
out, _ = pad(out, batch_first=True)
if self.args.lstm_attention:
# tanh_out = F.tanh(out)
# att_vec = self.att1(tanh_out)
# att_sm_vec = F.softmax(att_vec)
# out = F.mul(out,att_sm_vec)
out_list, weight_list = [], []
for idx in range(seq_len):
# slice_out = out[:,0:idx+1,:]
if idx + 2 > seq_len:
slice_out = out
else:
slice_out = out[:, 0:idx + 2, :]
# slice_out = out
slice_out, weights = self.attention(slice_out)
# slice_out, weights = SelfAttention(self.args.hidden_dim*2).forward(slice_out)
out_list.append(slice_out.unsqueeze(1))
weight_list.append(weights)
out = torch.cat(out_list, dim=1)
# pass
# out = F.tanh(self.att1(out))
# out = self.softmax(out)
# out = out*out
# out = self.att2(out)
# else:
out = self.hidden2tag(self.drop(out))
return out
def forward(self, word_inputs, feat_inputs, word_seq_length, char_inputs,
char_seq_length, char_recover, dict_inputs, mask, batch_bert):
"""
word_inputs: (batch_size,seq_len)
word_seq_length:()
"""
batch_size = word_inputs.size(0)
seq_len = word_inputs.size(1)
word_emb = self.word_embedding(word_inputs)
if self.args.use_elmo:
elmo_emb = self.elmo_embedding(word_inputs)
# if self.args.use_bert:
# word_emb = torch.cat((word_emb,torch.squeeze(batch_bert,2)),2)
#elmo_emb = self.drop(elmo_emb)
# word_rep = word_emb
if self.args.use_char:
size = char_inputs.size(0)
char_emb = self.char_embedding(char_inputs)
char_emb = pack(char_emb,
char_seq_length.cpu().numpy(),
batch_first=True)
char_lstm_out, char_hidden = self.char_feature(char_emb)
char_lstm_out = pad(char_lstm_out, batch_first=True)
char_hidden = char_hidden[0].transpose(1, 0).contiguous().view(
size, -1)
char_hidden = char_hidden[char_recover]
char_hidden = char_hidden.view(batch_size, seq_len, -1)
if self.args.attention:
word_rep = F.tanh(
self.attn1(word_emb) + self.attn2(char_hidden))
z = F.sigmoid(self.attn3(word_rep))
x = 1 - z
word_rep = F.mul(z, word_emb) + F.mul(x, char_hidden)
else:
word_rep = torch.cat((word_emb, char_hidden), 2)
word_rep = self.word_drop(word_rep) #word represent dropout
#if self.args.use_elmo:
# word_rep = torch.cat((word_rep, elmo_emb), 2)
if self.args.feature:
for idx in range(self.feature_num):
word_rep = torch.cat(
(word_rep, self.feature_embeddings[idx](feat_inputs[idx])),
2)
# batch_bert = torch.split(batch_bert,1,dim=2)
# normed_weights = F.softmax(self.scalar_parameters, dim=0)
# y = self.gamma * sum(weight * tensor.squeeze(2) for weight, tensor in zip(normed_weights,batch_bert))
# x = F.softmax(torch.mean(batch_bert,dim=2))
x = F.softmax(torch.mean(batch_bert, dim=2))
if self.args.use_bert:
word_rep = torch.cat((word_rep, x), 2)
word_rep = pack(word_rep,
word_seq_length.cpu().numpy(),
batch_first=True)
out, hidden = self.word_feature(word_rep)
out, _ = pad(out, batch_first=True)
if self.args.use_elmo:
out = torch.cat((out, elmo_emb), 2)
if self.args.out_dict:
dict_rep = pack(dict_inputs,
word_seq_length.cpu().numpy(),
batch_first=True)
dict_out, hidden = self.dict_feature(dict_rep)
dict_out, _ = pad(dict_out, batch_first=True)
#dict_out = self.dict_fc(dict_inputs)
out = torch.cat((out, dict_out), 2)
if self.args.lstm_attention:
out_list, weight_list = [], []
for idx in range(seq_len):
# slice_out = out[:,0:idx+1,:]
if idx + 2 > seq_len:
slice_out = out
else:
slice_out = out[:, 0:idx + 2, :]
# slice_out = out
slice_out, weights = self.attention(slice_out)
# slice_out, weights = SelfAttention(self.args.hidden_dim*2).forward(slice_out)
out_list.append(slice_out.unsqueeze(1))
weight_list.append(weights)
out = torch.cat(out_list, dim=1)
out = self.drop(out)
out = self.hidden2tag(out)
return out
def forward(self,
CNN_output,
gaz_input,
gaz_input_back,
global_matrix,
exper_input=None,
gaz_mask=None):
batch_size = global_matrix.size(0)
seq_len = global_matrix.size(1)
index = self.index.unsqueeze(1).repeat(1, seq_len, 1) #(4,l,4h)
index = index.view(1, -1,
self.hidden_dim * 4).repeat(batch_size, 1,
1) #(b,4l,4h)
index2 = self.index2.unsqueeze(1).repeat(1, seq_len, 1) #(4,l,3h)
index2 = index2.view(1, -1,
self.hidden_dim * 3).repeat(batch_size, 1,
1) #(b,4l,3h)
if self.use_gaz:
gaz_input = torch.cat(
[gaz_input, gaz_input_back, gaz_input, gaz_input_back],
dim=1) #(b,4l,i)
seq_len_cat = seq_len * 4
global_matrix = global_matrix.repeat(1, 4, 1) #(b,4l,h)
if exper_input is not None:
exper_input = exper_input.repeat(1, 4, 1) #(b,4l,h)
if self.use_gaz:
cat_input = torch.cat([CNN_output, gaz_input, global_matrix],
dim=2) #(b,4l,2*h+gaz_dim)
else:
cat_input = torch.cat([CNN_output, global_matrix],
dim=2) #(b,4l,2*h+gaz_dim)
cat_gates_ = self.cat2gates(cat_input) #(b,4l,4h*4)
cat_gates = torch.gather(cat_gates_, dim=-1, index=index) #(b,4l,4h)
new_state = torch.tanh(cat_gates[:, :, :self.hidden_dim]) #(b,4l,h)
gates = cat_gates[:, :, self.hidden_dim:] #(b,4l,3h)
if exper_input is not None:
exper_gates_ = self.exper2gates(exper_input) #(b,l,3h)
exper_gates = torch.gather(exper_gates_, dim=-1, index=index2)
gates = gates + exper_gates #(b,4l,3h)
state_cat = torch.cat([
new_state.unsqueeze(2),
CNN_output.unsqueeze(2),
exper_input.unsqueeze(2)
],
dim=2)
else:
gates = gates[:, :, :self.hidden_dim * 2] #(b,l,2h)
state_cat = torch.cat(
[new_state.unsqueeze(2),
CNN_output.unsqueeze(2)], dim=2)
gates = torch.sigmoid(gates)
gates = F.softmax(gates.view(batch_size, seq_len_cat, -1,
self.hidden_dim),
dim=2)
layer_output = torch.sum(F.mul(gates, state_cat), dim=2) #(b,4l,h)
output = torch.split(layer_output, seq_len, dim=1)
return output
def forward_turn(self, batch, slots2values, hidden):
"""
# :param x_user: shape (batch_size, user_embeddings_dim)
# :param x_action: shape (batch_size, action_embeddings_dim)
# :param x_sys: shape (batch_size, sys_embeddings_dim)
:param batch:
:param hidden: shape (batch_size, 1, hidden_dim)
:param slots2values: dict mapping slots to values to be tested
# :param labels: dict mapping slots to one-hot ground truth value
representations
:return: tuple (loss, probs, hidden), with `loss' being the overall
loss across slots, `probs' a dict mapping slots to probability
distributions over values, `hidden' the new hidden state
"""
batch_size = len(batch)
probs = defaultdict(list)
binary_filling_probs = {}
# user input encoding [batch_size, hidden_dim]
all_utt = [
torch.stack([turn.x_utt[k] for turn in batch])
for k in range(len(batch[0].x_utt))
]
fu = self.utt_enc(all_utt)
# system act input encoding [batch_size, hidden_dim]
all_act = torch.stack([turn.x_act for turn in batch])
fa = self.action_encoder(all_act)
if self.args.encode_sys_utt:
fy = self.utt_enc(torch.Tensor([turn.x_sys for turn in batch]))
f_turn_inputs = torch.cat((fu, fa, fy), 1)
else:
f_turn_inputs = torch.cat((fu, fa), 1)
# turn encodings [batch_size, hidden_dim]
# and RNN hidden state [batch_size, hidden_dim_rnn]
f_turn, hidden = self.turn_history_rnn(f_turn_inputs, hidden)
# keep track of number of loss updates for later averaging
loss_updates = torch.Tensor([0]).to(self.device)
# iterate over slots and values, compute probabilities
for slot_id in sorted(slots2values.keys()):
slot = slots2values[slot_id]
# compute encoding of inputs as described in StateNet paper, Sec. 2
fs = self.slot_encoder(slot.embedding)
# encoding of slot with turns in batch: [batch_size, hidden_dim]
f_slot_turn = F.mul(fs, f_turn)
# get binary prediction for slot presence {slot_id: [batch_size, 1]}
binary_filling_probs[slot_id] = torch.sigmoid(
self.slot_fill_indicator(f_slot_turn))
# get probability distribution over values...
values = slot.values
for t, turn in enumerate(batch):
probs[slot_id].append(None)
if binary_filling_probs[slot_id][t] > 0.5:
probs[slot_id][t] = torch.zeros(len(values))
for v, value in enumerate(values):
venc = self.value_encoder(value.embedding)
# by computing 2-Norm distance following paper, Sec. 2.6
probs[slot_id][t][v] = -torch.dist(f_slot_turn, venc)
# softmax it!
probs[slot_id][t] = F.softmax(probs[slot_id][t], 0)
loss = torch.Tensor([0]).to(self.device)
if self.training:
for slot_id in slots2values.keys():
# loss for binary slot presence
# gold: 1 if slot in turn.labels (meaning it's filled), else 0
# [batch_size, 1]
if binary_filling_probs[slot_id] is not None:
gold_slot_filling = torch.Tensor([
float(slot_id in turn.labels) for turn in batch
]).view(-1, 1).to(self.device)
loss += self.args.eta * F.binary_cross_entropy(
binary_filling_probs[slot_id], gold_slot_filling).to(
self.device)
loss_updates += 1
for t, turn in enumerate(batch):
# loss for slot-value pairing, if slot is present
if slot_id in turn.labels and \
binary_filling_probs[slot_id][t] > 0.5:
loss += F.binary_cross_entropy(
probs[slot_id][t],
turn.labels[slot_id]).to(self.device)
loss_updates += 1
loss = loss / loss_updates
mean_slots_filled = len(probs) / len(slots2values)
return loss, probs, hidden, mean_slots_filled
def _region_classification(self, fc7_roi, fc7_context, fc7_frame):
#cls_score = self.cls_score_net(fc7)
#det_score = self.det_score_net(fc7)
alpha = cfg.TRAIN.MIL_RECURRECT_WEIGHT
refine_score_1 = self.refine_net_1(fc7_roi)
refine_score_2 = self.refine_net_2(fc7_roi)
#refine_score_3 = self.refine_net_3(fc7)
#refine_prob_1 = F.softmax(refine_score_1, dim=1) #num x class_num+1
#refine_prob_2 = F.softmax(refine_score_2, dim=1) #num x class_num+1
#refine_prob_3 = F.softmax(refine_score_3, dim=1) #num x class_num+1
#recurrent_refine = torch.max(refine_prob_2, dim=1)[0]
#recurrent_refine = recurrent_refine.unsqueeze(1).expand_as(fc7)
cls_score = self.cls_score_net(fc7_roi)
context_score = self.det_score_net(fc7_context)
frame_score = self.det_score_net(fc7_frame)
det_score = frame_score - context_score
cls_prob = F.softmax(cls_score, dim=1) #num x class_num
det_prob = F.softmax(det_score, dim=0) #num x class_num
refine_prob_1 = F.softmax(refine_score_1, dim=1) #num x class_num+1
refine_prob_2 = F.softmax(refine_score_2, dim=1) #num x class_num+1
#refine_prob_3 = F.softmax(refine_score_3, dim=1) #num x class_num+1
det_cls_prob_product = F.mul(cls_score, det_prob) #num x class_num
det_cls_prob = torch.sum(det_cls_prob_product, 0) #1 x class_num or just a one dim vector whose size is class_num
# bbox_pred = self.bbox_pred_net(fc7)
bbox_pred = torch.zeros(cls_prob.shape[0], 80)
cls_pred = torch.max(cls_score, 1)[1]
#print('cls_score ', cls_score.shape)
#print('cls_pred ', cls_pred.shape)
#print('cls_prob', cls_prob.shape)
#print('cls_score ', cls_score)
#print('cls_pred ', cls_pred)
#print('cls_prob ', cls_prob)
#print('det_prob ', det_prob)
#print('det_cls_prob_product ', det_cls_prob_product)
#print('det_cls_prob ', det_cls_prob)
self._predictions["cls_score"] = cls_score
self._predictions['det_score'] = det_score
self._predictions["cls_prob"] = cls_prob
self._predictions["det_prob"] = det_prob
self._predictions['refine_prob_1'] = refine_prob_1
self._predictions['refine_prob_2'] = refine_prob_2
#self._predictions['refine_prob_3'] = refine_prob_3
self._predictions["cls_pred"] = cls_pred
self._predictions["bbox_pred"] = bbox_pred
self._predictions['det_cls_prob_product'] = det_cls_prob_product
self._predictions['det_cls_prob'] = det_cls_prob
return cls_prob, det_prob, bbox_pred, det_cls_prob_product, det_cls_prob
def forward(self, input, hx):
return self.activation(
F.linear(input, self.weight_ih, self.bias_ih) +
F.mul(self.weight_hh, hx))
def forward(self, input, hx=None):
# out = tanh(w_{ih} * x + b_{ih} + w_{hh} (*) h)
# (*) Hammard Product
return self.activation(
F.linear(input, self.weight_ih, self.bias_ih) +
F.mul(self.weight_hh, hx))
def forward(self, input):
return F.mul(input, self.weight).sum(dim=1) + self.bias
def forward(
self, input
): # input: (history_window, channels=n_road, in_features=1+status_hop)
return F.mul(input, self.weight).sum(dim=2) + self.bias
def IndRNNTanhCell(input, hidden, w_ih, w_hh, b_ih=None):
hy = F.tanh(F.linear(input, w_ih, b_ih) + F.mul(w_hh, hidden))
return hy
def IndRNNReLuCell(input, hidden, w_ih, w_hh, b_ih=None):
hy = F.relu(F.linear(input, w_ih, b_ih) + F.mul(w_hh, hidden))
return hy
def forward(self, forest, embeds):
child_c = child_h = None
if self.add_cuda:
forest_loss = autograd.Variable(torch.zeros(1).cuda())
else:
forest_loss = autograd.Variable(torch.zeros(1))
for level in range(forest.max_level + 1)[::-1]:
nodes = [node for node in forest.node_list if node.level == level]
nlen = len(nodes)
input_ix = []
chi_par = {}
max_childs, row = 0, 0
offset_pos, fx_offset, hc_offset, fc_offset = [], [], [], []
for idx, node in enumerate(nodes):
input_ix.append(node.forest_ix)
childs = []
if len(node.children) > max_childs:
max_childs = len(node.children)
for ch_ix, child in enumerate(node.children):
childs.append(ch_ix)
chi_par[idx] = childs
if child_h is None: # if no child nodes
max_childs = 1
# offset_pos = [0 for i in range(nlen)]
for key, val in chi_par.items():
if len(val) > 0:
for v in val:
offset_pos.append(key * max_childs + v)
fx_offset.append(key)
hc_offset.append(row)
row += 1
fc_offset.append(key * max_childs + v)
else:
row += 1
fx_offset.append(key)
fc_offset.append(key * max_childs)
fx_len = len(fx_offset)
# node_num = len(input_ix)
if self.add_cuda:
if child_h is None:
child_h = autograd.Variable(
torch.zeros(nlen, self.hidden_dim).cuda())
child_c = autograd.Variable(
torch.zeros(nlen, self.hidden_dim).cuda())
child_h_sum = autograd.Variable(
torch.zeros(nlen * max_childs, self.hidden_dim).cuda())
child_fh = autograd.Variable(
torch.zeros(fx_len, self.hidden_dim).cuda())
child_fc = autograd.Variable(
torch.zeros(fx_len, self.hidden_dim).cuda())
child_fc_sum = autograd.Variable(
torch.zeros(nlen * max_childs, self.hidden_dim).cuda())
select_indices = autograd.Variable(
torch.LongTensor(input_ix).cuda())
offset_pos = autograd.Variable(
torch.LongTensor(offset_pos).cuda())
fx_offset = autograd.Variable(
torch.LongTensor(fx_offset).cuda())
hc_offset = autograd.Variable(
torch.LongTensor(hc_offset).cuda())
fc_offset = autograd.Variable(
torch.LongTensor(fc_offset).cuda())
else:
if child_h is None:
child_h = autograd.Variable(
torch.zeros(nlen, self.hidden_dim))
child_c = autograd.Variable(
torch.zeros(nlen, self.hidden_dim))
child_h_sum = autograd.Variable(
torch.zeros(nlen * max_childs, self.hidden_dim))
child_fh = autograd.Variable(
torch.zeros(fx_len, self.hidden_dim))
child_fc = autograd.Variable(
torch.zeros(fx_len, self.hidden_dim))
child_fc_sum = autograd.Variable(
torch.zeros(nlen * max_childs, self.hidden_dim))
offset_pos = autograd.Variable(torch.LongTensor(offset_pos))
select_indices = autograd.Variable(torch.LongTensor(input_ix))
fx_offset = autograd.Variable(torch.LongTensor(fx_offset))
hc_offset = autograd.Variable(torch.LongTensor(hc_offset))
fc_offset = autograd.Variable(torch.LongTensor(fc_offset))
# embed_input = torch.index_select(embeds, 0, select_indices)
# test_start = time.time()
embed_input = embeds[select_indices]
fh = self.fh(child_h)
if len(offset_pos) > 0:
child_h_sum.index_copy_(0, offset_pos, child_h)
child_fh.index_copy_(0, hc_offset, fh)
child_fc.index_copy_(0, hc_offset, child_c)
f = F.sigmoid(self.fx(embed_input[fx_offset]) + child_fh)
fc = F.mul(f, child_fc)
child_fc_sum.index_copy_(0, fc_offset, fc)
child_h_sum = child_h_sum.view([nlen, max_childs, self.hidden_dim])
child_fc_sum = child_fc_sum.view(
[nlen, max_childs, self.hidden_dim])
child_c, child_h = self.batch_forward(embed_input,
torch.sum(child_h_sum, 1),
torch.sum(child_fc_sum, 1))
out = self.out(self.dropout(child_h))
out = torch.unsqueeze(out, 1)
# test_start = time.time()
for idx, node in enumerate(nodes):
node.dt_state = torch.unsqueeze(child_c[idx],
0), torch.unsqueeze(
child_h[idx], 0)
# if node.label is not None:
# if self.add_cuda:
# node_gold = autograd.Variable(torch.LongTensor([node.label]).cuda())
# else:
# node_gold = autograd.Variable(torch.LongTensor([node.label]))
# # out_list.append(out[idx])
# # gold_list.append(node_gold)
# forest_loss += self.loss_func(out[idx], node_gold)
# print("test time:",time.time()-test_start)
if level == 0:
# forest_loss = self.loss_func(torch.cat(out_list),torch.cat(gold_list))
# return torch.squeeze(out, 1), forest_loss
return torch.squeeze(out, 1)
def _add_losses(self, sigma_rpn=3.0):
## RPN, class loss
#rpn_cls_score = self._predictions['rpn_cls_score_reshape'].view(-1, 2)
#rpn_label = self._anchor_targets['rpn_labels'].view(-1)
#rpn_select = (rpn_label.data != -1).nonzero().view(-1)
#rpn_cls_score = rpn_cls_score.index_select(0, rpn_select).contiguous().view(-1, 2)
#rpn_label = rpn_label.index_select(0, rpn_select).contiguous().view(-1)
#rpn_cross_entropy = F.cross_entropy(rpn_cls_score, rpn_label)
## RPN, bbox loss
#rpn_bbox_pred = self._predictions['rpn_bbox_pred']
#rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets']
#rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights']
#rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights']
#rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights,
# rpn_bbox_outside_weights, sigma=sigma_rpn, dim=[1, 2, 3])
# RCNN, class loss
# Done in 2018/11/19
#cls_score = self._predictions["cls_score"]
#label = self._proposal_targets["labels"].view(-1)
#cross_entropy = F.cross_entropy(cls_score.view(-1, self._num_classes), label)
# RCNN, bbox loss
#bbox_pred = self._predictions['bbox_pred']
#bbox_targets = self._proposal_targets['bbox_targets']
#bbox_inside_weights = self._proposal_targets['bbox_inside_weights']
#bbox_outside_weights = self._proposal_targets['bbox_outside_weights']
#loss_box = self._smooth_l1_loss(bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights)
#self._losses['cross_entropy'] = cross_entropy
#self._losses['loss_box'] = loss_box
self._losses['cross_entropy'] = np.zeros(1)
self._losses['loss_box'] = np.zeros(1)
self._losses['rpn_cross_entropy'] = np.zeros(1)
self._losses['rpn_loss_box'] = np.zeros(1)
det_cls_prob = self._predictions['det_cls_prob']
det_cls_prob = det_cls_prob.view(-1)
label = self._image_level_label.view(-1)
det_cls_product = self._predictions['det_cls_prob_product']
refine_prob_1 = self._predictions['refine_prob_1']
refine_prob_2 = self._predictions['refine_prob_2']
#refine_prob_3 = self._predictions['refine_prob_3']
#caculating the loss of the first branch
roi_labels, roi_weights ,keep_inds = self.get_refine_supervision(det_cls_product, self._image_gt_summaries['ss_boxes'],
self._image_gt_summaries['image_level_label'])
roi_weights = torch.tensor(roi_weights).cuda()
roi_labels = torch.tensor(roi_labels, dtype=roi_weights.dtype).cuda()
#roi_labels = torch.mul(roi_labels, roi_weights)
refine_loss_1 = - torch.sum(torch.mul(roi_labels, torch.log(refine_prob_1[keep_inds]))) / roi_labels.shape[0]
#caculating the loss of the second branch
roi_labels, roi_weights, keep_inds = self.get_refine_supervision(refine_prob_1, self._image_gt_summaries['ss_boxes'],
self._image_gt_summaries['image_level_label'])
roi_weights = torch.tensor(roi_weights).cuda()
roi_labels = torch.tensor(roi_labels, dtype=roi_weights.dtype).cuda()
#roi_labels = torch.mul(roi_labels, roi_weights)
refine_loss_2 = - torch.sum(torch.mul(roi_labels, torch.log(refine_prob_2[keep_inds]))) / roi_labels.shape[0]
#roi_labels, roi_weights, keep_inds = self.get_refine_supervision(refine_prob_2, self._image_gt_summaries['ss_boxes'],
# self._image_gt_summaries['image_level_label'])
#roi_weights = torch.tensor(roi_weights).cuda()
#roi_labels = torch.tensor(roi_labels, dtype=roi_weights.dtype).cuda()
#roi_labels = torch.mul(roi_labels, roi_weights)
#refine_loss_3 = - torch.sum(torch.mul(roi_labels, torch.log(refine_prob_3[keep_inds]))) / roi_labels.shape[0]
self._losses['refine_loss_1'] = refine_loss_1
self._losses['refine_loss_2'] = refine_loss_2
#self._losses['refine_loss_3'] = refine_loss_3
#print('label ', label)
label = torch.tensor(label, dtype=det_cls_prob.dtype, device=det_cls_prob.device)
zeros = torch.zeros(det_cls_prob.shape, dtype=det_cls_prob.dtype, device=det_cls_prob.device)
max_zeros = torch.max(zeros, 1-F.mul(label, det_cls_prob))
cls_det_loss = torch.sum(max_zeros)
self._losses['cls_det_loss'] = cls_det_loss / 20
#for WSDNN or MultiBranch Learning
#cls_det_loss = - torch.sum(torch.log(det_cls_prob[label==1])) - torch.sum(torch.log(1 - det_cls_prob[label == -1]))
#self._losses['cls_det_loss'] = cls_det_loss / 20
#loss = cross_entropy + loss_box + rpn_cross_entropy + rpn_loss_box
loss = cls_det_loss / 20 + refine_loss_1*0.1 + refine_loss_2*0.1
self._losses['total_loss'] = loss
#print('loss ', loss)
for k in self._losses.keys():
self._event_summaries[k] = self._losses[k]
return loss
def forward(self, x):
return F.mul(x, self.scale)
