def batch_forward(self, inputs, child_h_sum, child_fc_sum): child_h_sum = torch.squeeze(child_h_sum, 1) child_fc_sum = torch.squeeze(child_fc_sum, 1) i = F.sigmoid(self.ix(inputs) + self.ih(child_h_sum)) o = F.sigmoid(self.ox(inputs) + self.oh(child_h_sum)) u = F.tanh(self.ux(inputs) + self.uh(child_h_sum)) c = F.mul(i, u) + child_fc_sum h = F.mul(o, F.tanh(c)) return c, h
def forward(self, rgb_inp, flow_inp): x_rgb = self.rgb_rnn(rgb_inp)[0] x_flow = self.rgb_rnn(flow_inp)[0] #fused_outputs = torch.cat([x_rgb, x_flow], dim=2) alpha = F.sigmoid(self.w_att(x_rgb) + self.w_att(x_flow)) fused_outputs = F.mul(alpha, x_rgb) + F.mul((1-alpha), x_flow) outputs = self.out(fused_outputs) outputs_mean = Variable(torch.zeros(outputs.size()[0], num_classes)).cuda(DEVICE) for i in range(outputs.size()[0]): outputs_mean[i] = outputs[i].mean(dim=0) return outputs_mean
def forward(self, x, hidden): hx, cx = hidden x = x.view(-1, x.size(1)) gates = self.x2h(x) + self.h2h(hx) gates = gates.squeeze() # 将gates按照通道数dim=1也就是4*hidden_size切分为四个 ingate, forgetgate, cellgate, outgate = gates.chunk(4, 1) ingate = F.sigmoid(ingate) forgetgate = F.sigmoid(forgetgate) cellgate = F.tanh(cellgate) outgate = F.sigmoid(outgate) cy = F.mul(cx, forgetgate) + F.mul(ingate, cellgate) hy = F.mul(outgate, F.tanh(cy)) return (hy, cy)
def GeneralizedDice(probs, onehot_labels): ''' :param probs: b2wh, the probs of last CNN layer input to F.log_softmax() :param onehot_labels: one-hot operation labels :return: ''' #assert utils.checkSimplex_(probs) and utils.checkSimplex_(onehot_labels) idc = [0, 1] #pc = probs[:, idc, ...].type(torch.float32) #pc: bcwh pc = probs.type(torch.float32) #tc = onehot_labels[:, idc, ...].type(torch.float32) tc = onehot_labels #convert ndarray to Tensor pc_ = torch.zeros(pc.shape[0], pc.shape[1]).cuda() # bc tc_ = torch.zeros(tc.shape[0], tc.shape[1]).cuda() #bc ## intersection = torch.einsum('bcwh, bcwh -> bc', [pc, tc]) temp = F.mul(tc, pc) #bcwh intersection_ = torch.zeros(pc.shape[0], pc.shape[1]) ## bellow operation equals as ## pc_= torch.einsum('bcwh', [pc]) tc_ = torch.einsum('bcwh -> bc', [tc]) for vi in range(pc.shape[0]): for vj in range(pc.shape[1]): pc_[vi][vj] = torch.sum(pc[vi, vj, ...]) tc_[vi][vj] = torch.sum(tc[vi, vj, :, :]) intersection_ = torch.sum( temp[vi, vj, :, :]) #intersection of pre mask and GT mask w = 1 / ((pc_.type(torch.float32) + 1e-10)**2) intersection = w * intersection_ union = w * (pc_ + tc_) # union of pre mask and GT mask divided = 1 - 2 * (torch.sum(intersection, 1) + 1e-10) / (torch.sum(union, 1) + 1e-10) #loss = divided.mean() loss = torch.mean(divided) return loss
def repar(self, mu, logvar): # the infamous reparamaterization trick (aka 4 lines of code) samp = self.sample() samp = F.mul((0.5 * logvar).exp(), samp) samp = samp + mu return samp
def __call__(self, tensor): """ Args: tensor (Tensor): Tensor image of size (C, H, W) to be normalized. Returns: Tensor: Normalized Tensor image. """ tensor = F.mul(tensor, self.factor) return tensor
def forward(self, data_in): out1 = self.dilated_conv_tanh(data_in) out2 = self.dilated_conv_sigmoid(data_in) tanh_out = torch.tanh(out1) sigm_out = torch.sigmoid(out2) data = F.mul(tanh_out, sigm_out) skip = self.skip(data) res = self.res(data) + data_in return res, skip
def forward(self, x): x_mean = torch.mean(torch.abs(x)) x = BinaryFunc.apply(x) if self.is_inference: x = self.layer(x) else: weight_full = self.layer.weight.data.clone() self.layer.weight.data.sign_() x = self.layer(x) self.layer.weight.data = weight_full x = F.mul(x, x_mean) return x
def forward(self, x): self.clip_recurrent_kernel() output = self.non_linearity( F.linear(x, self.input_kernel, self.bias) + F.mul(self.recurrent_kernel, self.h)) # self.clip_recurrent_kernel() # recurrent_update = self.h.mul( # self.recurrent_kernel.expand_as(self.h)) # gate_inputs += recurrent_update.expand_as(gate_inputs) # gate_inputs += self.bias.expand_as(gate_inputs) # output = self.non_linearity(gate_inputs) return output
def SurfaceLoss(probs, dis_map): #assert utils.checkSimplex_(probs) assert not utils.one_hot(dis_map) #if false throw exception e idc = [1] #pc = probs[:, idc, ...].type(torch.float32) #bcwh pc = probs[:, 1, :].type(torch.float32) dc = dis_map[:, 1, ...] #bcwh multipled = F.mul( pc, dc) #bcwh, equal to 'torch.einsum('bcwh, bcwh -> bcwh', [pc, dc])' #loss = multipled.mean() loss = torch.mean(multipled) return loss
def forward(self, x, z): x = self.alexnet(x) y = F.relu(self.Conv1(x)) y = F.relu(self.Conv2(y)) y = F.relu(self.Conv3(y)) x = F.dropout(F.relu(F.mul(x, y)), 0.5) x = x.view(x.size(0), -1) x = self.layer1(x) x = self.layer2(x) x = self.fc(x) return x
def forward(self, rgb_inputs, flow_inputs, lengths): rgb_inputs = pad_sequence(rgb_inputs, batch_first=False) rgb_inputs = pack_padded_sequence(rgb_inputs, lengths, batch_first=False) rgb_inputs = rgb_inputs.cuda() rgb_outputs, rgb_hiddens = self.indrnn(rgb_inputs) rgb_outputs, _ = pad_packed_sequence(rgb_outputs, batch_first=False) flow_inputs = pad_sequence(flow_inputs, batch_first=False) flow_inputs = pack_padded_sequence(flow_inputs, lengths, batch_first=False) flow_inputs = flow_inputs.cuda() flow_outputs, flow_hiddens = self.indrnn(flow_inputs) flow_outputs, _ = pad_packed_sequence(flow_outputs, batch_first=False) alpha = F.sigmoid(self.w_att(rgb_outputs) + self.w_att(flow_outputs)) fused_outputs = F.mul(alpha, rgb_outputs) + F.mul( (1 - alpha), flow_outputs) outputs = self.out(fused_outputs) #outputs = outputs.view(-1, num_classes) return outputs
def forward(self, word_inputs, word_seq_length, char_inputs, char_seq_length, char_recover): """ word_inputs: (batch_size,seq_len) word_seq_length:() """ batch_size = word_inputs.size(0) seq_len = word_inputs.size(1) word_emb = self.word_embedding(word_inputs) # word_rep = self.drop(word_emb) word_rep = word_emb if self.args.use_char: size = char_inputs.size(0) char_emb = self.char_embedding(char_inputs) char_emb = pack(char_emb, char_seq_length.numpy(), batch_first=True) char_lstm_out, char_hidden = self.char_feature(char_emb) char_lstm_out = pad(char_lstm_out, batch_first=True) char_hidden = char_hidden[0].transpose(1, 0).contiguous().view( size, -1) char_hidden = char_hidden[char_recover] char_hidden = char_hidden.view(batch_size, seq_len, -1) if self.args.attention: word_rep = F.tanh( self.attn1(word_emb) + self.attn2(char_hidden)) z = F.sigmoid(self.attn3(word_rep)) x = 1 - z word_rep = F.mul(z, word_emb) + F.mul(x, char_hidden) else: word_rep = torch.cat((word_emb, char_hidden), 2) word_rep = pack(word_rep, word_seq_length.cpu().numpy(), batch_first=True) out, hidden = self.word_feature(word_rep) out, _ = pad(out, batch_first=True) if self.args.lstm_attention: # tanh_out = F.tanh(out) # att_vec = self.att1(tanh_out) # att_sm_vec = F.softmax(att_vec) # out = F.mul(out,att_sm_vec) out_list, weight_list = [], [] for idx in range(seq_len): # slice_out = out[:,0:idx+1,:] if idx + 2 > seq_len: slice_out = out else: slice_out = out[:, 0:idx + 2, :] # slice_out = out slice_out, weights = self.attention(slice_out) # slice_out, weights = SelfAttention(self.args.hidden_dim*2).forward(slice_out) out_list.append(slice_out.unsqueeze(1)) weight_list.append(weights) out = torch.cat(out_list, dim=1) # pass # out = F.tanh(self.att1(out)) # out = self.softmax(out) # out = out*out # out = self.att2(out) # else: out = self.hidden2tag(self.drop(out)) return out
def forward(self, word_inputs, feat_inputs, word_seq_length, char_inputs, char_seq_length, char_recover, dict_inputs, mask, batch_bert): """ word_inputs: (batch_size,seq_len) word_seq_length:() """ batch_size = word_inputs.size(0) seq_len = word_inputs.size(1) word_emb = self.word_embedding(word_inputs) if self.args.use_elmo: elmo_emb = self.elmo_embedding(word_inputs) # if self.args.use_bert: # word_emb = torch.cat((word_emb,torch.squeeze(batch_bert,2)),2) #elmo_emb = self.drop(elmo_emb) # word_rep = word_emb if self.args.use_char: size = char_inputs.size(0) char_emb = self.char_embedding(char_inputs) char_emb = pack(char_emb, char_seq_length.cpu().numpy(), batch_first=True) char_lstm_out, char_hidden = self.char_feature(char_emb) char_lstm_out = pad(char_lstm_out, batch_first=True) char_hidden = char_hidden[0].transpose(1, 0).contiguous().view( size, -1) char_hidden = char_hidden[char_recover] char_hidden = char_hidden.view(batch_size, seq_len, -1) if self.args.attention: word_rep = F.tanh( self.attn1(word_emb) + self.attn2(char_hidden)) z = F.sigmoid(self.attn3(word_rep)) x = 1 - z word_rep = F.mul(z, word_emb) + F.mul(x, char_hidden) else: word_rep = torch.cat((word_emb, char_hidden), 2) word_rep = self.word_drop(word_rep) #word represent dropout #if self.args.use_elmo: # word_rep = torch.cat((word_rep, elmo_emb), 2) if self.args.feature: for idx in range(self.feature_num): word_rep = torch.cat( (word_rep, self.feature_embeddings[idx](feat_inputs[idx])), 2) # batch_bert = torch.split(batch_bert,1,dim=2) # normed_weights = F.softmax(self.scalar_parameters, dim=0) # y = self.gamma * sum(weight * tensor.squeeze(2) for weight, tensor in zip(normed_weights,batch_bert)) # x = F.softmax(torch.mean(batch_bert,dim=2)) x = F.softmax(torch.mean(batch_bert, dim=2)) if self.args.use_bert: word_rep = torch.cat((word_rep, x), 2) word_rep = pack(word_rep, word_seq_length.cpu().numpy(), batch_first=True) out, hidden = self.word_feature(word_rep) out, _ = pad(out, batch_first=True) if self.args.use_elmo: out = torch.cat((out, elmo_emb), 2) if self.args.out_dict: dict_rep = pack(dict_inputs, word_seq_length.cpu().numpy(), batch_first=True) dict_out, hidden = self.dict_feature(dict_rep) dict_out, _ = pad(dict_out, batch_first=True) #dict_out = self.dict_fc(dict_inputs) out = torch.cat((out, dict_out), 2) if self.args.lstm_attention: out_list, weight_list = [], [] for idx in range(seq_len): # slice_out = out[:,0:idx+1,:] if idx + 2 > seq_len: slice_out = out else: slice_out = out[:, 0:idx + 2, :] # slice_out = out slice_out, weights = self.attention(slice_out) # slice_out, weights = SelfAttention(self.args.hidden_dim*2).forward(slice_out) out_list.append(slice_out.unsqueeze(1)) weight_list.append(weights) out = torch.cat(out_list, dim=1) out = self.drop(out) out = self.hidden2tag(out) return out
def forward(self, CNN_output, gaz_input, gaz_input_back, global_matrix, exper_input=None, gaz_mask=None): batch_size = global_matrix.size(0) seq_len = global_matrix.size(1) index = self.index.unsqueeze(1).repeat(1, seq_len, 1) #(4,l,4h) index = index.view(1, -1, self.hidden_dim * 4).repeat(batch_size, 1, 1) #(b,4l,4h) index2 = self.index2.unsqueeze(1).repeat(1, seq_len, 1) #(4,l,3h) index2 = index2.view(1, -1, self.hidden_dim * 3).repeat(batch_size, 1, 1) #(b,4l,3h) if self.use_gaz: gaz_input = torch.cat( [gaz_input, gaz_input_back, gaz_input, gaz_input_back], dim=1) #(b,4l,i) seq_len_cat = seq_len * 4 global_matrix = global_matrix.repeat(1, 4, 1) #(b,4l,h) if exper_input is not None: exper_input = exper_input.repeat(1, 4, 1) #(b,4l,h) if self.use_gaz: cat_input = torch.cat([CNN_output, gaz_input, global_matrix], dim=2) #(b,4l,2*h+gaz_dim) else: cat_input = torch.cat([CNN_output, global_matrix], dim=2) #(b,4l,2*h+gaz_dim) cat_gates_ = self.cat2gates(cat_input) #(b,4l,4h*4) cat_gates = torch.gather(cat_gates_, dim=-1, index=index) #(b,4l,4h) new_state = torch.tanh(cat_gates[:, :, :self.hidden_dim]) #(b,4l,h) gates = cat_gates[:, :, self.hidden_dim:] #(b,4l,3h) if exper_input is not None: exper_gates_ = self.exper2gates(exper_input) #(b,l,3h) exper_gates = torch.gather(exper_gates_, dim=-1, index=index2) gates = gates + exper_gates #(b,4l,3h) state_cat = torch.cat([ new_state.unsqueeze(2), CNN_output.unsqueeze(2), exper_input.unsqueeze(2) ], dim=2) else: gates = gates[:, :, :self.hidden_dim * 2] #(b,l,2h) state_cat = torch.cat( [new_state.unsqueeze(2), CNN_output.unsqueeze(2)], dim=2) gates = torch.sigmoid(gates) gates = F.softmax(gates.view(batch_size, seq_len_cat, -1, self.hidden_dim), dim=2) layer_output = torch.sum(F.mul(gates, state_cat), dim=2) #(b,4l,h) output = torch.split(layer_output, seq_len, dim=1) return output
def forward_turn(self, batch, slots2values, hidden): """ # :param x_user: shape (batch_size, user_embeddings_dim) # :param x_action: shape (batch_size, action_embeddings_dim) # :param x_sys: shape (batch_size, sys_embeddings_dim) :param batch: :param hidden: shape (batch_size, 1, hidden_dim) :param slots2values: dict mapping slots to values to be tested # :param labels: dict mapping slots to one-hot ground truth value representations :return: tuple (loss, probs, hidden), with `loss' being the overall loss across slots, `probs' a dict mapping slots to probability distributions over values, `hidden' the new hidden state """ batch_size = len(batch) probs = defaultdict(list) binary_filling_probs = {} # user input encoding [batch_size, hidden_dim] all_utt = [ torch.stack([turn.x_utt[k] for turn in batch]) for k in range(len(batch[0].x_utt)) ] fu = self.utt_enc(all_utt) # system act input encoding [batch_size, hidden_dim] all_act = torch.stack([turn.x_act for turn in batch]) fa = self.action_encoder(all_act) if self.args.encode_sys_utt: fy = self.utt_enc(torch.Tensor([turn.x_sys for turn in batch])) f_turn_inputs = torch.cat((fu, fa, fy), 1) else: f_turn_inputs = torch.cat((fu, fa), 1) # turn encodings [batch_size, hidden_dim] # and RNN hidden state [batch_size, hidden_dim_rnn] f_turn, hidden = self.turn_history_rnn(f_turn_inputs, hidden) # keep track of number of loss updates for later averaging loss_updates = torch.Tensor([0]).to(self.device) # iterate over slots and values, compute probabilities for slot_id in sorted(slots2values.keys()): slot = slots2values[slot_id] # compute encoding of inputs as described in StateNet paper, Sec. 2 fs = self.slot_encoder(slot.embedding) # encoding of slot with turns in batch: [batch_size, hidden_dim] f_slot_turn = F.mul(fs, f_turn) # get binary prediction for slot presence {slot_id: [batch_size, 1]} binary_filling_probs[slot_id] = torch.sigmoid( self.slot_fill_indicator(f_slot_turn)) # get probability distribution over values... values = slot.values for t, turn in enumerate(batch): probs[slot_id].append(None) if binary_filling_probs[slot_id][t] > 0.5: probs[slot_id][t] = torch.zeros(len(values)) for v, value in enumerate(values): venc = self.value_encoder(value.embedding) # by computing 2-Norm distance following paper, Sec. 2.6 probs[slot_id][t][v] = -torch.dist(f_slot_turn, venc) # softmax it! probs[slot_id][t] = F.softmax(probs[slot_id][t], 0) loss = torch.Tensor([0]).to(self.device) if self.training: for slot_id in slots2values.keys(): # loss for binary slot presence # gold: 1 if slot in turn.labels (meaning it's filled), else 0 # [batch_size, 1] if binary_filling_probs[slot_id] is not None: gold_slot_filling = torch.Tensor([ float(slot_id in turn.labels) for turn in batch ]).view(-1, 1).to(self.device) loss += self.args.eta * F.binary_cross_entropy( binary_filling_probs[slot_id], gold_slot_filling).to( self.device) loss_updates += 1 for t, turn in enumerate(batch): # loss for slot-value pairing, if slot is present if slot_id in turn.labels and \ binary_filling_probs[slot_id][t] > 0.5: loss += F.binary_cross_entropy( probs[slot_id][t], turn.labels[slot_id]).to(self.device) loss_updates += 1 loss = loss / loss_updates mean_slots_filled = len(probs) / len(slots2values) return loss, probs, hidden, mean_slots_filled
def _region_classification(self, fc7_roi, fc7_context, fc7_frame): #cls_score = self.cls_score_net(fc7) #det_score = self.det_score_net(fc7) alpha = cfg.TRAIN.MIL_RECURRECT_WEIGHT refine_score_1 = self.refine_net_1(fc7_roi) refine_score_2 = self.refine_net_2(fc7_roi) #refine_score_3 = self.refine_net_3(fc7) #refine_prob_1 = F.softmax(refine_score_1, dim=1) #num x class_num+1 #refine_prob_2 = F.softmax(refine_score_2, dim=1) #num x class_num+1 #refine_prob_3 = F.softmax(refine_score_3, dim=1) #num x class_num+1 #recurrent_refine = torch.max(refine_prob_2, dim=1)[0] #recurrent_refine = recurrent_refine.unsqueeze(1).expand_as(fc7) cls_score = self.cls_score_net(fc7_roi) context_score = self.det_score_net(fc7_context) frame_score = self.det_score_net(fc7_frame) det_score = frame_score - context_score cls_prob = F.softmax(cls_score, dim=1) #num x class_num det_prob = F.softmax(det_score, dim=0) #num x class_num refine_prob_1 = F.softmax(refine_score_1, dim=1) #num x class_num+1 refine_prob_2 = F.softmax(refine_score_2, dim=1) #num x class_num+1 #refine_prob_3 = F.softmax(refine_score_3, dim=1) #num x class_num+1 det_cls_prob_product = F.mul(cls_score, det_prob) #num x class_num det_cls_prob = torch.sum(det_cls_prob_product, 0) #1 x class_num or just a one dim vector whose size is class_num # bbox_pred = self.bbox_pred_net(fc7) bbox_pred = torch.zeros(cls_prob.shape[0], 80) cls_pred = torch.max(cls_score, 1)[1] #print('cls_score ', cls_score.shape) #print('cls_pred ', cls_pred.shape) #print('cls_prob', cls_prob.shape) #print('cls_score ', cls_score) #print('cls_pred ', cls_pred) #print('cls_prob ', cls_prob) #print('det_prob ', det_prob) #print('det_cls_prob_product ', det_cls_prob_product) #print('det_cls_prob ', det_cls_prob) self._predictions["cls_score"] = cls_score self._predictions['det_score'] = det_score self._predictions["cls_prob"] = cls_prob self._predictions["det_prob"] = det_prob self._predictions['refine_prob_1'] = refine_prob_1 self._predictions['refine_prob_2'] = refine_prob_2 #self._predictions['refine_prob_3'] = refine_prob_3 self._predictions["cls_pred"] = cls_pred self._predictions["bbox_pred"] = bbox_pred self._predictions['det_cls_prob_product'] = det_cls_prob_product self._predictions['det_cls_prob'] = det_cls_prob return cls_prob, det_prob, bbox_pred, det_cls_prob_product, det_cls_prob
def forward(self, input, hx): return self.activation( F.linear(input, self.weight_ih, self.bias_ih) + F.mul(self.weight_hh, hx))
def forward(self, input, hx=None): # out = tanh(w_{ih} * x + b_{ih} + w_{hh} (*) h) # (*) Hammard Product return self.activation( F.linear(input, self.weight_ih, self.bias_ih) + F.mul(self.weight_hh, hx))
def forward(self, input): return F.mul(input, self.weight).sum(dim=1) + self.bias
def forward( self, input ): # input: (history_window, channels=n_road, in_features=1+status_hop) return F.mul(input, self.weight).sum(dim=2) + self.bias
def IndRNNTanhCell(input, hidden, w_ih, w_hh, b_ih=None): hy = F.tanh(F.linear(input, w_ih, b_ih) + F.mul(w_hh, hidden)) return hy
def IndRNNReLuCell(input, hidden, w_ih, w_hh, b_ih=None): hy = F.relu(F.linear(input, w_ih, b_ih) + F.mul(w_hh, hidden)) return hy
def forward(self, forest, embeds): child_c = child_h = None if self.add_cuda: forest_loss = autograd.Variable(torch.zeros(1).cuda()) else: forest_loss = autograd.Variable(torch.zeros(1)) for level in range(forest.max_level + 1)[::-1]: nodes = [node for node in forest.node_list if node.level == level] nlen = len(nodes) input_ix = [] chi_par = {} max_childs, row = 0, 0 offset_pos, fx_offset, hc_offset, fc_offset = [], [], [], [] for idx, node in enumerate(nodes): input_ix.append(node.forest_ix) childs = [] if len(node.children) > max_childs: max_childs = len(node.children) for ch_ix, child in enumerate(node.children): childs.append(ch_ix) chi_par[idx] = childs if child_h is None: # if no child nodes max_childs = 1 # offset_pos = [0 for i in range(nlen)] for key, val in chi_par.items(): if len(val) > 0: for v in val: offset_pos.append(key * max_childs + v) fx_offset.append(key) hc_offset.append(row) row += 1 fc_offset.append(key * max_childs + v) else: row += 1 fx_offset.append(key) fc_offset.append(key * max_childs) fx_len = len(fx_offset) # node_num = len(input_ix) if self.add_cuda: if child_h is None: child_h = autograd.Variable( torch.zeros(nlen, self.hidden_dim).cuda()) child_c = autograd.Variable( torch.zeros(nlen, self.hidden_dim).cuda()) child_h_sum = autograd.Variable( torch.zeros(nlen * max_childs, self.hidden_dim).cuda()) child_fh = autograd.Variable( torch.zeros(fx_len, self.hidden_dim).cuda()) child_fc = autograd.Variable( torch.zeros(fx_len, self.hidden_dim).cuda()) child_fc_sum = autograd.Variable( torch.zeros(nlen * max_childs, self.hidden_dim).cuda()) select_indices = autograd.Variable( torch.LongTensor(input_ix).cuda()) offset_pos = autograd.Variable( torch.LongTensor(offset_pos).cuda()) fx_offset = autograd.Variable( torch.LongTensor(fx_offset).cuda()) hc_offset = autograd.Variable( torch.LongTensor(hc_offset).cuda()) fc_offset = autograd.Variable( torch.LongTensor(fc_offset).cuda()) else: if child_h is None: child_h = autograd.Variable( torch.zeros(nlen, self.hidden_dim)) child_c = autograd.Variable( torch.zeros(nlen, self.hidden_dim)) child_h_sum = autograd.Variable( torch.zeros(nlen * max_childs, self.hidden_dim)) child_fh = autograd.Variable( torch.zeros(fx_len, self.hidden_dim)) child_fc = autograd.Variable( torch.zeros(fx_len, self.hidden_dim)) child_fc_sum = autograd.Variable( torch.zeros(nlen * max_childs, self.hidden_dim)) offset_pos = autograd.Variable(torch.LongTensor(offset_pos)) select_indices = autograd.Variable(torch.LongTensor(input_ix)) fx_offset = autograd.Variable(torch.LongTensor(fx_offset)) hc_offset = autograd.Variable(torch.LongTensor(hc_offset)) fc_offset = autograd.Variable(torch.LongTensor(fc_offset)) # embed_input = torch.index_select(embeds, 0, select_indices) # test_start = time.time() embed_input = embeds[select_indices] fh = self.fh(child_h) if len(offset_pos) > 0: child_h_sum.index_copy_(0, offset_pos, child_h) child_fh.index_copy_(0, hc_offset, fh) child_fc.index_copy_(0, hc_offset, child_c) f = F.sigmoid(self.fx(embed_input[fx_offset]) + child_fh) fc = F.mul(f, child_fc) child_fc_sum.index_copy_(0, fc_offset, fc) child_h_sum = child_h_sum.view([nlen, max_childs, self.hidden_dim]) child_fc_sum = child_fc_sum.view( [nlen, max_childs, self.hidden_dim]) child_c, child_h = self.batch_forward(embed_input, torch.sum(child_h_sum, 1), torch.sum(child_fc_sum, 1)) out = self.out(self.dropout(child_h)) out = torch.unsqueeze(out, 1) # test_start = time.time() for idx, node in enumerate(nodes): node.dt_state = torch.unsqueeze(child_c[idx], 0), torch.unsqueeze( child_h[idx], 0) # if node.label is not None: # if self.add_cuda: # node_gold = autograd.Variable(torch.LongTensor([node.label]).cuda()) # else: # node_gold = autograd.Variable(torch.LongTensor([node.label])) # # out_list.append(out[idx]) # # gold_list.append(node_gold) # forest_loss += self.loss_func(out[idx], node_gold) # print("test time:",time.time()-test_start) if level == 0: # forest_loss = self.loss_func(torch.cat(out_list),torch.cat(gold_list)) # return torch.squeeze(out, 1), forest_loss return torch.squeeze(out, 1)
def _add_losses(self, sigma_rpn=3.0): ## RPN, class loss #rpn_cls_score = self._predictions['rpn_cls_score_reshape'].view(-1, 2) #rpn_label = self._anchor_targets['rpn_labels'].view(-1) #rpn_select = (rpn_label.data != -1).nonzero().view(-1) #rpn_cls_score = rpn_cls_score.index_select(0, rpn_select).contiguous().view(-1, 2) #rpn_label = rpn_label.index_select(0, rpn_select).contiguous().view(-1) #rpn_cross_entropy = F.cross_entropy(rpn_cls_score, rpn_label) ## RPN, bbox loss #rpn_bbox_pred = self._predictions['rpn_bbox_pred'] #rpn_bbox_targets = self._anchor_targets['rpn_bbox_targets'] #rpn_bbox_inside_weights = self._anchor_targets['rpn_bbox_inside_weights'] #rpn_bbox_outside_weights = self._anchor_targets['rpn_bbox_outside_weights'] #rpn_loss_box = self._smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights, # rpn_bbox_outside_weights, sigma=sigma_rpn, dim=[1, 2, 3]) # RCNN, class loss # Done in 2018/11/19 #cls_score = self._predictions["cls_score"] #label = self._proposal_targets["labels"].view(-1) #cross_entropy = F.cross_entropy(cls_score.view(-1, self._num_classes), label) # RCNN, bbox loss #bbox_pred = self._predictions['bbox_pred'] #bbox_targets = self._proposal_targets['bbox_targets'] #bbox_inside_weights = self._proposal_targets['bbox_inside_weights'] #bbox_outside_weights = self._proposal_targets['bbox_outside_weights'] #loss_box = self._smooth_l1_loss(bbox_pred, bbox_targets, bbox_inside_weights, bbox_outside_weights) #self._losses['cross_entropy'] = cross_entropy #self._losses['loss_box'] = loss_box self._losses['cross_entropy'] = np.zeros(1) self._losses['loss_box'] = np.zeros(1) self._losses['rpn_cross_entropy'] = np.zeros(1) self._losses['rpn_loss_box'] = np.zeros(1) det_cls_prob = self._predictions['det_cls_prob'] det_cls_prob = det_cls_prob.view(-1) label = self._image_level_label.view(-1) det_cls_product = self._predictions['det_cls_prob_product'] refine_prob_1 = self._predictions['refine_prob_1'] refine_prob_2 = self._predictions['refine_prob_2'] #refine_prob_3 = self._predictions['refine_prob_3'] #caculating the loss of the first branch roi_labels, roi_weights ,keep_inds = self.get_refine_supervision(det_cls_product, self._image_gt_summaries['ss_boxes'], self._image_gt_summaries['image_level_label']) roi_weights = torch.tensor(roi_weights).cuda() roi_labels = torch.tensor(roi_labels, dtype=roi_weights.dtype).cuda() #roi_labels = torch.mul(roi_labels, roi_weights) refine_loss_1 = - torch.sum(torch.mul(roi_labels, torch.log(refine_prob_1[keep_inds]))) / roi_labels.shape[0] #caculating the loss of the second branch roi_labels, roi_weights, keep_inds = self.get_refine_supervision(refine_prob_1, self._image_gt_summaries['ss_boxes'], self._image_gt_summaries['image_level_label']) roi_weights = torch.tensor(roi_weights).cuda() roi_labels = torch.tensor(roi_labels, dtype=roi_weights.dtype).cuda() #roi_labels = torch.mul(roi_labels, roi_weights) refine_loss_2 = - torch.sum(torch.mul(roi_labels, torch.log(refine_prob_2[keep_inds]))) / roi_labels.shape[0] #roi_labels, roi_weights, keep_inds = self.get_refine_supervision(refine_prob_2, self._image_gt_summaries['ss_boxes'], # self._image_gt_summaries['image_level_label']) #roi_weights = torch.tensor(roi_weights).cuda() #roi_labels = torch.tensor(roi_labels, dtype=roi_weights.dtype).cuda() #roi_labels = torch.mul(roi_labels, roi_weights) #refine_loss_3 = - torch.sum(torch.mul(roi_labels, torch.log(refine_prob_3[keep_inds]))) / roi_labels.shape[0] self._losses['refine_loss_1'] = refine_loss_1 self._losses['refine_loss_2'] = refine_loss_2 #self._losses['refine_loss_3'] = refine_loss_3 #print('label ', label) label = torch.tensor(label, dtype=det_cls_prob.dtype, device=det_cls_prob.device) zeros = torch.zeros(det_cls_prob.shape, dtype=det_cls_prob.dtype, device=det_cls_prob.device) max_zeros = torch.max(zeros, 1-F.mul(label, det_cls_prob)) cls_det_loss = torch.sum(max_zeros) self._losses['cls_det_loss'] = cls_det_loss / 20 #for WSDNN or MultiBranch Learning #cls_det_loss = - torch.sum(torch.log(det_cls_prob[label==1])) - torch.sum(torch.log(1 - det_cls_prob[label == -1])) #self._losses['cls_det_loss'] = cls_det_loss / 20 #loss = cross_entropy + loss_box + rpn_cross_entropy + rpn_loss_box loss = cls_det_loss / 20 + refine_loss_1*0.1 + refine_loss_2*0.1 self._losses['total_loss'] = loss #print('loss ', loss) for k in self._losses.keys(): self._event_summaries[k] = self._losses[k] return loss
def forward(self, x): return F.mul(x, self.scale)