def loss(self, match_score, ids, id_weights, reduce=True): losses = dict() if self.dynamic: n = len(match_score) x_n = [s.size(0) for s in match_score] ids = torch.split(ids, x_n, dim=0) loss_match = 0. match_acc = 0. n_total = 0 batch_size = len(ids) for score, cur_ids, cur_weights in zip(match_score, ids, id_weights): valid_idx = torch.nonzero(cur_weights).squeeze() if len(valid_idx.size()) == 0: continue n_valid = valid_idx.size(0) n_total += n_valid loss_match += weighted_cross_entropy( score, cur_ids, cur_weights, reduce=reduce) match_acc += accuracy(torch.index_select(score, 0, valid_idx), torch.index_select(cur_ids, 0, valid_idx)) * n_valid losses['loss_match'] = loss_match / n if n_total > 0: losses['match_acc'] = match_acc / n_total else: if match_score is not None: valid_idx = torch.nonzero(cur_weights).squeeze() losses['loss_match'] = weighted_cross_entropy( match_score, ids, id_weights, reduce=reduce) losses['match_acc'] = accuracy(torch.index_select(match_score, 0, valid_idx), torch.index_select(ids, 0, valid_idx)) return losses
def loss(self, cls_score, bbox_pred, labels, label_weights, bbox_targets, bbox_weights, reduce=True, img_meta_2=None, img_meta_3=None): losses = dict() if cls_score is not None: if img_meta_3 is not None: losses['loss_cls_3'] = weighted_cross_entropy(cls_score, labels, label_weights, reduce=reduce) losses['acc_3'] = accuracy(cls_score, labels) elif img_meta_2 is not None: losses['loss_cls_2'] = weighted_cross_entropy(cls_score, labels, label_weights, reduce=reduce) losses['acc_2'] = accuracy(cls_score, labels) else: losses['loss_cls'] = weighted_cross_entropy(cls_score, labels, label_weights, reduce=reduce) losses['acc'] = accuracy(cls_score, labels) if bbox_pred is not None: pos_inds = labels > 0 if self.reg_class_agnostic: pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), 6)[pos_inds] else: pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), -1, 6)[pos_inds, labels[pos_inds]] if img_meta_3 is not None: losses['loss_reg_3'] = weighted_smoothl1( pos_bbox_pred, bbox_targets[pos_inds], bbox_weights[pos_inds], avg_factor=bbox_targets.size(0)) elif img_meta_2 is not None: losses['loss_reg_2'] = weighted_smoothl1( pos_bbox_pred, bbox_targets[pos_inds], bbox_weights[pos_inds], avg_factor=bbox_targets.size(0)) else: losses['loss_reg'] = weighted_smoothl1( pos_bbox_pred, bbox_targets[pos_inds], bbox_weights[pos_inds], avg_factor=bbox_targets.size(0)) return losses
def loss( self, cls_score, bbox_pred, labels, label_weights, bbox_targets, bbox_weights, reduce=True, ): losses = dict() if cls_score is not None: losses["loss_cls"] = weighted_cross_entropy(cls_score, labels, label_weights, reduce=reduce) losses["acc"] = accuracy(cls_score, labels) if bbox_pred is not None: pos_inds = labels > 0 if self.reg_class_agnostic: pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), 4)[pos_inds] else: pos_bbox_pred = bbox_pred.view(bbox_pred.size(0), -1, 4)[pos_inds, labels[pos_inds]] losses["loss_reg"] = weighted_smoothl1( pos_bbox_pred, bbox_targets[pos_inds], bbox_weights[pos_inds], avg_factor=bbox_targets.size(0), ) return losses
def loss(self, cls_score, bbox_pred, labels, label_weights, bbox_targets, bbox_weights, mix_inds=None, reduce=True): losses = dict() if cls_score is not None: losses['loss_cls'] = weighted_cross_entropy(cls_score, labels, label_weights, mix_inds=mix_inds, reduce=reduce) losses['acc'] = accuracy(cls_score, labels) if bbox_pred is not None: losses['loss_reg'] = weighted_smoothl1( bbox_pred, bbox_targets, bbox_weights, mix_inds=mix_inds, avg_factor=bbox_targets.size(0)) return losses
def loss_single(self, rpn_cls_score, rpn_bbox_pred, rpn_dir_pred, labels, label_weights, bbox_targets, bbox_weights, dir_labels, dir_weights, num_total_samples, cfg): # classification loss labels = labels.contiguous().view(-1) label_weights = label_weights.contiguous().view(-1) if self.use_sigmoid_cls: rpn_cls_score = rpn_cls_score.permute(0, 2, 3, 1).contiguous().view(-1) #criterion = weighted_binary_cross_entropy criterion = weighted_sigmoid_focal_loss else: rpn_cls_score = rpn_cls_score.permute(0, 2, 3, 1).contiguous().view(-1, 2) criterion = weighted_cross_entropy loss_cls = criterion(rpn_cls_score, labels, label_weights, avg_factor=num_total_samples) # regression loss bbox_targets = bbox_targets.contiguous().view(-1, 7) bbox_weights = bbox_weights.contiguous().view(-1, 7) rpn_bbox_pred = rpn_bbox_pred.permute(0, 2, 3, 1).contiguous().view(-1, 7) rpn_bbox_pred, bbox_targets = add_sin_difference( rpn_bbox_pred, bbox_targets) loss_reg = weighted_smoothl1(rpn_bbox_pred, bbox_targets, bbox_weights, beta=cfg.smoothl1_beta, avg_factor=num_total_samples) # direction loss dir_logits = rpn_dir_pred.permute(0, 2, 3, 1).contiguous().view(-1, 2) loss_dir = weighted_cross_entropy(dir_logits, dir_labels, dir_weights, avg_factor=num_total_samples) loss_reg *= 2 loss_dir *= .2 return loss_cls, loss_reg, loss_dir
def loss(self, cls_score, bbox_pred, labels, label_weights, bbox_targets, bbox_weights, reduce=True): losses = dict() if cls_score is not None: losses['loss_cls'] = weighted_cross_entropy(cls_score, labels, label_weights, reduce=reduce) losses['acc'] = accuracy(cls_score, labels) return losses
def loss(self, cls_score, bbox_pred, A_pred, A_gt, labels, label_weights, bbox_targets, bbox_weights, reduce=True): losses = dict() if cls_score is not None: losses['loss_cls'] = weighted_cross_entropy( cls_score, labels, label_weights, reduce=reduce) losses['acc'] = accuracy(cls_score, labels) if bbox_pred is not None: losses['loss_reg'] = weighted_smoothl1( bbox_pred, bbox_targets, bbox_weights, avg_factor=bbox_targets.size(0)) if A_pred: assert len(A_pred) == len(A_gt) assert A_pred[0].size() == A_gt[0].size() num_a_pred = len(A_pred) for i in range(num_a_pred): losses['loss_adj' + str(i)] = F.mse_loss(A_pred[i], A_gt[i].detach()) return losses
def loss(self, cls_score, bbox_pred, labels, label_weights, bbox_targets, bbox_weights, alpha=0.25, num_level=4, reduce=True): losses = dict() for i in range(num_level): cls_score_level_i = cls_score[i::num_level, :] bbox_pred_level_i = bbox_pred[i::num_level, :] losses['loss_cls_level%d' % i] = weighted_cross_entropy( cls_score_level_i, labels, label_weights, reduce=reduce) * alpha losses['loss_reg_level%d' % i] = weighted_smoothl1( bbox_pred_level_i, bbox_targets, bbox_weights, avg_factor=bbox_targets.size(0)) * alpha return losses
def forward_train(self, img, img_meta, gt_bboxes, gt_bboxes_8_coo, gt_labels, gt_bboxes_ignore=None, gt_masks=None, proposals=None): if gt_bboxes_ignore is None: gt_bboxes_ignore = [None for _ in range(len(gt_bboxes))] x = self.extract_feat(img) losses = dict() # RPN forward and loss if self.with_rpn: rpn_outs = self.rpn_head(x) rpn_loss_inputs = rpn_outs + (gt_bboxes, gt_bboxes_8_coo, img_meta, self.coo_num, self.train_cfg.rpn) rpn_losses = self.rpn_head.loss(*rpn_loss_inputs) losses.update(rpn_losses) proposal_inputs = rpn_outs + (img_meta, self.coo_num, self.test_cfg.rpn) proposal_list = self.rpn_head.get_proposals(*proposal_inputs) else: proposal_list = proposals if self.train_cfg.rpn.with_gt_bboxes: for i in range(len(gt_bboxes)): gt_bboxes_trans = [] for j in self.train_cfg.rpn.gt_bboxes_scale: x_center = (gt_bboxes[i][:, 0:1]+gt_bboxes[i][:, 2:3])/2 y_center = (gt_bboxes[i][:, 1:2]+gt_bboxes[i][:, 3:4])/2 left = torch.clamp(((gt_bboxes[i][:, 0:1] - x_center) * j + x_center), min=0) right = torch.clamp(((gt_bboxes[i][:, 2:3] - x_center) * j + x_center), max=img_meta[i]['img_shape'][1]) top = torch.clamp(((gt_bboxes[i][:, 1:2] - y_center) * j + y_center), min=0) bottom = torch.clamp(((gt_bboxes[i][:, 3:4] - y_center) * j + y_center), max=img_meta[i]['img_shape'][0]) trans_gt_bboxes = torch.cat([left, top, right, bottom], 1) gt_bboxes_trans.append(trans_gt_bboxes) gt_bboxes_trans = torch.cat(gt_bboxes_trans, 0) n = gt_bboxes_trans.shape[0] gt_bboxes_trans = torch.cat([gt_bboxes_trans, torch.ones([n, 1], device=gt_bboxes[i].device)], 1) proposal_list[i] = torch.cat([proposal_list[i], gt_bboxes_trans], 0) # assign gts and sample proposals _gt_bboxes_8_coo = [None for i in range(len(proposal_list))] if self.with_bbox or self.with_mask: assign_results, sampling_results = multi_apply( assign_and_sample, proposal_list, gt_bboxes, _gt_bboxes_8_coo, gt_bboxes_ignore, gt_labels, cfg=self.train_cfg.rcnn) # bbox head forward and loss if self.with_bbox: rois = bbox2roi([res.bboxes for res in sampling_results]) # TODO: a more flexible way to decide which feature maps to use bbox_feats = self.bbox_roi_extractor( x[:self.bbox_roi_extractor.num_inputs], rois, self.level, self.merge_mode) cls_score, bbox_pred = self.bbox_head(bbox_feats) bbox_targets = self.bbox_head.get_target( sampling_results, gt_bboxes, gt_labels, self.train_cfg.rcnn) loss_bbox = self.bbox_head.loss(cls_score, bbox_pred, *bbox_targets) if self.train_cfg.rcnn.with_hard_example_mining: pred_label = torch.argmax(cls_score, 1) labels, label_weights, bbox_gt, bbox_weights = bbox_targets ind = pred_label != labels if torch.sum(ind).item() != 0: x_stop_grad = [feature.data for feature in x] bbox_feats_stop_grad = self.bbox_roi_extractor( x_stop_grad[:self.bbox_roi_extractor.num_inputs], rois, self.level, self.merge_mode) cls_score_stop_grad, bbox_pred_stop_grad = self.bbox_head(bbox_feats_stop_grad) cls_score_stop_grad, labels, label_weights = cls_score_stop_grad[ind], labels[ind], label_weights[ind] num = cls_score.shape[0] loss_bbox['loss_cls'] = loss_bbox['loss_cls'] + weighted_cross_entropy( cls_score_stop_grad, labels, label_weights, avg_factor=num) if self.train_cfg.rcnn.with_reg: bbox_pred_stop_grad, bbox_gt, bbox_weights = bbox_pred_stop_grad[ind], bbox_gt[ind], bbox_weights[ind] loss_bbox['loss_reg'] = loss_bbox['loss_reg'] + weighted_smoothl1( bbox_pred_stop_grad, bbox_gt, bbox_weights, avg_factor=num) losses.update(loss_bbox) # mask head forward and loss if self.with_mask: pos_rois = bbox2roi([res.pos_bboxes for res in sampling_results]) mask_feats = self.mask_roi_extractor( x[:self.mask_roi_extractor.num_inputs], pos_rois) mask_pred = self.mask_head(mask_feats) mask_targets = self.mask_head.get_target( sampling_results, gt_masks, self.train_cfg.rcnn) pos_labels = torch.cat( [res.pos_gt_labels for res in sampling_results]) loss_mask = self.mask_head.loss(mask_pred, mask_targets, pos_labels) losses.update(loss_mask) return losses
def loss(self, match_score, shift, IoUs2, distances_xy, ids, id_weights, reduce=True): losses = dict() if self.dynamic: n = len(match_score) x_n = [s.size(0) for s in match_score] ids = torch.split(ids, x_n, dim=0) loss_match = 0. match_acc = 0. n_total = 0 batch_size = len(ids) for score, cur_ids, cur_weights in zip(match_score, ids, id_weights): valid_idx = torch.nonzero(cur_weights).squeeze() valid_cls = torch.index_select(cur_ids, 0, valid_idx) nonzero_ids = valid_cls.clone() nonzero_ids[nonzero_ids > 0] = 1 #loss_iou = torch.sum((1-IoUs[valid_idx , valid_cls])*nonzero_ids) #loss_iou = loss_iou/(torch.sum(nonzero_ids)+1e-7) #print(valid_idx) valid_iou = IoUs2[valid_idx, valid_cls] distance_mean = torch.zeros(2, device=torch.cuda.current_device()) for match_cls in valid_cls.unique(): if match_cls == 0: continue valid_iou[(valid_cls == match_cls) & ( valid_iou != valid_iou[valid_cls == match_cls][0])] = 0 valid_iou[valid_iou < 0.7] = 0 valid_xy = distances_xy[:, valid_idx, valid_cls] #dif_x = valid_xy[0]-shift[0][0] #dif_y = valid_xy[1]-shift[0][1] #distance_mean = torch.sum(torch.sqrt(dif_x*dif_x + dif_y*dif_y) * valid_iou / (torch.sum(valid_iou)+1e-7)) if len(valid_idx.size()) == 0: continue n_valid = valid_idx.size(0) n_total += n_valid loss_match += weighted_cross_entropy(score, cur_ids, cur_weights, reduce=reduce) match_acc += accuracy(torch.index_select(score, 0, valid_idx), valid_cls) * n_valid losses['loss_match'] = loss_match / n if n_total > 0: losses['match_acc'] = match_acc / n_total else: print('not dynamic') if match_score is not None: valid_idx = torch.nonzero(cur_weights).squeeze() losses['loss_match'] = weighted_cross_entropy(match_score, ids, id_weights, reduce=reduce) losses['match_acc'] = accuracy( torch.index_select(match_score, 0, valid_idx), torch.index_select(ids, 0, valid_idx)) return losses