def loss_orig(self, prediction_dict, feed_dict): # loss for cls loss_dict = {} targets = prediction_dict[constants.KEY_TARGETS] cls_target = targets[constants.KEY_CLASSES] loc1_target = targets[constants.KEY_BOXES_2D] loc2_target = targets[constants.KEY_BOXES_2D_REFINE] os_target = targets[constants.KEY_OBJECTNESS] rpn_cls_loss = common_loss.calc_loss(focal_loss_alt(self.num_classes), cls_target, normalize=False) rpn_loc1_loss = common_loss.calc_loss(self.rpn_bbox_loss, loc1_target) rpn_os_loss = common_loss.calc_loss(focal_loss_alt(2), os_target, normalize=False) rpn_loc2_loss = common_loss.calc_loss(self.rpn_bbox_loss, loc2_target) cls_targets = cls_target['target'] pos = cls_targets > 0 # [N,#anchors] num_pos = pos.data.long().sum().clamp(min=1).float() cls_loss = rpn_cls_loss / num_pos os_loss = rpn_os_loss / num_pos loss_dict.update({ 'rpn_cls_loss': cls_loss, 'rpn_loc1_loss': rpn_loc1_loss * 0.35, 'rpn_loc2_loss': rpn_loc2_loss * 0.5, 'rpn_os_loss': os_loss * 10 }) return loss_dict
def loss(self, prediction_dict, feed_dict): """ assign proposals label and subsample from them Then calculate loss """ loss_dict = {} # import ipdb # ipdb.set_trace() # submodule loss loss_dict.update(self.rpn_model.loss(prediction_dict, feed_dict)) targets = prediction_dict[constants.KEY_TARGETS] # cls_target = targets[0] # reg_target = targets[3] rcnn_cls_loss = 0 rcnn_reg_loss = 0 for cls_target in targets[::2]: rcnn_cls_loss = rcnn_cls_loss + common_loss.calc_loss( self.rcnn_cls_loss, cls_target) for reg_target in targets[1::2]: rcnn_reg_loss = rcnn_reg_loss + common_loss.calc_loss( self.rcnn_bbox_loss, reg_target) loss_dict.update({ 'rcnn_cls_loss': rcnn_cls_loss, 'rcnn_reg_loss': rcnn_reg_loss }) return loss_dict
def loss(self, prediction_dict, feed_dict): """ assign proposals label and subsample from them Then calculate loss """ loss_dict = super().loss(prediction_dict, feed_dict) targets = prediction_dict[constants.KEY_TARGETS] rcnn_corners_loss = 0 rcnn_dim_loss = 0 for stage_ind in range(self.num_stages): orient_target = targets[stage_ind][2] rcnn_corners_loss = rcnn_corners_loss + common_loss.calc_loss( self.rcnn_corners_loss, orient_target, True) dim_target = targets[stage_ind][3] rcnn_dim_loss = rcnn_dim_loss + common_loss.calc_loss( self.rcnn_bbox_loss, dim_target, True) loss_dict.update({ 'rcnn_corners_loss': rcnn_corners_loss, # 'rcnn_dim_loss': rcnn_dim_loss }) return loss_dict
def loss(self, prediction_dict, feed_dict): # loss for cls loss_dict = {} anchors = prediction_dict['anchors'] anchors_dict = {} anchors_dict[constants.KEY_PRIMARY] = anchors anchors_dict[ constants.KEY_BOXES_2D] = prediction_dict['rpn_bbox_preds'] anchors_dict[constants.KEY_CLASSES] = prediction_dict['rpn_cls_scores'] gt_dict = {} gt_dict[constants.KEY_PRIMARY] = feed_dict[ constants.KEY_LABEL_BOXES_2D] gt_dict[constants.KEY_CLASSES] = None gt_dict[constants.KEY_BOXES_2D] = None auxiliary_dict = {} auxiliary_dict[constants.KEY_BOXES_2D] = feed_dict[ constants.KEY_LABEL_BOXES_2D] gt_labels = feed_dict[constants.KEY_LABEL_CLASSES] auxiliary_dict[constants.KEY_CLASSES] = torch.ones_like(gt_labels) auxiliary_dict[constants.KEY_NUM_INSTANCES] = feed_dict[ constants.KEY_NUM_INSTANCES] auxiliary_dict[constants.KEY_PROPOSALS] = anchors # import ipdb # ipdb.set_trace() _, targets, _ = self.target_generators.generate_targets( anchors_dict, gt_dict, auxiliary_dict, subsample=False) cls_target = targets[constants.KEY_CLASSES] reg_target = targets[constants.KEY_BOXES_2D] # loss if self.use_focal_loss: # when using focal loss, dont normalize it by all samples cls_targets = cls_target['target'] pos = cls_targets > 0 # [N,#anchors] num_pos = pos.long().sum().clamp(min=1).float() rpn_cls_loss = common_loss.calc_loss( self.rpn_cls_loss, cls_target, normalize=False) / num_pos else: rpn_cls_loss = common_loss.calc_loss(self.rpn_cls_loss, cls_target) rpn_reg_loss = common_loss.calc_loss(self.rpn_bbox_loss, reg_target) loss_dict.update({ 'rpn_cls_loss': rpn_cls_loss, 'rpn_reg_loss': rpn_reg_loss }) return loss_dict
def loss(self, prediction_dict, feed_dict): """ assign proposals label and subsample from them Then calculate loss """ loss_dict = {} # import ipdb # ipdb.set_trace() # submodule loss loss_dict.update(self.rpn_model.loss(prediction_dict, feed_dict)) targets = prediction_dict[constants.KEY_TARGETS] rcnn_cls_loss = 0 rcnn_reg_loss = 0 for stage_ind in range(self.num_stages): cls_target = targets[stage_ind][0] if self.use_focal_loss: # when using focal loss, dont normalize it by all samples cls_targets = cls_target['target'] pos = cls_targets > 0 # [N,#anchors] num_pos = pos.long().sum().clamp(min=1).float() rcnn_cls_loss = common_loss.calc_loss( self.rcnn_cls_loss, cls_target, normalize=False) / num_pos else: rcnn_cls_loss = rcnn_cls_loss + common_loss.calc_loss( self.rcnn_cls_loss, cls_target) # rcnn_cls_loss = rcnn_cls_loss + common_loss.calc_loss( # self.rcnn_cls_loss, cls_target) reg_target = targets[stage_ind][1] rcnn_reg_loss = rcnn_reg_loss + common_loss.calc_loss( self.rcnn_bbox_loss, reg_target) loss_dict.update({ 'rcnn_cls_loss': rcnn_cls_loss, 'rcnn_reg_loss': rcnn_reg_loss }) return loss_dict
def loss(self, prediction_dict, feed_dict): # import ipdb # ipdb.set_trace() loss_dict = {} anchors = prediction_dict['anchors'] anchors_dict = {} anchors_dict[constants.KEY_PRIMARY] = anchors anchors_dict[ constants.KEY_BOXES_2D] = prediction_dict['rpn_bbox_preds'] anchors_dict[constants.KEY_CLASSES] = prediction_dict['rpn_cls_scores'] anchors_dict[ constants.KEY_CORNERS_3D_GRNET] = prediction_dict['corners_3d'] gt_dict = {} gt_dict[constants.KEY_PRIMARY] = feed_dict[ constants.KEY_LABEL_BOXES_2D] gt_dict[constants.KEY_CLASSES] = None gt_dict[constants.KEY_BOXES_2D] = None gt_dict[constants.KEY_CORNERS_3D_GRNET] = None auxiliary_dict = {} auxiliary_dict[constants.KEY_BOXES_2D] = feed_dict[ constants.KEY_LABEL_BOXES_2D] gt_labels = feed_dict[constants.KEY_LABEL_CLASSES] auxiliary_dict[constants.KEY_CLASSES] = torch.ones_like(gt_labels) auxiliary_dict[constants.KEY_NUM_INSTANCES] = feed_dict[ constants.KEY_NUM_INSTANCES] auxiliary_dict[constants.KEY_PROPOSALS] = anchors auxiliary_dict[constants.KEY_BOXES_3D] = feed_dict[ constants.KEY_LABEL_BOXES_3D] auxiliary_dict[constants.KEY_STEREO_CALIB_P2] = feed_dict[ constants.KEY_STEREO_CALIB_P2] # import ipdb # ipdb.set_trace() subsample = not self.use_focal_loss _, targets, _ = self.target_generators.generate_targets( anchors_dict, gt_dict, auxiliary_dict, subsample=subsample) cls_target = targets[constants.KEY_CLASSES] reg_target = targets[constants.KEY_BOXES_2D] # loss if self.use_focal_loss: # when using focal loss, dont normalize it by all samples cls_targets = cls_target['target'] pos = cls_targets > 0 # [N,#anchors] num_pos = pos.long().sum().clamp(min=1).float() rpn_cls_loss = common_loss.calc_loss( self.rpn_cls_loss, cls_target, normalize=False) / num_pos else: rpn_cls_loss = common_loss.calc_loss(self.rpn_cls_loss, cls_target) rpn_reg_loss = common_loss.calc_loss(self.rpn_bbox_loss, reg_target) loss_dict.update({ 'rpn_cls_loss': rpn_cls_loss, 'rpn_reg_loss': rpn_reg_loss }) # return loss_dict # super().loss(prediction_dict, feed_dict) # proposals = prediction_dict[constants.KEY_PROPOSALS] proposals = anchors_dict[constants.KEY_PRIMARY] p2 = feed_dict[constants.KEY_STEREO_CALIB_P2] image_info = feed_dict[constants.KEY_IMAGE_INFO] mean_dims = torch.tensor([1.8, 1.8, 3.7]).type_as(proposals) corners_2d_loss = 0 center_depth_loss = 0 location_loss = 0 corners_target = targets[constants.KEY_CORNERS_3D_GRNET] # rcnn_corners_loss = rcnn_corners_loss + common_loss.calc_loss( # self.rcnn_corners_loss, orient_target, True) preds = corners_target['pred'] targets = corners_target['target'] weights = corners_target['weight'] weights = weights.unsqueeze(-1) local_corners_gt = targets[:, :, :24] location_gt = targets[:, :, 24:27] dims_gt = targets[:, :, 27:] N, M = local_corners_gt.shape[:2] global_corners_gt = (local_corners_gt.view(N, M, 8, 3) + location_gt.view(N, M, 1, 3)).view(N, M, -1) center_depth_gt = location_gt[:, :, 2:] dims_preds = torch.exp(preds[:, :, :3]) * mean_dims # import ipdb # ipdb.set_trace() dims_loss = self.l1_loss(dims_preds, dims_gt) * weights ry_preds = preds[:, :, 3:4] # ray_angle = -torch.atan2(location_gt[:, :, 2], location_gt[:, :, 0]) # ry_preds = ry_preds + ray_angle.unsqueeze(-1) local_corners_preds = [] # calc local corners preds for batch_ind in range(N): local_corners_preds.append( self.calc_local_corners(dims_preds[batch_ind].detach(), ry_preds[batch_ind])) local_corners_preds = torch.stack(local_corners_preds, dim=0) center_2d_deltas_preds = preds[:, :, 4:6] center_depth_preds = preds[:, :, 6:] # import ipdb # ipdb.set_trace() # decode center_2d proposals_xywh = geometry_utils.torch_xyxy_to_xywh(proposals) center_depth_init = self.decode_center_depth(dims_preds, proposals_xywh, p2) center_depth_preds = center_depth_init * center_depth_preds center_2d_preds = (center_2d_deltas_preds * proposals_xywh[:, :, 2:] + proposals_xywh[:, :, :2]) # center_depth_preds_detach = center_depth_preds.detach() # import ipdb # ipdb.set_trace() # use gt depth to cal loss to make sure the gradient smooth location_preds = [] for batch_ind in range(N): location_preds.append( geometry_utils.torch_points_2d_to_points_3d( center_2d_preds[batch_ind], center_depth_gt[batch_ind], p2[batch_ind])) location_preds = torch.stack(location_preds, dim=0) global_corners_preds = (location_preds.view(N, M, 1, 3) + local_corners_preds.view(N, M, 8, 3)).view( N, M, -1) # import ipdb # ipdb.set_trace() # corners depth loss and center depth loss corners_depth_preds = local_corners_preds.view(N, M, 8, 3)[..., -1] corners_depth_gt = local_corners_gt.view(N, M, 8, 3)[..., -1] center_depth_loss = self.l1_loss(center_depth_preds, center_depth_gt) * weights # location loss location_loss = self.l1_loss(location_preds, location_gt) * weights # global corners loss global_corners_loss = self.l1_loss(global_corners_preds, global_corners_gt) * weights # proj 2d loss corners_2d_preds = [] corners_2d_gt = [] for batch_ind in range(N): corners_2d_preds.append( geometry_utils.torch_points_3d_to_points_2d( global_corners_preds[batch_ind].view(-1, 3), p2[batch_ind])) corners_2d_gt.append( geometry_utils.torch_points_3d_to_points_2d( global_corners_gt[batch_ind].view(-1, 3), p2[batch_ind])) corners_2d_preds = torch.stack(corners_2d_preds, dim=0).view(N, M, -1) corners_2d_gt = torch.stack(corners_2d_gt, dim=0).view(N, M, -1) # image filter # import ipdb # ipdb.set_trace() zeros = torch.zeros_like(image_info[:, 0]) image_shape = torch.stack( [zeros, zeros, image_info[:, 1], image_info[:, 0]], dim=-1) image_shape = image_shape.type_as(corners_2d_gt).view(-1, 4) image_filter = geometry_utils.torch_window_filter( corners_2d_gt.view(N, -1, 2), image_shape, deltas=200).float().view(N, M, -1) # import ipdb # ipdb.set_trace() encoded_corners_2d_gt = corners_2d_gt.view(N, M, 8, 2) encoded_corners_2d_preds = corners_2d_preds.view(N, M, 8, 2) corners_2d_loss = self.l2_loss(encoded_corners_2d_preds.view( N, M, -1), encoded_corners_2d_gt.view(N, M, -1)) * weights corners_2d_loss = (corners_2d_loss.view(N, M, 8, 2) * image_filter.unsqueeze(-1)) # import ipdb # ipdb.set_trace() # mask = self.select_corners(global_corners_gt) # mask = mask.unsqueeze(-1).expand_as(corners_2d_loss).float() corners_2d_loss = corners_2d_loss.view(N, M, -1) corners_depth_loss = self.l1_loss( corners_depth_preds, corners_depth_gt) * weights * image_filter # import ipdb # ipdb.set_trace() # corners_3d_gt = [] # for batch_ind in range(N): # corners_3d_gt.append( # geometry_utils.torch_points_2d_to_points_3d( # corners_2d_preds[batch_ind].view(-1, 2), # corners_depth_preds[batch_ind].view(-1), p2[batch_ind])) # corners_3d_gt = torch.stack(corners_3d_gt, dim=0).view(N, M, -1) # dim_target = targets[stage_ind][3] # rcnn_dim_loss = rcnn_dim_loss + common_loss.calc_loss( # self.rcnn_bbox_loss, dim_target, True) global_corners_loss = self.l1_loss(global_corners_preds, global_corners_gt) * weights # local_corners_loss = self.l1_loss(local_corners_preds, # local_corners_gt) * weights # import ipdb # ipdb.set_trace() num_pos = (weights > 0).long().sum().clamp(min=1).float() loss_dict.update({ # 'global_corners_loss': global_corners_loss, # 'local_corners_loss': local_corners_loss * 10, 'corners_2d_loss': corners_2d_loss, # 'center_depth_loss': center_depth_loss, # 'location_loss': location_loss, # 'corners_depth_loss': corners_depth_loss * 10, # 'rcnn_corners_loss': rcnn_corners_loss, # 'rcnn_dim_loss': rcnn_dim_loss # 'dims_loss': dims_loss }) return loss_dict