def loss_orig(self, prediction_dict, feed_dict):
# loss for cls
loss_dict = {}
targets = prediction_dict[constants.KEY_TARGETS]
cls_target = targets[constants.KEY_CLASSES]
loc1_target = targets[constants.KEY_BOXES_2D]
loc2_target = targets[constants.KEY_BOXES_2D_REFINE]
os_target = targets[constants.KEY_OBJECTNESS]
rpn_cls_loss = common_loss.calc_loss(focal_loss_alt(self.num_classes),
cls_target,
normalize=False)
rpn_loc1_loss = common_loss.calc_loss(self.rpn_bbox_loss, loc1_target)
rpn_os_loss = common_loss.calc_loss(focal_loss_alt(2),
os_target,
normalize=False)
rpn_loc2_loss = common_loss.calc_loss(self.rpn_bbox_loss, loc2_target)
cls_targets = cls_target['target']
pos = cls_targets > 0 # [N,#anchors]
num_pos = pos.data.long().sum().clamp(min=1).float()
cls_loss = rpn_cls_loss / num_pos
os_loss = rpn_os_loss / num_pos
loss_dict.update({
'rpn_cls_loss': cls_loss,
'rpn_loc1_loss': rpn_loc1_loss * 0.35,
'rpn_loc2_loss': rpn_loc2_loss * 0.5,
'rpn_os_loss': os_loss * 10
})
return loss_dict
def loss(self, prediction_dict, feed_dict):
"""
assign proposals label and subsample from them
Then calculate loss
"""
loss_dict = {}
# import ipdb
# ipdb.set_trace()
# submodule loss
loss_dict.update(self.rpn_model.loss(prediction_dict, feed_dict))
targets = prediction_dict[constants.KEY_TARGETS]
# cls_target = targets[0]
# reg_target = targets[3]
rcnn_cls_loss = 0
rcnn_reg_loss = 0
for cls_target in targets[::2]:
rcnn_cls_loss = rcnn_cls_loss + common_loss.calc_loss(
self.rcnn_cls_loss, cls_target)
for reg_target in targets[1::2]:
rcnn_reg_loss = rcnn_reg_loss + common_loss.calc_loss(
self.rcnn_bbox_loss, reg_target)
loss_dict.update({
'rcnn_cls_loss': rcnn_cls_loss,
'rcnn_reg_loss': rcnn_reg_loss
})
return loss_dict
def loss(self, prediction_dict, feed_dict):
"""
assign proposals label and subsample from them
Then calculate loss
"""
loss_dict = super().loss(prediction_dict, feed_dict)
targets = prediction_dict[constants.KEY_TARGETS]
rcnn_corners_loss = 0
rcnn_dim_loss = 0
for stage_ind in range(self.num_stages):
orient_target = targets[stage_ind][2]
rcnn_corners_loss = rcnn_corners_loss + common_loss.calc_loss(
self.rcnn_corners_loss, orient_target, True)
dim_target = targets[stage_ind][3]
rcnn_dim_loss = rcnn_dim_loss + common_loss.calc_loss(
self.rcnn_bbox_loss, dim_target, True)
loss_dict.update({
'rcnn_corners_loss': rcnn_corners_loss,
# 'rcnn_dim_loss': rcnn_dim_loss
})
return loss_dict
def loss(self, prediction_dict, feed_dict):
# loss for cls
loss_dict = {}
anchors = prediction_dict['anchors']
anchors_dict = {}
anchors_dict[constants.KEY_PRIMARY] = anchors
anchors_dict[
constants.KEY_BOXES_2D] = prediction_dict['rpn_bbox_preds']
anchors_dict[constants.KEY_CLASSES] = prediction_dict['rpn_cls_scores']
gt_dict = {}
gt_dict[constants.KEY_PRIMARY] = feed_dict[
constants.KEY_LABEL_BOXES_2D]
gt_dict[constants.KEY_CLASSES] = None
gt_dict[constants.KEY_BOXES_2D] = None
auxiliary_dict = {}
auxiliary_dict[constants.KEY_BOXES_2D] = feed_dict[
constants.KEY_LABEL_BOXES_2D]
gt_labels = feed_dict[constants.KEY_LABEL_CLASSES]
auxiliary_dict[constants.KEY_CLASSES] = torch.ones_like(gt_labels)
auxiliary_dict[constants.KEY_NUM_INSTANCES] = feed_dict[
constants.KEY_NUM_INSTANCES]
auxiliary_dict[constants.KEY_PROPOSALS] = anchors
# import ipdb
# ipdb.set_trace()
_, targets, _ = self.target_generators.generate_targets(
anchors_dict, gt_dict, auxiliary_dict, subsample=False)
cls_target = targets[constants.KEY_CLASSES]
reg_target = targets[constants.KEY_BOXES_2D]
# loss
if self.use_focal_loss:
# when using focal loss, dont normalize it by all samples
cls_targets = cls_target['target']
pos = cls_targets > 0 # [N,#anchors]
num_pos = pos.long().sum().clamp(min=1).float()
rpn_cls_loss = common_loss.calc_loss(
self.rpn_cls_loss, cls_target, normalize=False) / num_pos
else:
rpn_cls_loss = common_loss.calc_loss(self.rpn_cls_loss, cls_target)
rpn_reg_loss = common_loss.calc_loss(self.rpn_bbox_loss, reg_target)
loss_dict.update({
'rpn_cls_loss': rpn_cls_loss,
'rpn_reg_loss': rpn_reg_loss
})
return loss_dict
def loss(self, prediction_dict, feed_dict):
"""
assign proposals label and subsample from them
Then calculate loss
"""
loss_dict = {}
# import ipdb
# ipdb.set_trace()
# submodule loss
loss_dict.update(self.rpn_model.loss(prediction_dict, feed_dict))
targets = prediction_dict[constants.KEY_TARGETS]
rcnn_cls_loss = 0
rcnn_reg_loss = 0
for stage_ind in range(self.num_stages):
cls_target = targets[stage_ind][0]
if self.use_focal_loss:
# when using focal loss, dont normalize it by all samples
cls_targets = cls_target['target']
pos = cls_targets > 0 # [N,#anchors]
num_pos = pos.long().sum().clamp(min=1).float()
rcnn_cls_loss = common_loss.calc_loss(
self.rcnn_cls_loss, cls_target, normalize=False) / num_pos
else:
rcnn_cls_loss = rcnn_cls_loss + common_loss.calc_loss(
self.rcnn_cls_loss, cls_target)
# rcnn_cls_loss = rcnn_cls_loss + common_loss.calc_loss(
# self.rcnn_cls_loss, cls_target)
reg_target = targets[stage_ind][1]
rcnn_reg_loss = rcnn_reg_loss + common_loss.calc_loss(
self.rcnn_bbox_loss, reg_target)
loss_dict.update({
'rcnn_cls_loss': rcnn_cls_loss,
'rcnn_reg_loss': rcnn_reg_loss
})
return loss_dict
def loss(self, prediction_dict, feed_dict):
# import ipdb
# ipdb.set_trace()
loss_dict = {}
anchors = prediction_dict['anchors']
anchors_dict = {}
anchors_dict[constants.KEY_PRIMARY] = anchors
anchors_dict[
constants.KEY_BOXES_2D] = prediction_dict['rpn_bbox_preds']
anchors_dict[constants.KEY_CLASSES] = prediction_dict['rpn_cls_scores']
anchors_dict[
constants.KEY_CORNERS_3D_GRNET] = prediction_dict['corners_3d']
gt_dict = {}
gt_dict[constants.KEY_PRIMARY] = feed_dict[
constants.KEY_LABEL_BOXES_2D]
gt_dict[constants.KEY_CLASSES] = None
gt_dict[constants.KEY_BOXES_2D] = None
gt_dict[constants.KEY_CORNERS_3D_GRNET] = None
auxiliary_dict = {}
auxiliary_dict[constants.KEY_BOXES_2D] = feed_dict[
constants.KEY_LABEL_BOXES_2D]
gt_labels = feed_dict[constants.KEY_LABEL_CLASSES]
auxiliary_dict[constants.KEY_CLASSES] = torch.ones_like(gt_labels)
auxiliary_dict[constants.KEY_NUM_INSTANCES] = feed_dict[
constants.KEY_NUM_INSTANCES]
auxiliary_dict[constants.KEY_PROPOSALS] = anchors
auxiliary_dict[constants.KEY_BOXES_3D] = feed_dict[
constants.KEY_LABEL_BOXES_3D]
auxiliary_dict[constants.KEY_STEREO_CALIB_P2] = feed_dict[
constants.KEY_STEREO_CALIB_P2]
# import ipdb
# ipdb.set_trace()
subsample = not self.use_focal_loss
_, targets, _ = self.target_generators.generate_targets(
anchors_dict, gt_dict, auxiliary_dict, subsample=subsample)
cls_target = targets[constants.KEY_CLASSES]
reg_target = targets[constants.KEY_BOXES_2D]
# loss
if self.use_focal_loss:
# when using focal loss, dont normalize it by all samples
cls_targets = cls_target['target']
pos = cls_targets > 0 # [N,#anchors]
num_pos = pos.long().sum().clamp(min=1).float()
rpn_cls_loss = common_loss.calc_loss(
self.rpn_cls_loss, cls_target, normalize=False) / num_pos
else:
rpn_cls_loss = common_loss.calc_loss(self.rpn_cls_loss, cls_target)
rpn_reg_loss = common_loss.calc_loss(self.rpn_bbox_loss, reg_target)
loss_dict.update({
'rpn_cls_loss': rpn_cls_loss,
'rpn_reg_loss': rpn_reg_loss
})
# return loss_dict
# super().loss(prediction_dict, feed_dict)
# proposals = prediction_dict[constants.KEY_PROPOSALS]
proposals = anchors_dict[constants.KEY_PRIMARY]
p2 = feed_dict[constants.KEY_STEREO_CALIB_P2]
image_info = feed_dict[constants.KEY_IMAGE_INFO]
mean_dims = torch.tensor([1.8, 1.8, 3.7]).type_as(proposals)
corners_2d_loss = 0
center_depth_loss = 0
location_loss = 0
corners_target = targets[constants.KEY_CORNERS_3D_GRNET]
# rcnn_corners_loss = rcnn_corners_loss + common_loss.calc_loss(
# self.rcnn_corners_loss, orient_target, True)
preds = corners_target['pred']
targets = corners_target['target']
weights = corners_target['weight']
weights = weights.unsqueeze(-1)
local_corners_gt = targets[:, :, :24]
location_gt = targets[:, :, 24:27]
dims_gt = targets[:, :, 27:]
N, M = local_corners_gt.shape[:2]
global_corners_gt = (local_corners_gt.view(N, M, 8, 3) +
location_gt.view(N, M, 1, 3)).view(N, M, -1)
center_depth_gt = location_gt[:, :, 2:]
dims_preds = torch.exp(preds[:, :, :3]) * mean_dims
# import ipdb
# ipdb.set_trace()
dims_loss = self.l1_loss(dims_preds, dims_gt) * weights
ry_preds = preds[:, :, 3:4]
# ray_angle = -torch.atan2(location_gt[:, :, 2], location_gt[:, :, 0])
# ry_preds = ry_preds + ray_angle.unsqueeze(-1)
local_corners_preds = []
# calc local corners preds
for batch_ind in range(N):
local_corners_preds.append(
self.calc_local_corners(dims_preds[batch_ind].detach(),
ry_preds[batch_ind]))
local_corners_preds = torch.stack(local_corners_preds, dim=0)
center_2d_deltas_preds = preds[:, :, 4:6]
center_depth_preds = preds[:, :, 6:]
# import ipdb
# ipdb.set_trace()
# decode center_2d
proposals_xywh = geometry_utils.torch_xyxy_to_xywh(proposals)
center_depth_init = self.decode_center_depth(dims_preds,
proposals_xywh, p2)
center_depth_preds = center_depth_init * center_depth_preds
center_2d_preds = (center_2d_deltas_preds * proposals_xywh[:, :, 2:] +
proposals_xywh[:, :, :2])
# center_depth_preds_detach = center_depth_preds.detach()
# import ipdb
# ipdb.set_trace()
# use gt depth to cal loss to make sure the gradient smooth
location_preds = []
for batch_ind in range(N):
location_preds.append(
geometry_utils.torch_points_2d_to_points_3d(
center_2d_preds[batch_ind], center_depth_gt[batch_ind],
p2[batch_ind]))
location_preds = torch.stack(location_preds, dim=0)
global_corners_preds = (location_preds.view(N, M, 1, 3) +
local_corners_preds.view(N, M, 8, 3)).view(
N, M, -1)
# import ipdb
# ipdb.set_trace()
# corners depth loss and center depth loss
corners_depth_preds = local_corners_preds.view(N, M, 8, 3)[..., -1]
corners_depth_gt = local_corners_gt.view(N, M, 8, 3)[..., -1]
center_depth_loss = self.l1_loss(center_depth_preds,
center_depth_gt) * weights
# location loss
location_loss = self.l1_loss(location_preds, location_gt) * weights
# global corners loss
global_corners_loss = self.l1_loss(global_corners_preds,
global_corners_gt) * weights
# proj 2d loss
corners_2d_preds = []
corners_2d_gt = []
for batch_ind in range(N):
corners_2d_preds.append(
geometry_utils.torch_points_3d_to_points_2d(
global_corners_preds[batch_ind].view(-1, 3),
p2[batch_ind]))
corners_2d_gt.append(
geometry_utils.torch_points_3d_to_points_2d(
global_corners_gt[batch_ind].view(-1, 3), p2[batch_ind]))
corners_2d_preds = torch.stack(corners_2d_preds, dim=0).view(N, M, -1)
corners_2d_gt = torch.stack(corners_2d_gt, dim=0).view(N, M, -1)
# image filter
# import ipdb
# ipdb.set_trace()
zeros = torch.zeros_like(image_info[:, 0])
image_shape = torch.stack(
[zeros, zeros, image_info[:, 1], image_info[:, 0]], dim=-1)
image_shape = image_shape.type_as(corners_2d_gt).view(-1, 4)
image_filter = geometry_utils.torch_window_filter(
corners_2d_gt.view(N, -1, 2), image_shape,
deltas=200).float().view(N, M, -1)
# import ipdb
# ipdb.set_trace()
encoded_corners_2d_gt = corners_2d_gt.view(N, M, 8, 2)
encoded_corners_2d_preds = corners_2d_preds.view(N, M, 8, 2)
corners_2d_loss = self.l2_loss(encoded_corners_2d_preds.view(
N, M, -1), encoded_corners_2d_gt.view(N, M, -1)) * weights
corners_2d_loss = (corners_2d_loss.view(N, M, 8, 2) *
image_filter.unsqueeze(-1))
# import ipdb
# ipdb.set_trace()
# mask = self.select_corners(global_corners_gt)
# mask = mask.unsqueeze(-1).expand_as(corners_2d_loss).float()
corners_2d_loss = corners_2d_loss.view(N, M, -1)
corners_depth_loss = self.l1_loss(
corners_depth_preds, corners_depth_gt) * weights * image_filter
# import ipdb
# ipdb.set_trace()
# corners_3d_gt = []
# for batch_ind in range(N):
# corners_3d_gt.append(
# geometry_utils.torch_points_2d_to_points_3d(
# corners_2d_preds[batch_ind].view(-1, 2),
# corners_depth_preds[batch_ind].view(-1), p2[batch_ind]))
# corners_3d_gt = torch.stack(corners_3d_gt, dim=0).view(N, M, -1)
# dim_target = targets[stage_ind][3]
# rcnn_dim_loss = rcnn_dim_loss + common_loss.calc_loss(
# self.rcnn_bbox_loss, dim_target, True)
global_corners_loss = self.l1_loss(global_corners_preds,
global_corners_gt) * weights
# local_corners_loss = self.l1_loss(local_corners_preds,
# local_corners_gt) * weights
# import ipdb
# ipdb.set_trace()
num_pos = (weights > 0).long().sum().clamp(min=1).float()
loss_dict.update({
# 'global_corners_loss': global_corners_loss,
# 'local_corners_loss': local_corners_loss * 10,
'corners_2d_loss': corners_2d_loss,
# 'center_depth_loss': center_depth_loss,
# 'location_loss': location_loss,
# 'corners_depth_loss': corners_depth_loss * 10,
# 'rcnn_corners_loss': rcnn_corners_loss,
# 'rcnn_dim_loss': rcnn_dim_loss
# 'dims_loss': dims_loss
})
return loss_dict