def _forward_val(self, left_results, right_results, targets):
left_results, right_results = remove_empty_proposals(left_results, right_results)
pts_input = self.process_input_eval(left_results, right_results, targets,
threshold=0.5)
if pts_input.numel() != 0:
rpn_proposals, proposals_losses = self.rpn(pts_input)
pts_input = self.rotator.rotate_back(
(pts_input + self.pts_mean[:, None, :]).permute(0, 2, 1)
).permute(0, 2, 1).contiguous()
if hasattr(self, 'rcnn_net'):
rpn_proposals['backbone_xyz'] = self.rotator.rotate_back(
(rpn_proposals['backbone_xyz'] + self.pts_mean[:, None, :]).permute(0, 2, 1)
).permute(0, 2, 1)
rpn_proposals['pts_depth'] = torch.norm(rpn_proposals['backbone_xyz'], p=2, dim=2)
rpn_proposals['rpn_xyz'] = self.rotator.rotate_back(
(rpn_proposals['rpn_xyz'] + self.pts_mean[:, None, :]).permute(0, 2, 1)
).permute(0, 2, 1)
corners = Box3DList(rpn_proposals['roi_boxes3d'].view(-1, 7), (1, 1), 'xyzhwl_ry').convert(
'corners').bbox_3d.view(
pts_input.shape[0], -1, 24) # numroi,?,24
rpn_proposals['roi_boxes3d'] = Box3DList((self.rotator.rotate_back(
(corners.view(pts_input.shape[0], -1, 3) + self.pts_mean[:, None, :]
).permute(0, 2, 1)).permute(0, 2, 1)).contiguous(),
(1, 1),
'corners').convert('xyzhwl_ry'
).bbox_3d.view(pts_input.shape[0],
-1, 7)
proposals, rcnn_loss = self.rcnn_net(rpn_proposals)
left_results = combine_2d_3d(left_results, proposals)
else:
# inference with rpn
box3d = rpn_proposals['roi_boxes3d'][:, 0] # 1,7
box3d[:, 0:3] = box3d[:, 0:3] + self.pts_mean
corners_rot_back = self.rotator.rotate_back(
Box3DList(box3d,
size=(1280, 720),
mode='xyzhwl_ry').convert('corners').
bbox_3d.view(-1, 8, 3).permute(0, 2, 1)).permute(0, 2, 1)
score_3d = rpn_proposals['roi_scores_raw'][:, 0]
left_results[0].add_field('box3d',
Box3DList(corners_rot_back.reshape(-1, 24),
(1280, 720), 'corners').convert(
'xyzhwl_ry')
)
left_results[0].add_field('scores_3d', score_3d.cpu())
else:
for left_result in left_results:
left_result.add_field('box3d',
Box3DList(torch.empty((0, 7)), size=left_result.size, mode='ry_lhwxyz'))
left_result.add_field('scores_3d', torch.empty((0)))
return left_results, right_results, {}
def _forward_train(self, left_inputs, right_inputs, targets):
pts_input, rpn_cls_label, \
rpn_reg_label, matched_targets = self.process_input(
left_inputs, right_inputs, targets, self.masker_threshold)
# norm
loss_dict = {}
with torch.set_grad_enabled((not self.cfg.RPN.FIXED)
and self.training):
if self.cfg.RPN.FIXED:
self.rpn.eval()
proposals, proposals_losses = self.rpn(pts_input, rpn_cls_label,
rpn_reg_label,
matched_targets)
loss_dict.update(proposals_losses)
if self.cfg.RCNN.ENABLED and self.cfg.RCNN.TRAIN:
pts_input = pts_input + self.pts_mean[:, None, :]
pts_input = self.rotator.rotate_back(pts_input.permute(0, 2,
1)).permute(
0, 2, 1)
matched_targets[0].extra_fields['box3d'] = matched_targets[
0].extra_fields['box3d'].convert('corners')
matched_targets[0].extra_fields['box3d'].bbox_3d = (
self.rotator.rotate_back(
(matched_targets[0].extra_fields['box3d'].bbox_3d.view(
-1, 8, 3) + self.pts_mean[:, None, :]).permute(
0, 2, 1)).permute(0, 2, 1)).reshape(-1, 24)
proposals['backbone_xyz'] = self.rotator.rotate_back(
(proposals['backbone_xyz'] +
self.pts_mean[:, None, :]).permute(0, 2, 1)).permute(0, 2, 1)
proposals['pts_depth'] = torch.norm(proposals['backbone_xyz'],
p=2,
dim=2)
proposals['rpn_xyz'] = self.rotator.rotate_back(
(proposals['rpn_xyz'] + self.pts_mean[:, None, :]).permute(
0, 2, 1)).permute(0, 2, 1)
corners = Box3DList(proposals['roi_boxes3d'].view(-1, 7), (1, 1),
'xyzhwl_ry').convert('corners').bbox_3d.view(
pts_input.shape[0], -1, 24) # numroi,?,24
proposals['roi_boxes3d'] = Box3DList(
(self.rotator.rotate_back(
(corners.view(pts_input.shape[0], -1, 3) +
self.pts_mean[:, None, :]).permute(0, 2, 1)).permute(
0, 2, 1)).contiguous(), (1, 1),
'corners').convert('xyzhwl_ry').bbox_3d.view(
pts_input.shape[0], -1, 7)
proposals, matched_targets = filter_unmatched_idxs(
proposals, matched_targets)
proposals, rcnn_loss = self.rcnn_net(proposals, matched_targets)
loss_dict.update(rcnn_loss)
return proposals, loss_dict
def combine_2d_3d(left_inputs, proposals):
box_3ds_batch, scores = [], []
randoms = []
for proposal in proposals:
score = proposal.get_field('box3d_score')
box3d = proposal.get_field('box3d')
is_random = proposal.get_field('random')
maxidx = score.argmax()
box3d = box3d.bbox_3d[maxidx]
scores.append(score[maxidx].cpu())
box_3ds_batch.append(box3d.cpu())
randoms.append(is_random[maxidx].cpu())
box_3ds_batch = torch.stack(box_3ds_batch)
scores = torch.stack(scores)
randoms = torch.stack(randoms)
box_3ds_batch = torch.split(box_3ds_batch, [len(a) for a in left_inputs])
scores = torch.split(scores, [len(a) for a in left_inputs])
randoms = torch.split(randoms, [len(a) for a in left_inputs])
for left_input, box3d, score_3d, is_rand in zip(left_inputs, box_3ds_batch,
scores, randoms):
box3d = Box3DList(box3d, size=left_input.size, mode='ry_lhwxyz')
left_input.add_field('box3d', box3d)
left_input.add_field('scores_3d', score_3d)
left_input.add_field('random', is_rand)
return left_inputs
def get_ground_truth(self, index):
img_id = self.ids[index]
if not is_testing_split(self.split):
left_annotation = self.annotations['left'][int(img_id)]
right_annotation = self.annotations['right'][int(img_id)]
info = self.get_img_info(index)
height, width = info['height'], info['width']
# left target
left_target = BoxList(left_annotation["boxes"], (width, height), mode="xyxy")
left_target.add_field("labels", left_annotation["labels"])
# left_target.add_field("alphas", left_annotation['alphas'])
boxes_3d = Box3DList(left_annotation["boxes_3d"], (width, height), mode='ry_lhwxyz')
left_target.add_field("box3d", boxes_3d)
left_target.add_map('disparity', self.get_disparity(index))
left_target.add_field('kins_masks', self.get_kins_mask(index))
left_target.add_field('truncation', torch.tensor(self.truncations_list[int(img_id)]))
left_target.add_field('occlusion', torch.tensor(self.occlusions_list[int(img_id)]))
left_target.add_field('image_size', torch.tensor([[width, height]]).repeat(len(left_target), 1))
left_target.add_field('calib', Calib(self.get_calibration(index), (width, height)))
left_target.add_field('index', torch.full((len(left_target), 1), index, dtype=torch.long))
left_target.add_field('imgid', torch.full((len(left_target), 1), int(img_id), dtype=torch.long))
left_target = left_target.clip_to_image(remove_empty=True)
# right target
right_target = BoxList(right_annotation["boxes"], (width, height), mode="xyxy")
right_target.add_field("labels", right_annotation["labels"])
right_target = right_target.clip_to_image(remove_empty=True)
target = {'left': left_target, 'right': right_target}
return target
else:
fakebox = torch.tensor([[0, 0, 0, 0]])
info = self.get_img_info(index)
height, width = info['height'], info['width']
# left target
left_target = BoxList(fakebox, (width, height), mode="xyxy")
left_target.add_field('image_size', torch.tensor([[width, height]]).repeat(len(left_target), 1))
left_target.add_field('calib', Calib(self.get_calibration(index), (width, height)))
left_target.add_field('index', torch.full((len(left_target), 1), index, dtype=torch.long))
left_target.add_field('imgid', torch.full((len(left_target), 1), int(img_id), dtype=torch.long))
# right target
right_target = BoxList(fakebox, (width, height), mode="xyxy")
target = {'left': left_target, 'right': right_target}
return target
def cat_boxlist3d(bboxes):
"""
Concatenates a list of BoxList (having the same image size) into a
single BoxList
Arguments:
bboxes (list[Box3DList])
"""
assert isinstance(bboxes, (list, tuple))
assert all(isinstance(bbox, Box3DList) for bbox in bboxes)
size = bboxes[0].size
assert all(bbox.size == size for bbox in bboxes)
mode = bboxes[0].mode
assert all(bbox.mode == mode for bbox in bboxes)
cat_boxes = Box3DList(_cat([bbox.bbox_3d for bbox in bboxes], dim=0), size,
mode)
return cat_boxes
def __call__(self, output_dict, proposals):
"""
:param output_dict: dict_keys(['rcnn_cls', 'rcnn_reg'])
:param proposals: dict_keys(['rpn_cls', 'rpn_reg', 'backbone_xyz',
'backbone_features', 'rois', 'roi_scores_raw',
'seg_result', 'rpn_xyz', 'rpn_features',
'seg_mask', 'roi_boxes3d', 'pts_depth'])
:return:
"""
size = (1280, 720) # todo:replace
# batch_size = len(targets)
roi_boxes3d = proposals['roi_boxes3d'] # (B, M, 7)
batch_size = roi_boxes3d.shape[0]
rcnn_cls = output_dict['rcnn_cls'].view(
batch_size, -1, output_dict['rcnn_cls'].shape[1])
rcnn_reg = output_dict['rcnn_reg'].view(
batch_size, -1, output_dict['rcnn_reg'].shape[1]) # (B, M, C)
# bounding box regression
anchor_size = self.MEAN_SIZE
if self.cfg.RCNN.SIZE_RES_ON_ROI:
assert False
pred_boxes3d = decode_bbox_target(
roi_boxes3d.view(-1, 7),
rcnn_reg.view(-1, rcnn_reg.shape[-1]),
anchor_size=anchor_size,
loc_scope=self.cfg.RCNN.LOC_SCOPE,
loc_bin_size=self.cfg.RCNN.LOC_BIN_SIZE,
num_head_bin=self.cfg.RCNN.NUM_HEAD_BIN,
get_xz_fine=True,
get_y_by_bin=self.cfg.RCNN.LOC_Y_BY_BIN,
loc_y_scope=self.cfg.RCNN.LOC_Y_SCOPE,
loc_y_bin_size=self.cfg.RCNN.LOC_Y_BIN_SIZE,
get_ry_fine=True).view(batch_size, -1, 7)
# scoring
if rcnn_cls.shape[2] == 1:
raw_scores = rcnn_cls # (B, M, 1)
norm_scores = torch.sigmoid(raw_scores)
pred_classes = (norm_scores >
self.cfg.RCNN.SCORE_THRESH).long().squeeze(-1)
else:
pred_classes = torch.argmax(rcnn_cls, dim=1).view(-1)
cls_norm_scores = F.softmax(rcnn_cls, dim=1)
raw_scores = rcnn_cls[:, pred_classes]
norm_scores = cls_norm_scores[:, pred_classes]
inds = norm_scores > self.cfg.RCNN.SCORE_THRESH
# inds = norm_scores > 0.05
results = []
for k in range(batch_size):
cur_inds = inds[k].view(-1)
if cur_inds.sum() == 0:
# print('Low scores, random result.')
use_rpn_proposals = True
if not use_rpn_proposals:
bbox_3d = Box3DList(
torch.rand(1, 7).float(), size,
'xyzhwl_ry').convert("ry_lhwxyz")
bbox = torch.Tensor([0, 0, 0, 0]).repeat(1, 1).cuda()
bbox = BoxList(bbox, size, mode="xyxy")
bbox.add_field("box3d", bbox_3d)
# bbox.add_field("box3d_score", torch.Tensor(1).zero_())
bbox.add_field("box3d_score", torch.zeros(1) * (-10))
bbox.add_field("labels", torch.ones(1).cuda())
bbox.add_field("iou_score", torch.Tensor(1).zero_())
bbox.add_field("random", torch.ones((len(bbox))).long())
if self.cfg.RPN.EARLY_INTEGRATE:
bbox.add_field('det_id', torch.Tensor(1).zero_())
bbox.add_field('box3d_backend', bbox_3d)
bbox.add_field('box3d_backend_ids',
torch.Tensor(1).zero_())
bbox.add_field('box3d_backend_keep',
torch.Tensor(1).zero_())
results.append(bbox)
continue
else:
# print('use_rpn_proposals')
proposal_score = proposals['roi_scores_raw'][k]
select_idx = proposal_score.argmax()
b3d = roi_boxes3d[k][select_idx]
bbox_3d = Box3DList(b3d, size,
'xyzhwl_ry').convert("ry_lhwxyz")
bbox = torch.Tensor([0, 0, size[0],
size[1]]).repeat(b3d.shape[0],
1).cuda()
bbox = BoxList(bbox, size, mode="xyxy")
bbox.add_field("box3d", bbox_3d)
# bbox.add_field("box3d_score", torch.Tensor([proposal_score[select_idx]]))
bbox.add_field("box3d_score", torch.zeros(1))
bbox.add_field("labels", 1)
bbox.add_field("random", torch.ones((len(bbox))).long())
# bbox.add_field('iou_score', scores_selected)
results.append(bbox)
continue
pred_boxes3d_selected = pred_boxes3d[k, cur_inds]
raw_scores_selected = raw_scores[k, cur_inds]
norm_scores_selected = norm_scores[k, cur_inds]
pred_classes_selected = pred_classes[k, cur_inds]
# NMS thresh
# rotated nms
boxes_bev_selected = boxes3d_to_bev_torch(pred_boxes3d_selected)
keep_idx = nms_gpu(boxes_bev_selected, raw_scores_selected,
self.cfg.RCNN.NMS_THRESH).view(-1)
pred_boxes3d_selected = pred_boxes3d_selected[keep_idx]
scores_selected = raw_scores_selected[keep_idx].squeeze(1)
pred_classes_selected = pred_classes_selected[keep_idx]
bbox_3d = Box3DList(pred_boxes3d_selected, size,
'xyzhwl_ry').convert("ry_lhwxyz")
bbox = torch.Tensor([0, 0, size[0], size[1]
]).repeat(pred_boxes3d_selected.shape[0],
1).cuda()
bbox = BoxList(bbox, size, mode="xyxy")
bbox.add_field("box3d", bbox_3d)
bbox.add_field("box3d_score", scores_selected)
bbox.add_field("labels", pred_classes_selected)
bbox.add_field('iou_score', scores_selected)
bbox.add_field('random', torch.zeros((len(bbox))).long())
results.append(bbox)
return results
def _forward_val(self, left_results, right_results, targets):
left_results, right_results = remove_empty_proposals(
left_results, right_results)
pts_input = self.process_input_eval(left_results,
right_results,
targets,
threshold=0.5)
if pts_input.numel() != 0:
rpn_proposals, proposals_losses = self.rpn(pts_input)
pts_input = self.rotator.rotate_back(
(pts_input + self.pts_mean[:, None, :]).permute(
0, 2, 1)).permute(0, 2, 1).contiguous()
if hasattr(self, 'rcnn_net'):
rpn_proposals['backbone_xyz'] = self.rotator.rotate_back(
(rpn_proposals['backbone_xyz'] +
self.pts_mean[:, None, :]).permute(0, 2,
1)).permute(0, 2, 1)
rpn_proposals['pts_depth'] = torch.norm(
rpn_proposals['backbone_xyz'], p=2, dim=2)
rpn_proposals['rpn_xyz'] = self.rotator.rotate_back(
(rpn_proposals['rpn_xyz'] +
self.pts_mean[:, None, :]).permute(0, 2,
1)).permute(0, 2, 1)
corners = Box3DList(
rpn_proposals['roi_boxes3d'].view(-1, 7), (1, 1),
'xyzhwl_ry').convert('corners').bbox_3d.view(
pts_input.shape[0], -1, 24) # numroi,?,24
rpn_proposals['roi_boxes3d'] = Box3DList(
(self.rotator.rotate_back(
(corners.view(pts_input.shape[0], -1, 3) +
self.pts_mean[:, None, :]).permute(0, 2, 1)).permute(
0, 2, 1)).contiguous(), (1, 1),
'corners').convert('xyzhwl_ry').bbox_3d.view(
pts_input.shape[0], -1, 7)
proposals, rcnn_loss = self.rcnn_net(rpn_proposals)
left_results = combine_2d_3d(left_results, proposals)
else:
# inference with rpn
box3d = rpn_proposals['roi_boxes3d'] # 64,7
bsz = box3d.shape[0]
box3d[:, :, 0:3] = box3d[:, :, 0:3] + self.pts_mean[:, None, :]
corners_rot_back = self.rotator.rotate_back(
Box3DList(
box3d.reshape(-1, 7), size=(1, 1),
mode='xyzhwl_ry').convert('corners').bbox_3d.view(
bsz, -1,
3).permute(0, 2,
1)).permute(0, 2,
1).view(bsz, -1, 8, 3)
# score_3d = rpn_proposals['roi_scores_raw'][:, 0]
score_3d = rpn_proposals['roi_scores_raw']
ss, bs = [], []
for crb, s3d in zip(corners_rot_back, score_3d):
idx = torch.argmax(s3d)
ss.append(s3d[idx])
bs.append(crb[idx])
ss = torch.tensor(ss)
bs = torch.stack(bs)
bs = Box3DList(bs.reshape(-1, 24), (1, 1),
'corners').convert('xyzhwl_ry')
left_results[0].add_field('box3d', bs)
left_results[0].add_field('scores_3d', ss.cpu())
rpn_proposals['roi_boxes3d'] = bs.bbox_3d.unsqueeze(0)
else:
for left_result in left_results:
left_result.add_field(
'box3d',
Box3DList(torch.empty((0, 7)),
size=left_result.size,
mode='ry_lhwxyz'))
left_result.add_field('scores_3d', torch.empty((0)))
return left_results, right_results, {}