def generate_rpn_training_labels(pts_rect, gt_boxes3d):
cls_label = np.zeros((pts_rect.shape[0]), dtype=np.int32)
reg_label = np.zeros((pts_rect.shape[0], 7),
dtype=np.float32) # dx, dy, dz, ry, h, w, l
gt_corners = kitti_utils.boxes3d_to_corners3d(gt_boxes3d, rotate=True)
extend_gt_boxes3d = kitti_utils.enlarge_box3d(gt_boxes3d, extra_width=0.2)
extend_gt_corners = kitti_utils.boxes3d_to_corners3d(extend_gt_boxes3d,
rotate=True)
for k in range(gt_boxes3d.shape[0]):
box_corners = gt_corners[k]
fg_pt_flag = in_hull(pts_rect, box_corners)
fg_pts_rect = pts_rect[fg_pt_flag]
cls_label[fg_pt_flag] = 1
# enlarge the bbox3d, ignore nearby points
extend_box_corners = extend_gt_corners[k]
fg_enlarge_flag = in_hull(pts_rect, extend_box_corners)
ignore_flag = np.logical_xor(fg_pt_flag, fg_enlarge_flag)
cls_label[ignore_flag] = -1
# pixel offset of object center
center3d = gt_boxes3d[k][0:3].copy() # (x, y, z)
center3d[1] -= gt_boxes3d[k][3] / 2
reg_label[
fg_pt_flag, 0:
3] = center3d - fg_pts_rect # Now y is the true center of 3d box 20180928
# size and angle encoding
reg_label[fg_pt_flag, 3] = gt_boxes3d[k][3] # h
reg_label[fg_pt_flag, 4] = gt_boxes3d[k][4] # w
reg_label[fg_pt_flag, 5] = gt_boxes3d[k][5] # l
reg_label[fg_pt_flag, 6] = gt_boxes3d[k][6] # ry
return cls_label, reg_label
def simple_roipool3d_gpu(pts,
pts_feature,
boxes3d,
pool_extra_width,
sampled_pt_num=512):
"""
:param pts: (B, N, 3)
:param pts_feature: (B, N, C)
:param boxes3d: (B, M, 7)
:param pool_extra_width: float
:param sampled_pt_num: int
:return:
pooled_features: (B, M, 512, 3 + C)
pooled_empty_flag: (B, M)
"""
batch_size, boxes_num, feature_len = pts.shape[0], boxes3d.shape[
1], pts_feature.shape[2]
pooled_boxes3d = kitti_utils.enlarge_box3d(boxes3d.view(-1, 7),
pool_extra_width).view(
batch_size, -1, 7)
pts_idx = torch.cuda.IntTensor(
torch.Size((batch_size, boxes_num, sampled_pt_num))).zero_()
pooled_empty_flag = torch.cuda.IntTensor(
torch.Size((batch_size, boxes_num))).zero_()
simple_roipool3d_cuda.forward(pts.contiguous(),
pooled_boxes3d.contiguous(),
pts_feature.contiguous(), pts_idx,
pooled_empty_flag)
return pts_idx, pooled_empty_flag
def roipool3d_cpu(boxes3d,
pts,
pts_feature,
pts_extra_input,
pool_extra_width,
sampled_pt_num=512,
canonical_transform=True):
"""
:param boxes3d: (N, 7)
:param pts: (N, 3)
:param pts_feature: (N, C)
:param pts_extra_input: (N, C2)
:param pool_extra_width: constant
:param sampled_pt_num: constant
:return:
"""
pooled_boxes3d = kitti_utils.enlarge_box3d(boxes3d, pool_extra_width)
pts_feature_all = np.concatenate((pts_extra_input, pts_feature), axis=1)
# Note: if pooled_empty_flag[i] > 0, the pooled_pts[i], pooled_features[i] will be zero
pooled_pts, pooled_features, pooled_empty_flag = \
roipool_pc_cpu(torch.from_numpy(pts), torch.from_numpy(pts_feature_all),
torch.from_numpy(pooled_boxes3d), sampled_pt_num)
extra_input_len = pts_extra_input.shape[1]
sampled_pts_input = torch.cat(
(pooled_pts, pooled_features[:, :, 0:extra_input_len]), dim=2).numpy()
sampled_pts_feature = pooled_features[:, :, extra_input_len:].numpy()
if canonical_transform:
# Translate to the roi coordinates
roi_ry = boxes3d[:, 6] % (2 * np.pi) # 0~2pi
roi_center = boxes3d[:, 0:3]
# shift to center
sampled_pts_input[:, :,
0:3] = sampled_pts_input[:, :, 0:
3] - roi_center[:, np.
newaxis, :]
for k in range(sampled_pts_input.shape[0]):
sampled_pts_input[k] = kitti_utils.rotate_pc_along_y(
sampled_pts_input[k], roi_ry[k])
return sampled_pts_input, sampled_pts_feature
return sampled_pts_input, sampled_pts_feature, pooled_empty_flag.numpy()
def get_rcnn_sample_info(self, roi_info):
sample_id, gt_box3d = roi_info['sample_id'], roi_info['gt_box3d']
rpn_xyz, rpn_features, rpn_intensity, seg_mask = self.rpn_feature_list[sample_id]
# augmentation original roi by adding noise
roi_box3d = self.aug_roi_by_noise(roi_info)
# point cloud pooling based on roi_box3d
pooled_boxes3d = kitti_utils.enlarge_box3d(roi_box3d.reshape(1, 7), cfg.RCNN.POOL_EXTRA_WIDTH)
boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(rpn_xyz),
torch.from_numpy(pooled_boxes3d))
pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1)
cur_pts = rpn_xyz[pt_mask_flag].astype(np.float32)
# data augmentation
aug_pts = cur_pts.copy()
aug_gt_box3d = gt_box3d.copy().astype(np.float32)
aug_roi_box3d = roi_box3d.copy()
if cfg.AUG_DATA and self.mode == 'TRAIN':
# calculate alpha by ry
temp_boxes3d = np.concatenate([aug_roi_box3d.reshape(1, 7), aug_gt_box3d.reshape(1, 7)], axis=0)
temp_x, temp_z, temp_ry = temp_boxes3d[:, 0], temp_boxes3d[:, 2], temp_boxes3d[:, 6]
temp_beta = np.arctan2(temp_z, temp_x).astype(np.float64)
temp_alpha = -np.sign(temp_beta) * np.pi / 2 + temp_beta + temp_ry
# data augmentation
aug_pts, aug_boxes3d, aug_method = self.data_augmentation(aug_pts, temp_boxes3d, temp_alpha, mustaug=True, stage=2)
aug_roi_box3d, aug_gt_box3d = aug_boxes3d[0], aug_boxes3d[1]
aug_gt_box3d = aug_gt_box3d.astype(gt_box3d.dtype)
# Pool input points
valid_mask = 1 # whether the input is valid
if aug_pts.shape[0] == 0:
pts_features = np.zeros((1, 128), dtype=np.float32)
input_channel = 3 + int(cfg.RCNN.USE_INTENSITY) + int(cfg.RCNN.USE_MASK) + int(cfg.RCNN.USE_DEPTH)
pts_input = np.zeros((1, input_channel), dtype=np.float32)
valid_mask = 0
else:
pts_features = rpn_features[pt_mask_flag].astype(np.float32)
pts_intensity = rpn_intensity[pt_mask_flag].astype(np.float32)
pts_input_list = [aug_pts, pts_intensity.reshape(-1, 1)]
if cfg.RCNN.USE_INTENSITY:
pts_input_list = [aug_pts, pts_intensity.reshape(-1, 1)]
else:
pts_input_list = [aug_pts]
if cfg.RCNN.USE_MASK:
if cfg.RCNN.MASK_TYPE == 'seg':
pts_mask = seg_mask[pt_mask_flag].astype(np.float32)
elif cfg.RCNN.MASK_TYPE == 'roi':
pts_mask = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(aug_pts),
torch.from_numpy(aug_roi_box3d.reshape(1, 7)))
pts_mask = (pts_mask[0].numpy() == 1).astype(np.float32)
else:
raise NotImplementedError
pts_input_list.append(pts_mask.reshape(-1, 1))
if cfg.RCNN.USE_DEPTH:
pts_depth = np.linalg.norm(aug_pts, axis=1, ord=2)
pts_depth_norm = (pts_depth / 70.0) - 0.5
pts_input_list.append(pts_depth_norm.reshape(-1, 1))
pts_input = np.concatenate(pts_input_list, axis=1) # (N, C)
aug_gt_corners = kitti_utils.boxes3d_to_corners3d(aug_gt_box3d.reshape(-1, 7))
aug_roi_corners = kitti_utils.boxes3d_to_corners3d(aug_roi_box3d.reshape(-1, 7))
iou3d = kitti_utils.get_iou3d(aug_roi_corners, aug_gt_corners)
cur_iou = iou3d[0][0]
# regression valid mask
reg_valid_mask = 1 if cur_iou >= cfg.RCNN.REG_FG_THRESH and valid_mask == 1 else 0
# classification label
cls_label = 1 if cur_iou > cfg.RCNN.CLS_FG_THRESH else 0
if cfg.RCNN.CLS_BG_THRESH < cur_iou < cfg.RCNN.CLS_FG_THRESH or valid_mask == 0:
cls_label = -1
# canonical transform and sampling
pts_input_ct, gt_box3d_ct = self.canonical_transform(pts_input, aug_roi_box3d, aug_gt_box3d)
pts_input_ct, pts_features = self.rcnn_input_sample(pts_input_ct, pts_features)
sample_info = {'sample_id': sample_id,
'pts_input': pts_input_ct,
'pts_features': pts_features,
'cls_label': cls_label,
'reg_valid_mask': reg_valid_mask,
'gt_boxes3d_ct': gt_box3d_ct,
'roi_boxes3d': aug_roi_box3d,
'roi_size': aug_roi_box3d[3:6],
'gt_boxes3d': aug_gt_box3d}
return sample_info
