def process_input_eval(self, left_inputs, right_inputs, targets, threshold=0.7, padding=1):
depth_maps = []
mask_pred_list = []
fus = []
for left_prediction, right_prediction, target in zip(left_inputs, right_inputs, targets):
left_bbox = left_prediction.bbox
right_bbox = right_prediction.bbox
disparity_preds = left_prediction.get_field('disparity')
masks = left_prediction.get_field('mask')
masker = Masker(threshold=threshold, padding=padding)
mask_pred = masker([masks], [left_prediction])[0].squeeze(1)
# assert len(left_bbox) == len(right_bbox) == len(
# disparity_preds), f'{len(left_bbox), len(right_bbox), len(disparity_preds)}'
num_rois = len(left_bbox)
fus.extend([target.get_field('calib').calib.fu for _ in range(num_rois)])
depth_maps_per_img = []
disparity_maps_per_img = []
if num_rois != 0:
for left_roi, right_roi, disp_or_depth_roi, mask_p in zip(left_bbox, right_bbox,
disparity_preds, mask_pred):
x1, y1, x2, y2 = expand_box_to_integer(left_roi.tolist())
x1p, _, x2p, _ = expand_box_to_integer(right_roi.tolist())
depth_map_per_roi = torch.zeros((left_prediction.height, left_prediction.width)).cuda()
disparity_map_per_roi = torch.zeros_like(depth_map_per_roi)
mask = mask_p.squeeze(0)
disp_roi = DisparityMap(disp_or_depth_roi).resize(
(max(x2 - x1, x2p - x1p), y2 - y1)).crop(
(0, 0, x2 - x1, y2 - y1)).data
disp_roi = disp_roi + x1 - x1p
depth_roi = target.get_field('calib').stereo_fuxbaseline / (disp_roi + 1e-6)
depth_map_per_roi[y1:y2, x1:x2] = depth_roi.clamp(min=1.0)
disparity_map_per_roi[y1:y2, x1:x2] = disp_roi
disparity_map_per_roi = disparity_map_per_roi * mask.float().cuda()
# imageio.imsave('~/code/disprcnn_plus/tmp.jpg', depth_map_per_roi.cpu().numpy())
depth_maps_per_img.append(depth_map_per_roi)
disparity_maps_per_img.append(disparity_map_per_roi)
if len(depth_maps_per_img) != 0:
depth_maps_per_img = torch.stack(depth_maps_per_img)
disparity_maps_per_img = torch.stack(disparity_maps_per_img).sum(dim=0)
else:
depth_maps_per_img = torch.zeros((1, left_prediction.height, left_prediction.width))
disparity_maps_per_img = torch.zeros((left_prediction.height, left_prediction.width))
depth_maps.append(depth_maps_per_img)
mask_pred_list.append(mask_pred.cuda())
if len(depth_maps) != 0:
fus = torch.tensor(fus).cuda()
self.rotator = rotate_pc_along_y(left_inputs, fus)
pts = self.back_project(depth_maps, mask_pred_list, targets=targets, fix_seed=True)
pts = self.rotator.__call__(pts.permute(0, 2, 1)).permute(0, 2, 1) # Transformation of view cone of point cloud
# pts_tmp = pts.cpu().numpy()
# with open('/home/liangzx/code/disprcnn_plus/tmp2.obj', 'w+') as f:
# for i in range(pts_tmp.shape[1]):
# f.write("v" + " " + str(pts_tmp[0,i,0]) + " " + str(pts_tmp[0,i,1]) + " " + str(pts_tmp[0,i,2]) + "\n")
pts_mean = pts.mean(1)
self.pts_mean = pts_mean
pts = pts - pts_mean[:, None, :]
else:
pts = torch.empty((0, 768, 3)).cuda()
return pts
def process_input(self,
left_inputs,
right_inputs,
targets,
threshold=0.7,
padding=1):
left_inputs, right_inputs = remove_empty_proposals(
left_inputs, right_inputs)
left_inputs, right_inputs = remove_too_right_proposals(
left_inputs, right_inputs)
depth_maps = []
mask_pred_list = []
matched_targets = []
fus = []
for left_prediction, right_prediction, target_per_image in zip(
left_inputs, right_inputs, targets):
if len(target_per_image) != 0:
matched_target = self.match_targets_to_proposals(
left_prediction, target_per_image)
matched_targets.append(matched_target)
else:
continue
left_bbox = left_prediction.bbox
right_bbox = right_prediction.bbox
disparity_or_depth_preds = left_prediction.get_field('disparity')
masks = left_prediction.get_field('mask')
masker = Masker(threshold=threshold, padding=padding)
mask_pred = masker([masks], [left_prediction])[0].squeeze(1)
num_rois = len(left_bbox)
fus.extend([
target_per_image.get_field('calib').calib.fu
for _ in range(num_rois)
])
depth_maps_per_img = []
# mask_preds_per_img = []
if num_rois != 0:
for left_roi, right_roi, disp_or_depth_roi, maskp in zip(
left_bbox, right_bbox, disparity_or_depth_preds,
mask_pred):
x1, y1, x2, y2 = expand_box_to_integer(left_roi.tolist())
x1p, _, x2p, _ = expand_box_to_integer(right_roi.tolist())
depth_map_per_roi = torch.zeros(
(left_prediction.height,
left_prediction.width)).cuda()
disp_roi = DisparityMap(disp_or_depth_roi).resize(
(max(x2 - x1, x2p - x1p), y2 - y1)).crop(
(0, 0, x2 - x1, y2 - y1)).data
disp_roi = disp_roi + x1 - x1p
depth_roi = target_per_image.get_field(
'calib').stereo_fuxbaseline / (disp_roi + 1e-6)
depth_map_per_roi[y1:y2, x1:x2] = depth_roi
depth_maps_per_img.append(depth_map_per_roi)
depth_maps.append(depth_maps_per_img)
mask_pred_list.append(mask_pred.cuda())
depth_full_image = [torch.stack(d) for d in depth_maps]
mask_pred_all = mask_pred_list
pts = self.back_project(depth_full_image, mask_pred_all, targets)
fus = torch.tensor(fus).cuda()
gt_box3d_xyzhwlry = torch.cat([
t.get_field('box3d').convert('xyzhwl_ry').bbox_3d.view(-1, 7)
for t in matched_targets
])
# aug
# scale
if not self.cfg.RPN.FIXED:
scale = np.random.uniform(0.95, 1.05)
pts = pts * scale
gt_box3d_xyzhwlry[:, 0:6] = gt_box3d_xyzhwlry[:, 0:6] * scale
# flip
if not self.cfg.RPN.FIXED:
do_flip = np.random.random() < 0.5
else:
do_flip = False
if do_flip:
pts[:, :, 0] = -pts[:, :, 0]
gt_box3d_xyzhwlry[:, 0] = -gt_box3d_xyzhwlry[:, 0]
gt_box3d_xyzhwlry[:, 6] = torch.sign(
gt_box3d_xyzhwlry[:, 6]) * np.pi - gt_box3d_xyzhwlry[:, 6]
# rotate
self.rotator = rotate_pc_along_y(left_inputs, fus)
self.rotator.rot_angle *= -1
else:
# rotate
self.rotator = rotate_pc_along_y(left_inputs, fus)
gt_box3d_xyzhwlry_batch_splited = torch.split(
gt_box3d_xyzhwlry, [len(b) for b in matched_targets])
for i in range(len(matched_targets)):
matched_targets[i].extra_fields['box3d'] = matched_targets[
i].extra_fields['box3d'].convert('xyzhwl_ry')
matched_targets[i].extra_fields[
'box3d'].bbox_3d = gt_box3d_xyzhwlry_batch_splited[i]
# rotate
pts = self.rotator.__call__(pts.permute(0, 2, 1)).permute(0, 2, 1)
target_corners = self.rotator.__call__(
torch.cat([
t.get_field('box3d').convert('corners').bbox_3d.view(
-1, 8, 3).permute(0, 2, 1) for t in matched_targets
])).permute(0, 2, 1)
# translate
pts_mean = pts.mean(1)
self.pts_mean = pts_mean
pts = pts - pts_mean[:, None, :]
target_corners = target_corners - pts_mean[:, None, :]
target_corners_splited = torch.split(target_corners,
[len(b) for b in matched_targets])
for i in range(len(matched_targets)):
matched_targets[i].extra_fields['box3d'] = matched_targets[
i].extra_fields['box3d'].convert('corners')
matched_targets[i].extra_fields[
'box3d'].bbox_3d = target_corners_splited[i].contiguous().view(
-1, 24)
cls_label, reg_label = generate_rpn_training_labels(
pts, matched_targets)
return pts, cls_label, reg_label, matched_targets