def get_rcnn_sample_jit(self, index): sample_id = int(self.sample_id_list[index]) rpn_xyz, rpn_features, rpn_intensity, seg_mask = \ self.get_rpn_features(self.rcnn_training_feature_dir, sample_id) # load rois and gt_boxes3d for this sample roi_file = os.path.join(self.rcnn_training_roi_dir, '%06d.txt' % sample_id) roi_obj_list = kitti_utils.get_objects_from_label(roi_file) roi_boxes3d = kitti_utils.objs_to_boxes3d(roi_obj_list) # roi_scores = kitti_utils.objs_to_scores(roi_obj_list) gt_obj_list = self.filtrate_objects(self.get_label(sample_id)) gt_boxes3d = kitti_utils.objs_to_boxes3d(gt_obj_list) sample_info = { 'sample_id': sample_id, 'rpn_xyz': rpn_xyz, 'rpn_features': rpn_features, 'rpn_intensity': rpn_intensity, 'seg_mask': seg_mask, 'roi_boxes3d': roi_boxes3d, 'gt_boxes3d': gt_boxes3d, 'pts_depth': np.linalg.norm(rpn_xyz, ord=2, axis=1) } return sample_info
def load_label(frame, level_difficulty, class_of_interest, verbose=False): """ Params: frame: frame ID level_difficulty: 1, 2, 3, 4 (Easy, Moderate, Hard, Unknown) class_of_interest: [1,2,3,4] {'Car': 1, 'Pedestrian': 2, 'Cyclist': 3, 'Van': 4} label_stat = {'frame_id', 'truncated', 'occlusion' , 'labels', '3D_bboxes', '2D_bboxes'} 'truncated' : Float from 0 (non-truncated) to 1 (truncated), where truncated refers to the object leaving image boundaries 'occlusion' : Integer (0,1,2,3) indicating occlusion state: 0 = fully visible, 1 = partly occluded, 2 = largely occluded, 3 = unknown 'labels' : class label '3D_bboxes' : [[x,y,z,w,h,l,ry]] in camera coordinates '2D_bboxes' : 2D bounding box of object in the image (0-based index): contains left, top, right, bottom pixel coordinates """ # label_stat = {'frame_id': frame} label_filename = os.path.join(LABEL_ROOT, '{0:06d}.txt'.format(frame)) assert os.path.exists(label_filename) objs = kitti_utils.get_objects_from_label(label_filename) objs = filter_object(objs, level_difficulty, class_of_interest) # for obj in objs: # if obj is not None: # print(obj) return objs
def get_label(self, idx, aug=False): if self.split == 'train_aug': label_file = os.path.join(self.aug_label_dir, '%s.txt' % idx) else: label_file = os.path.join(self.label_dir, '%s.txt' % idx) assert os.path.exists(label_file) return kitti_utils.get_objects_from_label(label_file, self.type_to_id)
def get_label(self, idx): if idx < 10000: label_file = os.path.join(self.label_dir, '%06d.txt' % idx) else: label_file = os.path.join(self.aug_label_dir, '%06d.txt' % idx) assert os.path.exists(label_file) return kitti_utils.get_objects_from_label(label_file)
def get_label(self, idx): label_file = self.label_pathlist[idx] try: assert os.path.exists(label_file) ret = kitti_utils.get_objects_from_label(label_file) except Exception as e: print(e, " No label presnet") ret = None return ret
def fpnet_test(self, input_data, sample_id, rpn_scores_raw): input_data = input_data[0].cpu().numpy() sample_id = sample_id[0] import numpy as np import os fpnet_file = os.path.join('../data/KITTI/frustum_net/data/', '%06d.txt' % sample_id) import lib.utils.kitti_utils as kitti_utils obj_list = kitti_utils.get_objects_from_label(fpnet_file) for i in range(len(input_data)): d = input_data[i] if self.is_input_data_in_box3d(obj_list, d): rpn_scores_raw[0,i] = rpn_scores_raw[0,i] + 0.5 #print("----------HAHAHAHAHAHAHAHA--------------> update rpn_scores_raw") return rpn_scores_raw
def get_label(self, idx): label_file = self.label_pathlist[idx] assert os.path.exists(label_file) return kitti_utils.get_objects_from_label(label_file)
def get_noise_label(self, idx): label_file = os.path.join(self.noise_label_dir, '%06d.txt' % idx) assert os.path.exists(label_file) return kitti_utils.get_objects_from_label(label_file)
def get_label(self, idx): # Check for Kitti Utils assert os.path.exists(self.label_pathlist[idx]) return kitti_utils.get_objects_from_label(self.label_pathlist[idx])
def get_rcnn_training_sample_batch(self, index): sample_id = int(self.sample_id_list[index]) rpn_xyz, rpn_features, rpn_intensity, seg_mask = \ self.get_rpn_features(self.rcnn_training_feature_dir, sample_id) # load rois and gt_boxes3d for this sample roi_file = os.path.join(self.rcnn_training_roi_dir, '%06d.txt' % sample_id) roi_obj_list = kitti_utils.get_objects_from_label(roi_file) roi_boxes3d = kitti_utils.objs_to_boxes3d(roi_obj_list) # roi_scores = kitti_utils.objs_to_scores(roi_obj_list) gt_obj_list = self.filtrate_objects(self.get_label(sample_id)) gt_boxes3d = kitti_utils.objs_to_boxes3d(gt_obj_list) # calculate original iou iou3d = kitti_utils.get_iou3d(kitti_utils.boxes3d_to_corners3d(roi_boxes3d), kitti_utils.boxes3d_to_corners3d(gt_boxes3d)) max_overlaps, gt_assignment = iou3d.max(axis=1), iou3d.argmax(axis=1) max_iou_of_gt, roi_assignment = iou3d.max(axis=0), iou3d.argmax(axis=0) roi_assignment = roi_assignment[max_iou_of_gt > 0].reshape(-1) # sample fg, easy_bg, hard_bg fg_rois_per_lidar = int(np.round(cfg.RCNN.FG_RATIO * cfg.RCNN.ROI_PER_lidar)) fg_thresh = min(cfg.RCNN.REG_FG_THRESH, cfg.RCNN.CLS_FG_THRESH) fg_inds = np.nonzero(max_overlaps >= fg_thresh)[0] fg_inds = np.concatenate((fg_inds, roi_assignment), axis=0) # consider the roi which has max_overlaps with gt as fg easy_bg_inds = np.nonzero((max_overlaps < cfg.RCNN.CLS_BG_THRESH_LO))[0] hard_bg_inds = np.nonzero((max_overlaps < cfg.RCNN.CLS_BG_THRESH) & (max_overlaps >= cfg.RCNN.CLS_BG_THRESH_LO))[0] fg_num_rois = fg_inds.size bg_num_rois = hard_bg_inds.size + easy_bg_inds.size if fg_num_rois > 0 and bg_num_rois > 0: # sampling fg fg_rois_per_this_lidar = min(fg_rois_per_lidar, fg_num_rois) rand_num = np.random.permutation(fg_num_rois) fg_inds = fg_inds[rand_num[:fg_rois_per_this_lidar]] # sampling bg bg_rois_per_this_lidar = cfg.RCNN.ROI_PER_lidar - fg_rois_per_this_lidar bg_inds = self.sample_bg_inds(hard_bg_inds, easy_bg_inds, bg_rois_per_this_lidar) elif fg_num_rois > 0 and bg_num_rois == 0: # sampling fg rand_num = np.floor(np.random.rand(cfg.RCNN.ROI_PER_lidar ) * fg_num_rois) rand_num = torch.from_numpy(rand_num).type_as(gt_boxes3d).long() fg_inds = fg_inds[rand_num] fg_rois_per_this_lidar = cfg.RCNN.ROI_PER_lidar bg_rois_per_this_lidar = 0 elif bg_num_rois > 0 and fg_num_rois == 0: # sampling bg bg_rois_per_this_lidar = cfg.RCNN.ROI_PER_lidar bg_inds = self.sample_bg_inds(hard_bg_inds, easy_bg_inds, bg_rois_per_this_lidar) fg_rois_per_this_lidar = 0 else: import pdb pdb.set_trace() raise NotImplementedError # augment the rois by noise roi_list, roi_iou_list, roi_gt_list = [], [], [] if fg_rois_per_this_lidar > 0: fg_rois_src = roi_boxes3d[fg_inds].copy() gt_of_fg_rois = gt_boxes3d[gt_assignment[fg_inds]] fg_rois, fg_iou3d = self.aug_roi_by_noise_batch(fg_rois_src, gt_of_fg_rois, aug_times=10) roi_list.append(fg_rois) roi_iou_list.append(fg_iou3d) roi_gt_list.append(gt_of_fg_rois) if bg_rois_per_this_lidar > 0: bg_rois_src = roi_boxes3d[bg_inds].copy() gt_of_bg_rois = gt_boxes3d[gt_assignment[bg_inds]] bg_rois, bg_iou3d = self.aug_roi_by_noise_batch(bg_rois_src, gt_of_bg_rois, aug_times=1) roi_list.append(bg_rois) roi_iou_list.append(bg_iou3d) roi_gt_list.append(gt_of_bg_rois) rois = np.concatenate(roi_list, axis=0) iou_of_rois = np.concatenate(roi_iou_list, axis=0) gt_of_rois = np.concatenate(roi_gt_list, axis=0) # collect extra features for point cloud pooling if cfg.RCNN.USE_INTENSITY: pts_extra_input_list = [rpn_intensity.reshape(-1, 1), seg_mask.reshape(-1, 1)] else: pts_extra_input_list = [seg_mask.reshape(-1, 1)] if cfg.RCNN.USE_DEPTH: pts_depth = (np.linalg.norm(rpn_xyz, ord=2, axis=1) / 70.0) - 0.5 pts_extra_input_list.append(pts_depth.reshape(-1, 1)) pts_extra_input = np.concatenate(pts_extra_input_list, axis=1) pts_input, pts_features, pts_empty_flag = roipool3d_utils.roipool3d_cpu(rois, rpn_xyz, rpn_features, pts_extra_input, cfg.RCNN.POOL_EXTRA_WIDTH, sampled_pt_num=cfg.RCNN.NUM_POINTS, canonical_transform=False) valid_mask = (pts_empty_flag == 0).astype(np.int32) # regression valid mask reg_valid_mask = (iou_of_rois > cfg.RCNN.REG_FG_THRESH).astype(np.int32) & valid_mask # classification label cls_label = (iou_of_rois > cfg.RCNN.CLS_FG_THRESH).astype(np.int32) invalid_mask = (iou_of_rois > cfg.RCNN.CLS_BG_THRESH) & (iou_of_rois < cfg.RCNN.CLS_FG_THRESH) cls_label[invalid_mask] = -1 cls_label[valid_mask == 0] = -1 # canonical transform and sampling pts_input_ct, gt_boxes3d_ct = self.canonical_transform_batch(pts_input, rois, gt_of_rois) sample_info = {'sample_id': sample_id, 'pts_input': pts_input_ct, 'pts_features': pts_features, 'cls_label': cls_label, 'reg_valid_mask': reg_valid_mask, 'gt_boxes3d_ct': gt_boxes3d_ct, 'roi_boxes3d': rois, 'roi_size': rois[:, 3:6], 'gt_boxes3d': gt_of_rois} return sample_info
def get_proposal_from_file(self, index): sample_id = int(self.lidar_idx_list[index]) proposal_file = os.path.join(self.rcnn_eval_roi_dir, '%06d.txt' % sample_id) roi_obj_list = kitti_utils.get_objects_from_label(proposal_file) rpn_xyz, rpn_features, rpn_intensity, seg_mask = self.get_rpn_features(self.rcnn_eval_feature_dir, sample_id) pts_rect, pts_rpn_features, pts_intensity = rpn_xyz, rpn_features, rpn_intensity roi_box3d_list, roi_scores = [], [] for obj in roi_obj_list: box3d = np.array([obj.pos[0], obj.pos[1], obj.pos[2], obj.h, obj.w, obj.l, obj.ry], dtype=np.float32) roi_box3d_list.append(box3d.reshape(1, 7)) roi_scores.append(obj.score) roi_boxes3d = np.concatenate(roi_box3d_list, axis=0) # (N, 7) roi_scores = np.array(roi_scores, dtype=np.float32) # (N) if cfg.RCNN.ROI_SAMPLE_JIT: sample_dict = {'sample_id': sample_id, 'rpn_xyz': rpn_xyz, 'rpn_features': rpn_features, 'seg_mask': seg_mask, 'roi_boxes3d': roi_boxes3d, 'roi_scores': roi_scores, 'pts_depth': np.linalg.norm(rpn_xyz, ord=2, axis=1)} if self.mode != 'TEST': gt_obj_list = self.filtrate_objects(self.get_label(sample_id)) gt_boxes3d = kitti_utils.objs_to_boxes3d(gt_obj_list) roi_corners = kitti_utils.boxes3d_to_corners3d(roi_boxes3d) gt_corners = kitti_utils.boxes3d_to_corners3d(gt_boxes3d) iou3d = kitti_utils.get_iou3d(roi_corners, gt_corners) if gt_boxes3d.shape[0] > 0: gt_iou = iou3d.max(axis=1) else: gt_iou = np.zeros(roi_boxes3d.shape[0]).astype(np.float32) sample_dict['gt_boxes3d'] = gt_boxes3d sample_dict['gt_iou'] = gt_iou return sample_dict if cfg.RCNN.USE_INTENSITY: pts_extra_input_list = [pts_intensity.reshape(-1, 1), seg_mask.reshape(-1, 1)] else: pts_extra_input_list = [seg_mask.reshape(-1, 1)] if cfg.RCNN.USE_DEPTH: cur_depth = np.linalg.norm(pts_rect, axis=1, ord=2) cur_depth_norm = (cur_depth / 70.0) - 0.5 pts_extra_input_list.append(cur_depth_norm.reshape(-1, 1)) pts_extra_input = np.concatenate(pts_extra_input_list, axis=1) pts_input, pts_features = roipool3d_utils.roipool3d_cpu(roi_boxes3d, pts_rect, pts_rpn_features, pts_extra_input, cfg.RCNN.POOL_EXTRA_WIDTH, sampled_pt_num=cfg.RCNN.NUM_POINTS) sample_dict = {'sample_id': sample_id, 'pts_input': pts_input, 'pts_features': pts_features, 'roi_boxes3d': roi_boxes3d, 'roi_scores': roi_scores, 'roi_size': roi_boxes3d[:, 3:6]} if self.mode == 'TEST': return sample_dict gt_obj_list = self.filtrate_objects(self.get_label(sample_id)) gt_boxes3d = np.zeros((gt_obj_list.__len__(), 7), dtype=np.float32) for k, obj in enumerate(gt_obj_list): gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \ = obj.pos, obj.h, obj.w, obj.l, obj.ry if gt_boxes3d.__len__() == 0: gt_iou = np.zeros((roi_boxes3d.shape[0]), dtype=np.float32) else: roi_corners = kitti_utils.boxes3d_to_corners3d(roi_boxes3d) gt_corners = kitti_utils.boxes3d_to_corners3d(gt_boxes3d) iou3d = kitti_utils.get_iou3d(roi_corners, gt_corners) gt_iou = iou3d.max(axis=1) sample_dict['gt_boxes3d'] = gt_boxes3d sample_dict['gt_iou'] = gt_iou return sample_dict
def get_label(self, idx): label_file = os.path.join(self.label_dir, f'{idx}.txt') assert os.path.exists(label_file) return kitti_utils.get_objects_from_label(label_file)