def generate_gt_database(self): gt_database = [] for idx, sample_id in enumerate(self.image_idx_list): # sample_id = int(sample_id) print('process gt sample (id=%s)' % sample_id) pts_lidar = self.get_lidar(sample_id) calib = self.get_calib(sample_id) pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3]) pts_intensity = pts_lidar[:, 3] obj_list = self.filtrate_objects(self.get_label(sample_id)) gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32) for k, obj in enumerate(obj_list): gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \ = obj.pos, obj.h, obj.w, obj.l, obj.ry if gt_boxes3d.__len__() == 0: print('No gt object') if (self.split != "test"): continue boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu( torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d)) for k in range(boxes_pts_mask_list.__len__()): pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1) cur_pts = pts_rect[pt_mask_flag].astype(np.float32) cur_pts_intensity = pts_intensity[pt_mask_flag].astype( np.float32) if (self.split == "test"): gt_box3d = None else: gt_box3d = gt_boxes3d[k] sample_dict = { 'sample_id': sample_id, 'cls_type': obj_list[k].cls_type, 'gt_box3d': gt_boxes3d[k], 'points': cur_pts, 'intensity': cur_pts_intensity, 'obj': obj_list[k] } gt_database.append(sample_dict) save_file_name = os.path.join( args.save_dir, '%s_gt_database_3level_%s.joblib' % (args.split, self.classes[-1])) with open(save_file_name, 'wb') as f: joblib.dump(gt_database, f) # loaded_model = joblib.load(filename) # pickle.dump(gt_database, f) self.gt_database = gt_database print('Save refine training sample info file to %s' % save_file_name)
def generate_gt_database(self): gt_database = [] for idx, data in enumerate(self.__handle): sample_id = int(data["sample_id"]) # print('process gt sample (id=%06d)' % sample_id) # pts_lidar = self.get_lidar(sample_id) # calib = self.get_calib(sample_id) # pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3]) pts_rect = data['points'] pts_intensity = np.ones((len(pts_rect), 1)) # obj_list = self.filtrate_objects(self.get_label(sample_id)) # gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32) # for k, obj in enumerate(data["boxes"]): # gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \ # = obj[:3], obj[5], obj[4], obj[3], obj[6] gt_boxes3d = data["boxes"][:, [0, 1, 2, 5, 4, 3, 6]] if gt_boxes3d.__len__() == 0: print('No gt object') continue boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu( torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d)) for k in range(boxes_pts_mask_list.__len__()): pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1) cur_pts = pts_rect[pt_mask_flag].astype(np.float32) cur_pts_intensity = pts_intensity[pt_mask_flag].astype( np.float32) sample_dict = { 'sample_id': sample_id, 'cls_type': "Pedestrian", 'gt_box3d': gt_boxes3d[k], 'points': cur_pts, 'intensity': cur_pts_intensity, 'obj': data["boxes"][k] } gt_database.append(sample_dict) save_file_name = os.path.join( args.save_dir, '%s_gt_database_3level_%s.pkl' % (args.split, self.classes[-1])) with open(save_file_name, 'wb') as f: pickle.dump(gt_database, f) self.gt_database = gt_database print('Save refine training sample info file to %s' % save_file_name)
def generate_gt_database(self): gt_database = [] #### print(self.image_idx_list) : data/KITTI/ImageSets train.txt #### >> ['000000', '000003', '000007', '000009', '000010', '000011', '000012', '000013', '000014', '000016', ...] for idx, sample_id in enumerate(self.image_idx_list): sample_id = int(sample_id) # print('process gt sample (id=%06d)' % sample_id) pts_lidar = self.get_lidar(sample_id) calib = self.get_calib(sample_id) pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3]) pts_intensity = pts_lidar[:, 3] obj_list = self.filtrate_objects(self.get_label(sample_id)) #### Try it! gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32) for k, obj in enumerate(obj_list): gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \ = obj.pos, obj.h, obj.w, obj.l, obj.ry # if gt_boxes3d.__len__() == 0: # print('No gt object') # continue # else: # print('gt object detected!') boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d)) for k in range(boxes_pts_mask_list.__len__()): pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1) cur_pts = pts_rect[pt_mask_flag].astype(np.float32) cur_pts_intensity = pts_intensity[pt_mask_flag].astype(np.float32) sample_dict = {'sample_id': sample_id, 'cls_type': obj_list[k].cls_type, 'gt_box3d': gt_boxes3d[k], 'points': cur_pts, 'intensity': cur_pts_intensity, 'obj': obj_list[k]} gt_database.append(sample_dict) #### print(gt_database) save_file_name = os.path.join(args.save_dir, '%s_gt_database_3level_Multiclass.pkl' % (args.split)) #### Car -> Multiclass with open(save_file_name, 'wb') as f: pickle.dump(gt_database, f) self.gt_database = gt_database print('Save refine training sample info file to %s' % save_file_name)
def aug_one_scene(self, sample_id, pts_rect, pts_intensity, all_gt_boxes3d): """ :param pts_rect: (N, 3) :param gt_boxes3d: (M1, 7) :param all_gt_boxex3d: (M2, 7) :return: """ assert self.gt_database is not None extra_gt_num = np.random.randint(10, 15) try_times = 50 cnt = 0 cur_gt_boxes3d = all_gt_boxes3d.copy() cur_gt_boxes3d[:, 4] += 0.5 cur_gt_boxes3d[:, 5] += 0.5 # enlarge new added box to avoid too nearby boxes extra_gt_obj_list = [] extra_gt_boxes3d_list = [] new_pts_list, new_pts_intensity_list = [], [] src_pts_flag = np.ones(pts_rect.shape[0], dtype=np.int32) road_plane = self.get_road_plane(sample_id) a, b, c, d = road_plane while try_times > 0: try_times -= 1 rand_idx = np.random.randint(0, self.gt_database.__len__() - 1) new_gt_dict = self.gt_database[rand_idx] new_gt_box3d = new_gt_dict['gt_box3d'].copy() new_gt_points = new_gt_dict['points'].copy() new_gt_intensity = new_gt_dict['intensity'].copy() new_gt_obj = new_gt_dict['obj'] center = new_gt_box3d[0:3] if PC_REDUCE_BY_RANGE and (self.check_pc_range(center) is False): continue if cnt > extra_gt_num: break if new_gt_points.__len__() < 5: # too few points continue # put it on the road plane cur_height = (-d - a * center[0] - c * center[2]) / b move_height = new_gt_box3d[1] - cur_height new_gt_box3d[1] -= move_height new_gt_points[:, 1] -= move_height cnt += 1 iou3d = iou3d_utils.boxes_iou3d_gpu(torch.from_numpy(new_gt_box3d.reshape(1, 7)).cuda(), torch.from_numpy(cur_gt_boxes3d).cuda()).cpu().numpy() valid_flag = iou3d.max() < 1e-8 if not valid_flag: continue enlarged_box3d = new_gt_box3d.copy() enlarged_box3d[3] += 2 # remove the points above and below the object boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(pts_rect), torch.from_numpy(enlarged_box3d.reshape(1, 7))) pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1) src_pts_flag[pt_mask_flag] = 0 # remove the original points which are inside the new box new_pts_list.append(new_gt_points) new_pts_intensity_list.append(new_gt_intensity) enlarged_box3d = new_gt_box3d.copy() enlarged_box3d[4] += 0.5 enlarged_box3d[5] += 0.5 # enlarge new added box to avoid too nearby boxes cur_gt_boxes3d = np.concatenate((cur_gt_boxes3d, enlarged_box3d.reshape(1, 7)), axis=0) extra_gt_boxes3d_list.append(new_gt_box3d.reshape(1, 7)) extra_gt_obj_list.append(new_gt_obj) if new_pts_list.__len__() == 0: return False, pts_rect, pts_intensity, None, None extra_gt_boxes3d = np.concatenate(extra_gt_boxes3d_list, axis=0) # remove original points and add new points pts_rect = pts_rect[src_pts_flag == 1] pts_intensity = pts_intensity[src_pts_flag == 1] new_pts_rect = np.concatenate(new_pts_list, axis=0) new_pts_intensity = np.concatenate(new_pts_intensity_list, axis=0) pts_rect = np.concatenate((pts_rect, new_pts_rect), axis=0) pts_intensity = np.concatenate((pts_intensity, new_pts_intensity), axis=0) return True, pts_rect, pts_intensity, extra_gt_boxes3d, extra_gt_obj_list
def get_rcnn_sample_info(self, roi_info): sample_id, gt_box3d = roi_info['sample_id'], roi_info['gt_box3d'] rpn_xyz, rpn_features, rpn_intensity, seg_mask = self.rpn_feature_list[sample_id] # augmentation original roi by adding noise roi_box3d = self.aug_roi_by_noise(roi_info) # point cloud pooling based on roi_box3d pooled_boxes3d = kitti_utils.enlarge_box3d(roi_box3d.reshape(1, 7), cfg.RCNN.POOL_EXTRA_WIDTH) boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(rpn_xyz), torch.from_numpy(pooled_boxes3d)) pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1) cur_pts = rpn_xyz[pt_mask_flag].astype(np.float32) # data augmentation aug_pts = cur_pts.copy() aug_gt_box3d = gt_box3d.copy().astype(np.float32) aug_roi_box3d = roi_box3d.copy() if cfg.AUG_DATA and self.mode == 'TRAIN': # calculate alpha by ry temp_boxes3d = np.concatenate([aug_roi_box3d.reshape(1, 7), aug_gt_box3d.reshape(1, 7)], axis=0) temp_x, temp_z, temp_ry = temp_boxes3d[:, 0], temp_boxes3d[:, 2], temp_boxes3d[:, 6] temp_beta = np.arctan2(temp_z, temp_x).astype(np.float64) temp_alpha = -np.sign(temp_beta) * np.pi / 2 + temp_beta + temp_ry # data augmentation aug_pts, aug_boxes3d, aug_method = self.data_augmentation(aug_pts, temp_boxes3d, temp_alpha, mustaug=True, stage=2) aug_roi_box3d, aug_gt_box3d = aug_boxes3d[0], aug_boxes3d[1] aug_gt_box3d = aug_gt_box3d.astype(gt_box3d.dtype) # Pool input points valid_mask = 1 # whether the input is valid if aug_pts.shape[0] == 0: pts_features = np.zeros((1, 128), dtype=np.float32) input_channel = 3 + int(cfg.RCNN.USE_INTENSITY) + int(cfg.RCNN.USE_MASK) + int(cfg.RCNN.USE_DEPTH) pts_input = np.zeros((1, input_channel), dtype=np.float32) valid_mask = 0 else: pts_features = rpn_features[pt_mask_flag].astype(np.float32) pts_intensity = rpn_intensity[pt_mask_flag].astype(np.float32) pts_input_list = [aug_pts, pts_intensity.reshape(-1, 1)] if cfg.RCNN.USE_INTENSITY: pts_input_list = [aug_pts, pts_intensity.reshape(-1, 1)] else: pts_input_list = [aug_pts] if cfg.RCNN.USE_MASK: if cfg.RCNN.MASK_TYPE == 'seg': pts_mask = seg_mask[pt_mask_flag].astype(np.float32) elif cfg.RCNN.MASK_TYPE == 'roi': pts_mask = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(aug_pts), torch.from_numpy(aug_roi_box3d.reshape(1, 7))) pts_mask = (pts_mask[0].numpy() == 1).astype(np.float32) else: raise NotImplementedError pts_input_list.append(pts_mask.reshape(-1, 1)) if cfg.RCNN.USE_DEPTH: pts_depth = np.linalg.norm(aug_pts, axis=1, ord=2) pts_depth_norm = (pts_depth / 70.0) - 0.5 pts_input_list.append(pts_depth_norm.reshape(-1, 1)) pts_input = np.concatenate(pts_input_list, axis=1) # (N, C) aug_gt_corners = kitti_utils.boxes3d_to_corners3d(aug_gt_box3d.reshape(-1, 7)) aug_roi_corners = kitti_utils.boxes3d_to_corners3d(aug_roi_box3d.reshape(-1, 7)) iou3d = kitti_utils.get_iou3d(aug_roi_corners, aug_gt_corners) cur_iou = iou3d[0][0] # regression valid mask reg_valid_mask = 1 if cur_iou >= cfg.RCNN.REG_FG_THRESH and valid_mask == 1 else 0 # classification label cls_label = 1 if cur_iou > cfg.RCNN.CLS_FG_THRESH else 0 if cfg.RCNN.CLS_BG_THRESH < cur_iou < cfg.RCNN.CLS_FG_THRESH or valid_mask == 0: cls_label = -1 # canonical transform and sampling pts_input_ct, gt_box3d_ct = self.canonical_transform(pts_input, aug_roi_box3d, aug_gt_box3d) pts_input_ct, pts_features = self.rcnn_input_sample(pts_input_ct, pts_features) sample_info = {'sample_id': sample_id, 'pts_input': pts_input_ct, 'pts_features': pts_features, 'cls_label': cls_label, 'reg_valid_mask': reg_valid_mask, 'gt_boxes3d_ct': gt_box3d_ct, 'roi_boxes3d': aug_roi_box3d, 'roi_size': aug_roi_box3d[3:6], 'gt_boxes3d': aug_gt_box3d} return sample_info
def apply_gt_aug_to_one_scene(self, sample_id, pts_rect, pts_intensity, all_gt_boxes3d): """ :param pts_rect: (N, 3) :param all_gt_boxex3d: (M2, 7) :return: """ assert self.gt_database is not None # extra_gt_num = np.random.randint(10, 15) # try_times = 50 if cfg.GT_AUG_RAND_NUM: extra_gt_num = np.random.randint(10, cfg.GT_EXTRA_NUM) else: extra_gt_num = cfg.GT_EXTRA_NUM try_times = 100 cnt = 0 cur_gt_boxes3d = all_gt_boxes3d.copy() cur_gt_boxes3d[:, 4] += 0.5 # TODO: consider different objects cur_gt_boxes3d[:, 5] += 0.5 # enlarge new added box to avoid too nearby boxes cur_gt_corners = kitti_utils.boxes3d_to_corners3d(cur_gt_boxes3d) extra_gt_obj_list = [] extra_gt_boxes3d_list = [] new_pts_list, new_pts_intensity_list = [], [] src_pts_flag = np.ones(pts_rect.shape[0], dtype=np.int32) road_plane = self.get_road_plane(sample_id) a, b, c, d = road_plane while try_times > 0: if cnt > extra_gt_num: break try_times -= 1 if cfg.GT_AUG_HARD_RATIO > 0: p = np.random.rand() if p > cfg.GT_AUG_HARD_RATIO: # use easy sample rand_idx = np.random.randint(0, len(self.gt_database[0])) new_gt_dict = self.gt_database[0][rand_idx] else: # use hard sample rand_idx = np.random.randint(0, len(self.gt_database[1])) new_gt_dict = self.gt_database[1][rand_idx] else: rand_idx = np.random.randint(0, self.gt_database.__len__()) new_gt_dict = self.gt_database[rand_idx] new_gt_box3d = new_gt_dict['gt_box3d'].copy() new_gt_points = new_gt_dict['points'].copy() new_gt_intensity = new_gt_dict['intensity'].copy() new_gt_obj = new_gt_dict['obj'] center = new_gt_box3d[0:3] if cfg.PC_REDUCE_BY_RANGE and (self.check_pc_range(center) is False): continue if new_gt_points.__len__() < 5: # too few points continue # put it on the road plane cur_height = (-d - a * center[0] - c * center[2]) / b move_height = new_gt_box3d[1] - cur_height new_gt_box3d[1] -= move_height new_gt_points[:, 1] -= move_height new_gt_obj.pos[1] -= move_height new_enlarged_box3d = new_gt_box3d.copy() new_enlarged_box3d[4] += 0.5 new_enlarged_box3d[5] += 0.5 # enlarge new added box to avoid too nearby boxes cnt += 1 new_corners = kitti_utils.boxes3d_to_corners3d(new_enlarged_box3d.reshape(1, 7)) iou3d = kitti_utils.get_iou3d(new_corners, cur_gt_corners) valid_flag = iou3d.max() < 1e-8 if not valid_flag: continue enlarged_box3d = new_gt_box3d.copy() enlarged_box3d[3] += 2 # remove the points above and below the object boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu( torch.from_numpy(pts_rect), torch.from_numpy(enlarged_box3d.reshape(1, 7))) pt_mask_flag = (boxes_pts_mask_list[0].numpy() == 1) src_pts_flag[pt_mask_flag] = 0 # remove the original points which are inside the new box new_pts_list.append(new_gt_points) new_pts_intensity_list.append(new_gt_intensity) cur_gt_boxes3d = np.concatenate((cur_gt_boxes3d, new_enlarged_box3d.reshape(1, 7)), axis=0) cur_gt_corners = np.concatenate((cur_gt_corners, new_corners), axis=0) extra_gt_boxes3d_list.append(new_gt_box3d.reshape(1, 7)) extra_gt_obj_list.append(new_gt_obj) if new_pts_list.__len__() == 0: return False, pts_rect, pts_intensity, None, None extra_gt_boxes3d = np.concatenate(extra_gt_boxes3d_list, axis=0) # remove original points and add new points pts_rect = pts_rect[src_pts_flag == 1] pts_intensity = pts_intensity[src_pts_flag == 1] new_pts_rect = np.concatenate(new_pts_list, axis=0) new_pts_intensity = np.concatenate(new_pts_intensity_list, axis=0) pts_rect = np.concatenate((pts_rect, new_pts_rect), axis=0) pts_intensity = np.concatenate((pts_intensity, new_pts_intensity), axis=0) return True, pts_rect, pts_intensity, extra_gt_boxes3d, extra_gt_obj_list
def generate_gt_database(self): gt_database = [] # read (have it as a reference) the KITTI dataset documentation. # it is a text file in the library they have for handeling data. # Download object development kit (1 MB) (including 3D object detection and bird's eye view evaluation code) # http://www.cvlibs.net/datasets/kitti/eval_object.php?obj_benchmark=3d # image_idx_list was defined in the parent # it is a list of ids for data to choose for test/train for idx, sample_id in enumerate(self.image_idx_list): # get the id of the image/pointcloud (corresponding to 1 scene) sample_id = int(sample_id) print('process gt sample (id=%06d)' % sample_id) # for later: There is the same number of images and point clouds so I think it is 1 image per point cloud # read point cloud data pts_lidar = self.get_lidar(sample_id) calib = self.get_calib(sample_id) # coordinate calibration on x,y,z (not intensity!) (rect stands for recalibrated ?!) pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3]) # store intensity seperately pts_intensity = pts_lidar[:, 3] # first get a list of objects in the scene using the label data # then only pick (filter) those that are in the self.classes obj_list = self.filtrate_objects(self.get_label(sample_id)) # for each object (e.g. car) save the center (x,y,z) dimensions (w,h,l) and angle in the x-z plane (see paper figure3) gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32) for k, obj in enumerate(obj_list): gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \ = obj.pos, obj.h, obj.w, obj.l, obj.ry # if no objects of interest go to the next scene if gt_boxes3d.__len__() == 0: print('No gt object') continue # send the whole scene to 'pts_in_boxes3d_cpu' and the boxes in the scene # determin which points are inside the box the returned value is a 2d array # each row i is a mask of points that belong the box i and it is a pytorch tensor boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu(torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d)) # for each bounding box # choose the points inside a bounding box and label them as a single instance for k in range(boxes_pts_mask_list.__len__()): # 'boxes_pts_mask_list[k]' is a pytorch tensor pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1) #choose points that are inside the current box ('pts_rect' is a numpy array) cur_pts = pts_rect[pt_mask_flag].astype(np.float32) cur_pts_intensity = pts_intensity[pt_mask_flag].astype(np.float32) sample_dict = {'sample_id': sample_id, # scene id 'cls_type': obj_list[k].cls_type, # class (e.g. Car) 'gt_box3d': gt_boxes3d[k], # bounding box 'points': cur_pts, # point coords 'intensity': cur_pts_intensity, 'obj': obj_list[k]} # this includes all the other details we don't have above (as an object with attributes) gt_database.append(sample_dict) save_file_name = os.path.join(args.save_dir, '%s_gt_database_3level_%s.pkl' % (args.split, self.classes[-1])) with open(save_file_name, 'wb') as f: pickle.dump(gt_database, f) self.gt_database = gt_database print('Save refine training sample info file to %s' % save_file_name)
def roi_based_sample(pts_rect, pts_intensity, prev_pts_rect, prev_pts_intensity, roi_scores, roi_boxes3d, ratio, threshold): """ Samples points from pts_rect and pts_intensity based on the entropy gained as shown in roi_scores and roi_boxes3d. Uses points found in last pass and adds to them :param pts_rect: (N, 3) :param pts_intensity: (N, 1) :param prev_pts_rect: (N, 3) :param prev_pts_intensity: (N, 1) :param roi_scores: (M) :param roi_boxes3d: (M, 7) :param ratio: ratio of total pts_rect to return :param threshold: rois below this threshold will not be observed :return: pts_rect, pts_intensity """ # send everything to cpu roi_scores = roi_scores.cpu() roi_boxes3d = roi_boxes3d.cpu() pts_and_intensity = np.append(pts_rect, pts_intensity, axis=1) prev_pts_and_intensity = np.append(prev_pts_rect, prev_pts_intensity, axis=1) # Sets of tuples of points to calculate diff prev_pts_set = set(map(tuple, prev_pts_and_intensity)) pts_set = set(map(tuple, pts_and_intensity)) # Get pts not in previous pass pts_diff_set = pts_set - prev_pts_set pts_diff = np.array(list(pts_diff_set)) # pts and intensity diff pts_rect_diff = pts_diff[:, :3] # get entropy confidence = (1 / (1 + np.exp(-roi_scores.numpy()))) entropy = das_utils.confidence_to_entropy(confidence) # preen elements below threshold threshold_mask = entropy > threshold entropy = entropy[threshold_mask] roi_scores = roi_scores[threshold_mask] roi_boxes3d = roi_boxes3d[threshold_mask] # print(len(entropy), len(roi_scores), len(roi_boxes3d)) normalized_entropy_distribution = (1 / sum(entropy)) * entropy seen = set() point_sets = list(range(len(roi_scores))) # Changed to False reverse temporarily to see the difference highest_to_lowest_entropy = sorted(range(len(entropy)), key=entropy.__getitem__, reverse=True) # Points in boxes in order of decreasing entropy to prevent duplicates for i in highest_to_lowest_entropy: # Add new sampled points to set pts_in_box_mask = pts_in_boxes3d_cpu(torch.Tensor(pts_rect_diff), roi_boxes3d[i].unsqueeze(0))[0] pts_in_box = pts_diff[pts_in_box_mask] for pt in pts_in_box: if tuple(pt) in seen: np.delete(pts_in_box, np.argwhere(pts_in_box == pt)) else: seen.add(tuple(pt)) point_sets[i] = pts_in_box # Sampled points (add prev points) sampled_points = prev_pts_and_intensity # Fill with random points if less points than desired if (len(pts_rect) * ratio > sum( [len(point_sets[i]) for i in range(len(point_sets))])): new_sampled_points = pts_and_intensity[pts_in_boxes3d_cpu( torch.Tensor(pts_rect), roi_boxes3d)[0]] shuffled_pts = np.random.permutation(pts_and_intensity) i = 0 j = int(len(pts_rect) * ratio) - len(new_sampled_points) while i < j: if tuple(shuffled_pts[i]) in seen: j += 1 else: new_sampled_points = np.concatenate( (new_sampled_points, shuffled_pts[i].reshape(1, 4)), axis=0) i += 1 sampled_points = np.concatenate((sampled_points, new_sampled_points), axis=0) return np.array(sampled_points)[:, :3], np.array(sampled_points)[:, 3] # Sampling stage # Flattened indices to sample from indices = np.array([(i, j) for i in range(len(point_sets)) for j in range(len(point_sets[i]))]) # Probabilities for all of the indices probs = np.array([ normalized_entropy_distribution[i] / len(point_sets[i]) for i in range(len(point_sets)) for j in range(len(point_sets[i])) ]) probs /= probs.sum() sampled_indices_indices = np.random.choice(range(len(indices)), size=int(ratio * len(pts_rect)), p=probs, replace=False) sampled_indices = indices[sampled_indices_indices] new_sampled_points = [ point_sets[idx[0]][idx[1]] for idx in sampled_indices ] sampled_points = np.concatenate((sampled_points, new_sampled_points), axis=0) # print(sampled_points) return np.array(sampled_points)[:, :3], np.array(sampled_points)[:, 3]
def generate_gt_database(self): gt_database = [] for idx, sample_id in enumerate(self.image_idx_list): sample_id = int(sample_id) print('process gt sample (id=%06d)' % sample_id) pts_lidar = self.get_lidar(sample_id) calib = self.get_calib(sample_id) pts_rect = calib.lidar_to_rect(pts_lidar[:, 0:3]) pts_intensity = pts_lidar[:, 3] # (H,W,3) img = self.get_image_rgb_with_normal(sample_id) pts_img, pts_depth = calib.rect_to_img(pts_rect) obj_list = self.filtrate_objects(self.get_label(sample_id)) gt_boxes3d = np.zeros((obj_list.__len__(), 7), dtype=np.float32) for k, obj in enumerate(obj_list): gt_boxes3d[k, 0:3], gt_boxes3d[k, 3], gt_boxes3d[k, 4], gt_boxes3d[k, 5], gt_boxes3d[k, 6] \ = obj.pos, obj.h, obj.w, obj.l, obj.ry if gt_boxes3d.__len__() == 0: print('No gt object') continue boxes_pts_mask_list = roipool3d_utils.pts_in_boxes3d_cpu( torch.from_numpy(pts_rect), torch.from_numpy(gt_boxes3d)) shape = self.image_hw_with_padding_np.reshape((1, 1, 2)) for k in range(boxes_pts_mask_list.__len__()): pt_mask_flag = (boxes_pts_mask_list[k].numpy() == 1) cur_pts = pts_rect[pt_mask_flag].astype(np.float32) cur_pts_intensity = pts_intensity[pt_mask_flag].astype( np.float32) # add img rbg into point cloud # (N,2) cur_pts_img_xy = pts_img[pt_mask_flag].astype(np.float32) cur_pts_rgb = interpolate_img_by_xy(img, cur_pts_img_xy, shape) sample_dict = { 'sample_id': sample_id, 'cls_type': obj_list[k].cls_type, 'gt_box3d': gt_boxes3d[k], 'points': cur_pts, 'rgb': cur_pts_rgb, 'intensity': cur_pts_intensity, 'obj': obj_list[k] } gt_database.append(sample_dict) save_file_name = os.path.join( args.save_dir, '%s_gt_database_3level_%s.pkl' % (args.split, self.classes[-1])) with open(save_file_name, 'wb') as f: pickle.dump(gt_database, f) self.gt_database = gt_database print('Save refine training sample info file to %s' % save_file_name)