def get_label_objects(self, idx, drive_idx): assert (self.split == 'training') label_filename = os.path.join(self.label_dir, '%04d.txt' % (drive_idx)) if not self.rgb_detections: return utils.read_label(label_filename, from_video=True, frame=idx) else: # This part returns the 2d detection results and tracking results without gt labels id_list, type_list, box2d_list, prob_list, drive_id_list,\ track_id_list, label_dict = read_det_file_tracking(self.rgb_detection_files[drive_idx]) frame_objs = label_dict[drive_idx][idx] id_list_d = [] type_list_d = [] box2d_list_d = [] prob_list_d = [] drive_id_list_d = [] track_id_list_d = [] for obj in frame_objs: id_list_d.append(obj['frame_id']) type_list_d.append(obj['cls_id']) box2d_list_d.append(obj['bbox']) prob_list_d.append(obj['scr']) drive_id_list_d.append(obj['drive_id']) track_id_list_d.append(obj['track_id']) return id_list_d, type_list_d, box2d_list_d, prob_list_d, drive_id_list_d, track_id_list_d
def dataset_viz_pred(pred_label_dir): dataset = kitti_object(os.path.join(ROOT_DIR, 'dataset/KITTI/object')) split = 'training' save2ddir = os.path.join(ROOT_DIR, 'dataset/KITTI/object', split, 'vis2d') save3ddir = os.path.join(ROOT_DIR, 'dataset/KITTI/object', split, 'vis3d') save2ddir_pred = os.path.join(ROOT_DIR, 'dataset/KITTI/object', split, 'vis2d_pred') save3ddir_pred = os.path.join(ROOT_DIR, 'dataset/KITTI/object', split, 'vis3d_pred') if os.path.isdir(save2ddir) == True: print('previous save2ddir found. deleting...') shutil.rmtree(save2ddir) os.makedirs(save2ddir) if os.path.isdir(save3ddir) == True: print('previous save3ddir found. deleting...') shutil.rmtree(save3ddir) os.makedirs(save3ddir) if os.path.isdir(save2ddir_pred) == True: print('previous save2ddir_pred found. deleting...') shutil.rmtree(save2ddir_pred) os.makedirs(save2ddir_pred) if os.path.isdir(save3ddir_pred) == True: print('previous save3ddir_pred found. deleting...') shutil.rmtree(save3ddir_pred) os.makedirs(save3ddir_pred) for data_idx in range(len(dataset)): # Load data from dataset objects = dataset.get_label_objects(data_idx) objects_pred_label_filename = os.path.join(pred_label_dir, '%06d.txt' % (data_idx)) if os.path.exists(objects_pred_label_filename): objects_pred = utils.read_label(objects_pred_label_filename) #objects[0].print_object() img = dataset.get_image(data_idx) #img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) img_height, img_width, img_channel = img.shape print(('Image shape: ', img.shape)) #pc_velo = dataset.get_lidar(data_idx)[:,0:3] calib = dataset.get_calibration(data_idx) # Draw 2d and 3d boxes on image # show_image_with_boxes(img, objects, calib, False) img1, img2 = return_image_with_boxes(img, objects, calib, True) cv2.imwrite(os.path.join(save2ddir, str(data_idx).zfill(6) + '.png'), img1) cv2.imwrite(os.path.join(save3ddir, str(data_idx).zfill(6) + '.png'), img2) if os.path.exists(objects_pred_label_filename): print('writing...') img1_pred, img2_pred = return_image_with_boxes( img, objects_pred, calib, True) cv2.imwrite( os.path.join(save2ddir_pred, str(data_idx).zfill(6) + '.png'), img1_pred) cv2.imwrite( os.path.join(save3ddir_pred, str(data_idx).zfill(6) + '.png'), img2_pred)
def get_pred_objects(self, idx): assert (idx < self.num_samples and self.split == 'training') pred_filename = os.path.join(self.pred_dir, '%06d.txt' % (idx)) is_exist = os.path.exists(pred_filename) if is_exist: return utils.read_label(pred_filename) else: return None
def get_label_objects(self, idx): idx = self.sample_ids[idx] if self.subset == "training": label_filename = os.path.join(self.label_dir, "%06d.txt" % (idx)) return utils.read_label(label_filename) else: print('WARNING: Testing set does not have label!') return None
def get_pred_objects(self, idx): assert idx < self.num_samples pred_filename = os.path.join(self.pred_dir, "%06d.txt" % (idx)) is_exist = os.path.exists(pred_filename) if is_exist: return utils.read_label(pred_filename) else: return None
def get_pred_objects(self, idx): # assert idx < self.num_samples pred_filename = os.path.join(self.pred_dir, f"{self.index_format}.txt" % (idx)) is_exist = os.path.exists(pred_filename) if is_exist: return utils.read_label(pred_filename) else: print(f'{pred_filename} does not exist') return None
def get_pred_objects(self, idx): #assert idx < self.num_samples idx = idx.spilt('-') #ly pred_filename = os.path.join(self.pred_dir, "%s.txt" % (idx[0])) #ly is_exist = os.path.exists(pred_filename) if is_exist: return utils.read_label(pred_filename) else: return None
def get_pred_objects(self, idx): if self.pred_dir is None: raise RuntimeError('Prediction folder not provided!') idx = self.sample_ids[idx] pred_filename = os.path.join(self.pred_dir, "%06d.txt" % (idx)) if os.path.exists(pred_filename): return utils.read_label(pred_filename) else: print('WARNING: Prediction file not found!') return None
def get_label_objects(self, idx): """ :param idx: :return: """ assert (idx < self.num_samples and self.split == 'training') label_filename = os.path.join( self.label_dir, '%06d.txt' % (idx)) # label_filename format like 000001.txt return utils.read_label(label_filename)
def get_label_objects(self, idx):#读取物体标签-H assert(idx<self.num_samples and self.split=='training') #断言,若不满足条件就报错-H label_filename = os.path.join(self.label_dir, '%06d.txt'%(idx)) return utils.read_label(label_filename)
def get_label_objects(self, idx): assert idx < self.num_samples and self.split == "training" label_filename = os.path.join(self.label_dir, "%06d.txt" % (idx)) return utils.read_label(label_filename)
def get_label_objects(self, idx): assert(idx<self.num_samples and self.split=='training') label_filename = os.path.join(self.label_dir, '%06d.txt'%(idx)) return utils.read_label(label_filename)
def extract_frustum_data_rgb_detection(idx_filename, split, output_filename, res_label_dir, type_whitelist=['Car'], img_height_threshold=5, lidar_point_threshold=1): ''' Extract point clouds in frustums extruded from 2D detection boxes. Update: Lidar points and 3d boxes are in *rect camera* coord system (as that in 3d box label files) Input: det_filename: string, each line is img_path typeid confidence xmin ymin xmax ymax split: string, either trianing or testing output_filename: string, the name for output .pickle file type_whitelist: a list of strings, object types we are interested in. img_height_threshold: int, neglect image with height lower than that. lidar_point_threshold: int, neglect frustum with too few points. Output: None (will write a .pickle file to the disk) ''' dataset = kitti_object(os.path.join(ROOT_DIR, 'data/kitti'), split) data_idx_list = [int(line.rstrip()) for line in open(idx_filename)] id_list = [] type_list = [] box2d_list = [] prob_list = [] input_list = [] # channel number = 4, xyz,intensity in rect camera coord frustum_angle_list = [] # angle of 2d box center from pos x-axis box3d_pred_list = [] calib_list = [] enlarge_box3d_list = [] enlarge_box3d_size_list = [] enlarge_box3d_angle_list = [] for data_idx in data_idx_list: print('------------- ', data_idx) calib = dataset.get_calibration(data_idx) # 3 by 4 matrix pc_velo = dataset.get_lidar(data_idx) pc_rect = np.zeros_like(pc_velo) pc_rect[:, 0:3] = calib.project_velo_to_rect(pc_velo[:, 0:3]) pc_rect[:, 3] = pc_velo[:, 3] img = dataset.get_image(data_idx) img_height, img_width, img_channel = img.shape _, pc_image_coord, img_fov_inds = get_lidar_in_image_fov(pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True) pc_image_coord = pc_image_coord[img_fov_inds] pc_rect = pc_rect[img_fov_inds] label_filename = os.path.join(res_label_dir, '%06d.txt' % (data_idx)) objects = utils.read_label(label_filename) for obj_idx in range(len(objects)): if objects[obj_idx].type not in type_whitelist: continue # 2D BOX: Get pts rect backprojected box2d = objects[obj_idx].box2d xmin, ymin, xmax, ymax = box2d obj = objects[obj_idx] l, w, h = obj.l, obj.w, obj.h cx, cy, cz = obj.t ry = obj.ry cy = cy - h / 2 obj_array = np.array([cx, cy, cz, l, w, h, ry]) box3d_pts_3d = compute_box_3d_obj_array(obj_array) ratio = 1.2 enlarge_obj_array = obj_array.copy() enlarge_obj_array[3:6] = enlarge_obj_array[3:6] * ratio box3d_pts_3d_l = compute_box_3d_obj_array(enlarge_obj_array) _, inds = extract_pc_in_box3d(pc_rect, box3d_pts_3d_l) pc_in_cuboid = pc_rect[inds] pc_box_image_coord = pc_image_coord[inds] box3d_center = enlarge_obj_array[:3] frustum_angle = -1 * np.arctan2(box3d_center[2], box3d_center[0]) # Pass objects that are too small if ymax - ymin < img_height_threshold or xmax - xmin < 1 or \ len(pc_in_cuboid) < lidar_point_threshold: continue id_list.append(data_idx) input_list.append(pc_in_cuboid.astype(np.float32, copy=False)) type_list.append(objects[obj_idx].type) frustum_angle_list.append(frustum_angle) prob_list.append(obj.score) box2d_list.append(box2d) box3d_pred_list.append(box3d_pts_3d) enlarge_box3d_list.append(box3d_pts_3d_l) enlarge_box3d_size_list.append(enlarge_obj_array[3:6]) enlarge_box3d_angle_list.append(enlarge_obj_array[-1]) calib_list.append(calib.calib_dict) with open(output_filename, 'wb') as fp: pickle.dump(id_list, fp, -1) pickle.dump(box2d_list, fp, -1) pickle.dump(input_list, fp, -1) pickle.dump(type_list, fp, -1) pickle.dump(frustum_angle_list, fp, -1) pickle.dump(prob_list, fp, -1) pickle.dump(calib_list, fp, -1) pickle.dump(enlarge_box3d_list, fp, -1) pickle.dump(enlarge_box3d_size_list, fp, -1) pickle.dump(enlarge_box3d_angle_list, fp, -1) print(len(id_list)) print('save in {}'.format(output_filename))
def extract_frustum_det_data(idx_filename, split, output_filename, res_label_dir, perturb_box2d=False, augmentX=1, type_whitelist=['Car'], remove_diff=False): ''' Extract point clouds and corresponding annotations in frustums defined generated from 2D bounding boxes Lidar points and 3d boxes are in *rect camera* coord system (as that in 3d box label files) Input: idx_filename: string, each line of the file is a sample ID split: string, either trianing or testing output_filename: string, the name for output .pickle file viz: bool, whether to visualize extracted data perturb_box2d: bool, whether to perturb the box2d (used for data augmentation in train set) augmentX: scalar, how many augmentations to have for each 2D box. type_whitelist: a list of strings, object types we are interested in. Output: None (will write a .pickle file to the disk) ''' dataset = kitti_object(os.path.join(ROOT_DIR, 'data/kitti'), split) data_idx_list = [int(line.rstrip()) for line in open(idx_filename)] id_list = [] # int number box3d_list = [] # (8,3) array in rect camera coord input_list = [] # channel number = 4, xyz,intensity in rect camera coord label_list = [] # 1 for roi object, 0 for clutter type_list = [] # string e.g. Car heading_list = [] # ry (along y-axis in rect camera coord) radius of # (cont.) clockwise angle from positive x axis in velo coord. box3d_size_list = [] # array of l,w,h frustum_angle_list = [] # angle of 2d box center from pos x-axis gt_box2d_list = [] calib_list = [] enlarge_box3d_list = [] enlarge_box3d_size_list = [] enlarge_box3d_angle_list = [] pos_cnt = 0 all_cnt = 0 thresh = 0.5 if 'Car' in type_whitelist else 0.25 for data_idx in data_idx_list: print('------------- ', data_idx) calib = dataset.get_calibration(data_idx) # 3 by 4 matrix # objects = dataset.get_label_objects(data_idx) gt_objects = dataset.get_label_objects(data_idx) gt_objects, gt_boxes_2d, gt_boxes_3d = extract_boxes( gt_objects, type_whitelist, remove_diff) if len(gt_objects) == 0: continue pc_velo = dataset.get_lidar(data_idx) pc_rect = np.zeros_like(pc_velo) pc_rect[:, 0:3] = calib.project_velo_to_rect(pc_velo[:, 0:3]) pc_rect[:, 3] = pc_velo[:, 3] img = dataset.get_image(data_idx) img_height, img_width, img_channel = img.shape _, pc_image_coord, img_fov_inds = get_lidar_in_image_fov(pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True) pc_rect = pc_rect[img_fov_inds, :] pc_image_coord = pc_image_coord[img_fov_inds] label_filename = os.path.join(res_label_dir, '%06d.txt' % (data_idx)) objects = utils.read_label(label_filename) for obj_idx in range(len(objects)): if objects[obj_idx].type not in type_whitelist: continue obj = objects[obj_idx] l, w, h = obj.l, obj.w, obj.h cx, cy, cz = obj.t ry = obj.ry cy = cy - h / 2 obj_array = np.array([cx, cy, cz, l, w, h, ry]) ratio = 1.2 enlarge_obj_array = obj_array.copy() enlarge_obj_array[3:6] = enlarge_obj_array[3:6] * ratio overlap = rbbox_iou_3d(obj_array.reshape(-1, 7), gt_boxes_3d) overlap = overlap[0] max_overlap = overlap.max(0) max_idx = overlap.argmax(0) # print(max_overlap) if max_overlap < thresh: continue gt_obj = gt_objects[max_idx] gt_box2d = gt_objects[max_idx].box2d l, w, h = gt_obj.l, gt_obj.w, gt_obj.h cx, cy, cz = gt_obj.t ry = gt_obj.ry cy = cy - h / 2 gt_obj_array = np.array([cx, cy, cz, l, w, h, ry]) box3d_pts_3d = compute_box_3d_obj_array(gt_obj_array) for _ in range(augmentX): if perturb_box2d: # print(box3d_align) enlarge_obj_array = random_shift_rotate_box3d( enlarge_obj_array, 0.05) box3d_corners_enlarge = compute_box_3d_obj_array( enlarge_obj_array) else: box3d_corners_enlarge = compute_box_3d_obj_array( enlarge_obj_array) _, inds = extract_pc_in_box3d(pc_rect, box3d_corners_enlarge) pc_in_cuboid = pc_rect[inds] # pc_box_image_coord = pc_image_coord[inds] _, inds = extract_pc_in_box3d(pc_in_cuboid, box3d_pts_3d) label = np.zeros((pc_in_cuboid.shape[0])) label[inds] = 1 # _, inds = extract_pc_in_box3d(pc_rect, box3d_pts_3d) # print(np.sum(label), np.sum(inds)) # Get 3D BOX heading heading_angle = gt_obj.ry # Get 3D BOX size box3d_size = np.array([gt_obj.l, gt_obj.w, gt_obj.h]) # Reject too far away object or object without points if np.sum(label) == 0: continue box3d_center = enlarge_obj_array[:3] frustum_angle = -1 * np.arctan2(box3d_center[2], box3d_center[0]) id_list.append(data_idx) box3d_list.append(box3d_pts_3d) input_list.append(pc_in_cuboid) label_list.append(label) type_list.append(objects[obj_idx].type) heading_list.append(heading_angle) box3d_size_list.append(box3d_size) frustum_angle_list.append(frustum_angle) gt_box2d_list.append(gt_box2d) calib_list.append(calib.calib_dict) enlarge_box3d_list.append(box3d_corners_enlarge) enlarge_box3d_size_list.append(enlarge_obj_array[3:6]) enlarge_box3d_angle_list.append(enlarge_obj_array[-1]) # collect statistics pos_cnt += np.sum(label) all_cnt += pc_in_cuboid.shape[0] print('total_objects %d' % len(id_list)) print('Average pos ratio: %f' % (pos_cnt / float(all_cnt))) print('Average npoints: %f' % (float(all_cnt) / len(id_list))) with open(output_filename, 'wb') as fp: pickle.dump(id_list, fp, -1) pickle.dump(box3d_list, fp, -1) pickle.dump(input_list, fp, -1) pickle.dump(label_list, fp, -1) pickle.dump(type_list, fp, -1) pickle.dump(heading_list, fp, -1) pickle.dump(box3d_size_list, fp, -1) pickle.dump(frustum_angle_list, fp, -1) pickle.dump(gt_box2d_list, fp, -1) pickle.dump(calib_list, fp, -1) pickle.dump(enlarge_box3d_list, fp, -1) pickle.dump(enlarge_box3d_size_list, fp, -1) pickle.dump(enlarge_box3d_angle_list, fp, -1) print('save in {}'.format(output_filename))
def get_label_objects(self, idx): assert (self.split == 'val') label_filename = os.path.join(self.label_dir, '%06d.txt' % (idx)) return utils.read_label(label_filename)
def get_label_objects(self, idx): assert (idx < self.num_samples and self.split == 'training') label_filename = os.path.join(self.label_dir, '%d.txt' % (idx)) return utils.read_label(label_filename)
def get_lidar_label(self, idx): assert(idx<self.num_samples) lidar_label_filename = os.path.join(self.label_dir, '%06d.txt'%(idx))#bin return utils.read_label(lidar_label_filename)
args = parser.parse_args() assert os.path.isdir(args.depth_dir) assert os.path.isdir(args.calib_dir) assert os.path.isdir(args.label_dir) if not os.path.isdir(args.save_dir): os.makedirs(args.save_dir) depths = [ x for x in os.listdir(args.depth_dir) if x[-3:] == 'npy' and 'std' not in x ] depths = sorted(depths) for fn in depths: predix = fn[:-4] calib_file = '{}/{}.txt'.format(args.calib_dir, predix) label_file = '{}/{}.txt'.format(args.label_dir, predix) calib = kitti_util.Calibration(calib_file) objects = kitti_util.read_label(label_file) depth_map = np.load(args.depth_dir + '/' + fn) lidar = project_depth_to_lidar(calib, depth_map, objects, args.max_high) # pad 1 in the indensity dimension lidar = np.concatenate([lidar, np.ones((lidar.shape[0], 1))], 1) lidar = lidar.astype(np.float32) lidar.tofile('{}/{}.bin'.format(args.save_dir, predix)) print('Finish Depth {}'.format(predix))
def load_frame_data(self, data_idx_str, random_flip=False, random_rotate=False, random_shift=False, pca_jitter=False): '''load one frame''' if self.use_aug_scene: data_idx = int(data_idx_str[2:]) else: data_idx = int(data_idx_str) # print(data_idx_str) calib = self.kitti_dataset.get_calibration(data_idx) # 3 by 4 matrix image = self.kitti_dataset.get_image(data_idx) img_height, img_width = image.shape[0:2] # data augmentation if pca_jitter: image = apply_pca_jitter(image)[0] objects = [] if not self.use_aug_scene or data_idx_str[:2] == '00': pc_velo = self.kitti_dataset.get_lidar(data_idx) _, pc_image_coord, img_fov_inds = get_lidar_in_image_fov( pc_velo[:, 0:3], calib, 0, 0, img_width, img_height, True) pc_velo = pc_velo[img_fov_inds, :] choice = np.random.choice(pc_velo.shape[0], self.npoints, replace=True) point_set = pc_velo[choice, :] pc_rect = np.zeros_like(point_set) pc_rect[:, 0:3] = calib.project_velo_to_rect(point_set[:, 0:3]) pc_rect[:, 3] = point_set[:, 3] if self.is_training: objects = self.kitti_dataset.get_label_objects(data_idx) else: pc_rect = utils.load_velo_scan( os.path.join( self.kitti_path, 'aug_scene/rectified_data/{0}.bin'.format(data_idx_str))) choice = np.random.choice(pc_rect.shape[0], self.npoints, replace=True) pc_rect = pc_rect[choice, :] if self.is_training: objects = utils.read_label( os.path.join( self.kitti_path, 'aug_scene/aug_label/{0}.txt'.format(data_idx_str))) objects = filter( lambda obj: obj.type in self.types_list and obj.difficulty in self. difficulties_list, objects) gt_boxes = [] # ground truth boxes #start = time.time() seg_mask = np.zeros((pc_rect.shape[0])) # data augmentation if random_flip and np.random.random() > 0.5: # 50% chance flipping pc_rect[:, 0] *= -1 for obj in objects: obj.t = [-obj.t[0], obj.t[1], obj.t[2]] # ensure that ry is [-pi, pi] if obj.ry >= 0: obj.ry = np.pi - obj.ry else: obj.ry = -np.pi - obj.ry if random_rotate: ry = (np.random.random() - 0.5) * math.radians( 20) # -10~10 degrees pc_rect[:, 0:3] = rotate_points_along_y(pc_rect[:, 0:3], ry) for obj in objects: obj.t = rotate_points_along_y(obj.t, ry) obj.ry -= ry # ensure that ry is [-pi, pi] if obj.ry > np.pi: obj.ry -= 2 * np.pi elif obj.ry < -np.pi: obj.ry += 2 * np.pi proposal_of_point = {} # point index to proposal vector gt_box_of_point = {} # point index to corners_3d for obj in objects: _, obj_box_3d = utils.compute_box_3d(obj, calib.P) _, obj_mask = extract_pc_in_box3d(pc_rect, obj_box_3d) if np.sum(obj_mask) == 0: # label without 3d points # print('skip object without points') continue # IMPORTANT: this must match with NUM_SEG_CLASSES #seg_mask[obj_mask] = g_type2onehotclass[obj.type] seg_mask[obj_mask] = 1 gt_boxes.append(obj_box_3d) obj_idxs = np.where(obj_mask)[0] # data augmentation # FIXME: jitter point will make valid loss growing # Also may go out of image view if random_shift and False: # jitter object points pc_rect[obj_idxs, :3] = shift_point_cloud( pc_rect[obj_idxs, :3], 0.02) for idx in obj_idxs: proposal_of_point[idx] = box_encoder.encode( obj, pc_rect[idx, :3]) gt_box_of_point[idx] = obj_box_3d # self.viz_frame(pc_rect, seg_mask, gt_boxes) # return pc_rect, seg_mask, proposal_of_point, gt_box_of_point, gt_boxes calib_matrix = np.copy(calib.P) calib_matrix[0, :] *= (1200.0 / image.shape[1]) calib_matrix[1, :] *= (360.0 / image.shape[0]) #print('construct', time.time() - start) return { 'pointcloud': pc_rect, 'image': cv2.resize(image, (1200, 360)), 'calib': calib_matrix, 'mask_label': seg_mask, 'proposal_of_point': self.get_proposal_out(proposal_of_point), 'gt_box_of_point': self.get_gt_box_of_points(gt_box_of_point), 'gt_boxes': gt_boxes, 'pc_choice': choice }
def get_label_objects(idx): label_filename = path.join(label_dir, "%06d.txt" % (idx)) return utils.read_label(label_filename)
def get_label_objects(self, idx): label_filename = os.path.join(self.label_dir, '{}.txt'.format(idx)) return utils.read_label(label_filename)
def get_predict_objects(self, idx): # assert (idx < self.num_samples and self.split == 'training') label_filename = os.path.join( self.root_pred, '%06d.txt' % (idx)) # label_filename format like 000001.txt return utils.read_label(label_filename)