def preprocess(self, indices): """Preprocesses anchor info and saves info to files Args: indices (int array): sample indices to process. If None, processes all samples """ # Get anchor stride for class anchor_strides = self._anchor_strides dataset = self._dataset dataset_utils = self._dataset.kitti_utils classes_name = dataset.classes_name # Make folder if it doesn't exist yet output_dir = self.mini_batch_utils.get_file_path(classes_name, anchor_strides, sample_name=None) os.makedirs(output_dir, exist_ok=True) # Get clusters for class all_clusters_sizes, _ = dataset.get_cluster_info() anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Load indices of data_split all_samples = dataset.sample_list if indices is None: indices = np.arange(len(all_samples)) num_samples = len(indices) # For each image in the dataset, save info on the anchors for sample_idx in indices: # Get image name for given cluster sample_name = all_samples[sample_idx].name img_idx = int(sample_name) # Check for existing files and skip to the next if self._check_for_existing(classes_name, anchor_strides, sample_name): print("{} / {}: Sample already preprocessed".format( sample_idx + 1, num_samples, sample_name)) continue # Get ground truth and filter based on difficulty ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx) # Filter objects to dataset classes filtered_gt_list = dataset_utils.filter_labels(ground_truth_list) filtered_gt_list = np.asarray(filtered_gt_list) # Filtering by class has no valid ground truth, skip this image if len(filtered_gt_list) == 0: print("{} / {} No {}s for sample {} " "(Ground Truth Filter)".format( sample_idx + 1, num_samples, classes_name, sample_name)) # Output an empty file and move on to the next image. self._save_to_file(classes_name, anchor_strides, sample_name) continue # Get ground plane ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) image = Image.open(dataset.get_rgb_image_path(sample_name)) image_shape = [image.size[1], image.size[0]] # Generate sliced 2D voxel grid for filtering vx_grid_2d = dataset_utils.create_sliced_voxel_grid_2d( sample_name, source=dataset.bev_source, image_shape=image_shape) # List for merging all anchors all_anchor_boxes_3d = [] # Create anchors for each class for class_idx in range(len(dataset.classes)): # Generate anchors for all classes grid_anchor_boxes_3d = anchor_generator.generate( area_3d=self._area_extents, anchor_3d_sizes=all_clusters_sizes[class_idx], anchor_stride=self._anchor_strides[class_idx], ground_plane=ground_plane) all_anchor_boxes_3d.extend(grid_anchor_boxes_3d) # Filter empty anchors all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d) anchors = box_3d_encoder.box_3d_to_anchor(all_anchor_boxes_3d) empty_anchor_filter = anchor_filter.get_empty_anchor_filter_2d( anchors, vx_grid_2d, self._density_threshold) # Calculate anchor info anchors_info = self._calculate_anchors_info( all_anchor_boxes_3d, empty_anchor_filter, filtered_gt_list) anchor_ious = anchors_info[:, self.mini_batch_utils.col_ious] valid_iou_indices = np.where(anchor_ious > 0.0)[0] print("{} / {}:" "{:>6} anchors, " "{:>6} iou > 0.0, " "for {:>3} {}(s) for sample {}".format( sample_idx + 1, num_samples, len(anchors_info), len(valid_iou_indices), len(filtered_gt_list), classes_name, sample_name )) # Save anchors info self._save_to_file(classes_name, anchor_strides, sample_name, anchors_info)
def _fill_anchor_pl_inputs(self, anchors_info, ground_plane, image_shape, stereo_calib_p2, sample_name, sample_augs): """ Fills anchor placeholder inputs with corresponding data Args: anchors_info: anchor info from mini_batch_utils ground_plane: ground plane coefficients image_shape: image shape (h, w), used for projecting anchors sample_name: name of the sample, e.g. "000001" sample_augs: list of sample augmentations """ # Lists for merging anchors info all_anchor_boxes_3d = [] anchors_ious = [] anchor_offsets = [] anchor_classes = [] # Create anchors for each class if len(self.dataset.classes) > 1: for class_idx in range(len(self.dataset.classes)): # Generate anchors for all classes grid_anchor_boxes_3d = self._anchor_generator.generate( area_3d=self._area_extents, anchor_3d_sizes=self._cluster_sizes[class_idx], anchor_stride=self._anchor_strides[class_idx], ground_plane=ground_plane) all_anchor_boxes_3d.append(grid_anchor_boxes_3d) all_anchor_boxes_3d = np.concatenate(all_anchor_boxes_3d) else: # Don't loop for a single class class_idx = 0 grid_anchor_boxes_3d = self._anchor_generator.generate( area_3d=self._area_extents, anchor_3d_sizes=self._cluster_sizes[class_idx], anchor_stride=self._anchor_strides[class_idx], ground_plane=ground_plane) all_anchor_boxes_3d = grid_anchor_boxes_3d # Filter empty anchors # Skip if anchors_info is [] sample_has_labels = True if self._train_val_test in ['train', 'val']: # Read in anchor info during training / validation if anchors_info: anchor_indices, anchors_ious, anchor_offsets, \ anchor_classes = anchors_info anchor_boxes_3d_to_use = all_anchor_boxes_3d[anchor_indices] else: train_cond = (self._train_val_test == "train" and self._train_on_all_samples) eval_cond = (self._train_val_test == "val" and self._eval_all_samples) if train_cond or eval_cond: sample_has_labels = False else: sample_has_labels = False if not sample_has_labels: # During testing, or validation with no anchor info, manually # filter empty anchors # TODO: share voxel_grid_2d with BEV generation if possible voxel_grid_2d = \ self.dataset.kitti_utils.create_sliced_voxel_grid_2d( sample_name, self.dataset.bev_source, image_shape=image_shape) # Convert to anchors and filter anchors_to_use = box_3d_encoder.box_3d_to_anchor( all_anchor_boxes_3d) empty_filter = anchor_filter.get_empty_anchor_filter_2d( anchors_to_use, voxel_grid_2d, density_threshold=1) anchor_boxes_3d_to_use = all_anchor_boxes_3d[empty_filter] # Convert lists to ndarrays anchor_boxes_3d_to_use = np.asarray(anchor_boxes_3d_to_use) anchors_ious = np.asarray(anchors_ious) anchor_offsets = np.asarray(anchor_offsets) anchor_classes = np.asarray(anchor_classes) # Flip anchors and centroid x offsets for augmented samples if kitti_aug.AUG_FLIPPING in sample_augs: anchor_boxes_3d_to_use = kitti_aug.flip_boxes_3d( anchor_boxes_3d_to_use, flip_ry=False) if anchors_info: anchor_offsets[:, 0] = -anchor_offsets[:, 0] # Convert to anchors anchors_to_use = box_3d_encoder.box_3d_to_anchor( anchor_boxes_3d_to_use) num_anchors = len(anchors_to_use) # Project anchors into bev bev_anchors, bev_anchors_norm = anchor_projector.project_to_bev( anchors_to_use, self._bev_extents) # Project box_3d anchors into image space img_anchors, img_anchors_norm = \ anchor_projector.project_to_image_space( anchors_to_use, stereo_calib_p2, image_shape) # Reorder into [y1, x1, y2, x2] for tf.crop_and_resize op self._bev_anchors_norm = bev_anchors_norm[:, [1, 0, 3, 2]] self._img_anchors_norm = img_anchors_norm[:, [1, 0, 3, 2]] # Fill in placeholder inputs self._placeholder_inputs[self.PL_ANCHORS] = anchors_to_use # If we are in train/validation mode, and the anchor infos # are not empty, store them. Checking for just anchors_ious # to be non-empty should be enough. if self._train_val_test in ['train', 'val'] and \ len(anchors_ious) > 0: self._placeholder_inputs[self.PL_ANCHOR_IOUS] = anchors_ious self._placeholder_inputs[self.PL_ANCHOR_OFFSETS] = anchor_offsets self._placeholder_inputs[self.PL_ANCHOR_CLASSES] = anchor_classes # During test, or val when there is no anchor info elif self._train_val_test in ['test'] or \ len(anchors_ious) == 0: # During testing, or validation with no gt, fill these in with 0s self._placeholder_inputs[self.PL_ANCHOR_IOUS] = \ np.zeros(num_anchors) self._placeholder_inputs[self.PL_ANCHOR_OFFSETS] = \ np.zeros([num_anchors, 6]) self._placeholder_inputs[self.PL_ANCHOR_CLASSES] = \ np.zeros(num_anchors) else: raise ValueError( 'Got run mode {}, and non-empty anchor info'.format( self._train_val_test)) self._placeholder_inputs[self.PL_BEV_ANCHORS] = bev_anchors self._placeholder_inputs[self.PL_BEV_ANCHORS_NORM] = \ self._bev_anchors_norm self._placeholder_inputs[self.PL_IMG_ANCHORS] = img_anchors self._placeholder_inputs[self.PL_IMG_ANCHORS_NORM] = \ self._img_anchors_norm
def preprocess(self, indices): """Preprocesses anchor info and saves info to files Args: indices (int array): sample indices to process. If None, processes all samples """ # Get anchor stride for class anchor_params = self._anchor_params dataset = self._dataset dataset_utils = self._dataset.kitti_utils classes_name = dataset.classes_name anchor_strides = anchor_params['anchor_strides'] # Make folder if it doesn't exist yet output_dir = self.mini_batch_utils.get_file_path(classes_name, anchor_strides, sample_name=None) os.makedirs(output_dir, exist_ok=True) # Get clusters for class #all_clusters_sizes, _ = dataset.get_cluster_info() anchor_generator = grid_anchor_bev_generator.GridAnchorBevGenerator() #anchor_type = self._dataset.kitti_utils.anchor_type # Load indices of data_split all_samples = dataset.sample_list if indices is None: indices = np.arange(len(all_samples)) #indices = indices[:10] num_samples = len(indices) # For each image in the dataset, save info on the anchors for sample_idx in indices: # Get image name for given cluster sample_name = all_samples[sample_idx].name img_idx = int(sample_name) # Check for existing files and skip to the next if self._check_for_existing(classes_name, anchor_strides, sample_name): print("{} / {}: Sample already preprocessed".format( sample_idx + 1, num_samples, sample_name)) #continue # Get ground truth and filter based on difficulty ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx) # Filter objects to dataset classes filtered_gt_list = dataset_utils.filter_labels(ground_truth_list) filtered_gt_list = np.asarray(filtered_gt_list) # Filtering by class has no valid ground truth, skip this image if len(filtered_gt_list) == 0: print("{} / {} No {}s for sample {} " "(Ground Truth Filter)".format(sample_idx + 1, num_samples, classes_name, sample_name)) # Output an empty file and move on to the next image. #comment out for DEBUG self._save_to_file(classes_name, anchor_strides, sample_name) continue # Get ground plane ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) image = Image.open(dataset.get_rgb_image_path(sample_name)) image_shape = [image.size[1], image.size[0]] # List for merging all anchors all_level_anchor_boxes_bev = anchor_generator.generate(\ image_shapes=anchor_params['image_shapes'], anchor_base_sizes=anchor_params['anchor_base_sizes'], anchor_strides=anchor_params['anchor_strides'], anchor_ratios=anchor_params['anchor_ratios'], anchor_scales=anchor_params['anchor_scales'], anchor_init_ry_type=anchor_params['anchor_init_ry_type']) #concate all levels anchors #commentt out for DEBUG all_anchor_boxes_bev = np.concatenate(all_level_anchor_boxes_bev) #all_anchor_boxes_bev = all_level_anchor_boxes_bev[-1] # Filter empty anchors (whose pts num < density_threshold) # prepare for anchors_3d which dont need ry. anchors_bev = all_anchor_boxes_bev.copy() if anchor_params['anchor_init_ry_type'] == -90: anchors_bev[:, [2, 3]] = anchors_bev[:, [3, 2]] anchors_3d = box_bev_encoder.box_bev_to_anchor_3d(anchors_bev, \ bev_shape=self._bev_shape, \ bev_extents=self._dataset.kitti_utils.area_extents[[0, 2]]) #print(anchors_3d) image = Image.open(dataset.get_rgb_image_path(sample_name)) image_shape = [image.size[1], image.size[0]] # Generate sliced 2D voxel grid for filtering vx_grid_2d = dataset_utils.create_sliced_voxel_grid_2d( sample_name, source=dataset.bev_source, image_shape=image_shape) empty_anchor_filter = anchor_filter.get_empty_anchor_filter_2d( anchors_3d, vx_grid_2d, self._density_threshold) print( f'Non empty anchor: {np.sum(empty_anchor_filter)} / {len(all_anchor_boxes_bev)}, \ sample_name: {sample_name}') #empty_anchor_filter = np.ones(all_anchor_boxes_bev.shape[0], dtype=bool) # Calculate anchor info anchors_info = self._calculate_anchors_info( all_anchor_boxes_bev, empty_anchor_filter, filtered_gt_list) n_invalid = np.sum(np.isnan(anchors_info)) if n_invalid > 0: raise ValueError( 'Invalid value occur at anchors_info: nan, sample: ', sample_name) # anchor_ious = anchors_info[:, self.mini_batch_utils.col_ious] valid_iou_indices = np.where(anchor_ious > 0.0)[0] print("{} / {}:" "{:>6} anchors, " "{:>6} iou > 0.0, " "for {:>3} {}(s) for sample {}".format( sample_idx + 1, num_samples, len(anchors_info), len(valid_iou_indices), len(filtered_gt_list), classes_name, sample_name)) # Save anchors info #comment out for DEBUG self._save_to_file(classes_name, anchor_strides, sample_name, anchors_info)