def test_project_to_image_space_tensors(self): anchors = np.asarray([[0, 0, 3, 2, 0, 6], [3, 0, 3, 2, 0, 2]], dtype=np.float64) img_idx = int('000217') img_shape = [375, 1242] dataset_config = DatasetBuilder.copy_config( DatasetBuilder.KITTI_UNITTEST) dataset_config.data_split = 'train' dataset_config.dataset_dir = tests.test_path() + \ "/datasets/Kitti/object" dataset = DatasetBuilder().build_kitti_dataset(dataset_config) stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 # Project the 3D points in numpy space img_corners, img_corners_norm = anchor_projector.project_to_image_space( anchors, stereo_calib_p2, img_shape) # convert the required params to tensors tf_stereo_calib_p2 = tf.convert_to_tensor(stereo_calib_p2, dtype=tf.float32) tf_anchors = tf.convert_to_tensor(anchors, dtype=tf.float32) tf_img_shape = tf.convert_to_tensor(img_shape, dtype=tf.float32) # Project the 3D points in tensor space img_corners_tensor, img_corners_norm_tensor = \ anchor_projector.tf_project_to_image_space(tf_anchors, tf_stereo_calib_p2, tf_img_shape) sess = tf.Session() with sess.as_default(): img_corners_out = img_corners_tensor.eval() img_corners_norm_out = img_corners_norm_tensor.eval() np.testing.assert_allclose(img_corners, img_corners_out, atol=1e-04, err_msg='Incorrect corner projection') np.testing.assert_allclose( img_corners_norm, img_corners_norm_out, atol=1e-04, err_msg='Incorrect normalized corner projection')
def _fill_anchor_pl_inputs(self, anchors_info, ground_plane, image_shape, stereo_calib_p2, sample_name, sample_augs): """ Fills anchor placeholder inputs with corresponding data Args: anchors_info: anchor info from mini_batch_utils ground_plane: ground plane coefficients image_shape: image shape (h, w), used for projecting anchors sample_name: name of the sample, e.g. "000001" sample_augs: list of sample augmentations """ # Lists for merging anchors info all_anchor_boxes_3d = [] anchors_ious = [] anchor_offsets = [] anchor_classes = [] # Create anchors for each class if len(self.dataset.classes) > 1: for class_idx in range(len(self.dataset.classes)): # Generate anchors for all classes grid_anchor_boxes_3d = self._anchor_generator.generate( area_3d=self._area_extents, anchor_3d_sizes=self._cluster_sizes[class_idx], anchor_stride=self._anchor_strides[class_idx], ground_plane=ground_plane) all_anchor_boxes_3d.append(grid_anchor_boxes_3d) all_anchor_boxes_3d = np.concatenate(all_anchor_boxes_3d) else: # Don't loop for a single class class_idx = 0 grid_anchor_boxes_3d = self._anchor_generator.generate( area_3d=self._area_extents, anchor_3d_sizes=self._cluster_sizes[class_idx], anchor_stride=self._anchor_strides[class_idx], ground_plane=ground_plane) all_anchor_boxes_3d = grid_anchor_boxes_3d # Filter empty anchors # Skip if anchors_info is [] sample_has_labels = True if self._train_val_test in ['train', 'val']: # Read in anchor info during training / validation if anchors_info: anchor_indices, anchors_ious, anchor_offsets, \ anchor_classes = anchors_info anchor_boxes_3d_to_use = all_anchor_boxes_3d[anchor_indices] else: train_cond = (self._train_val_test == "train" and self._train_on_all_samples) eval_cond = (self._train_val_test == "val" and self._eval_all_samples) if train_cond or eval_cond: sample_has_labels = False else: sample_has_labels = False if not sample_has_labels: # During testing, or validation with no anchor info, manually # filter empty anchors # TODO: share voxel_grid_2d with BEV generation if possible voxel_grid_2d = \ self.dataset.kitti_utils.create_sliced_voxel_grid_2d( sample_name, self.dataset.bev_source, image_shape=image_shape) # Convert to anchors and filter anchors_to_use = box_3d_encoder.box_3d_to_anchor( all_anchor_boxes_3d) empty_filter = anchor_filter.get_empty_anchor_filter_2d( anchors_to_use, voxel_grid_2d, density_threshold=1) anchor_boxes_3d_to_use = all_anchor_boxes_3d[empty_filter] # Convert lists to ndarrays anchor_boxes_3d_to_use = np.asarray(anchor_boxes_3d_to_use) anchors_ious = np.asarray(anchors_ious) anchor_offsets = np.asarray(anchor_offsets) anchor_classes = np.asarray(anchor_classes) # Flip anchors and centroid x offsets for augmented samples if kitti_aug.AUG_FLIPPING in sample_augs: anchor_boxes_3d_to_use = kitti_aug.flip_boxes_3d( anchor_boxes_3d_to_use, flip_ry=False) if anchors_info: anchor_offsets[:, 0] = -anchor_offsets[:, 0] # Convert to anchors anchors_to_use = box_3d_encoder.box_3d_to_anchor( anchor_boxes_3d_to_use) num_anchors = len(anchors_to_use) # Project anchors into bev bev_anchors, bev_anchors_norm = anchor_projector.project_to_bev( anchors_to_use, self._bev_extents) # Project box_3d anchors into image space img_anchors, img_anchors_norm = \ anchor_projector.project_to_image_space( anchors_to_use, stereo_calib_p2, image_shape) # Reorder into [y1, x1, y2, x2] for tf.crop_and_resize op self._bev_anchors_norm = bev_anchors_norm[:, [1, 0, 3, 2]] self._img_anchors_norm = img_anchors_norm[:, [1, 0, 3, 2]] # Fill in placeholder inputs self._placeholder_inputs[self.PL_ANCHORS] = anchors_to_use # If we are in train/validation mode, and the anchor infos # are not empty, store them. Checking for just anchors_ious # to be non-empty should be enough. if self._train_val_test in ['train', 'val'] and \ len(anchors_ious) > 0: self._placeholder_inputs[self.PL_ANCHOR_IOUS] = anchors_ious self._placeholder_inputs[self.PL_ANCHOR_OFFSETS] = anchor_offsets self._placeholder_inputs[self.PL_ANCHOR_CLASSES] = anchor_classes # During test, or val when there is no anchor info elif self._train_val_test in ['test'] or \ len(anchors_ious) == 0: # During testing, or validation with no gt, fill these in with 0s self._placeholder_inputs[self.PL_ANCHOR_IOUS] = \ np.zeros(num_anchors) self._placeholder_inputs[self.PL_ANCHOR_OFFSETS] = \ np.zeros([num_anchors, 6]) self._placeholder_inputs[self.PL_ANCHOR_CLASSES] = \ np.zeros(num_anchors) else: raise ValueError( 'Got run mode {}, and non-empty anchor info'.format( self._train_val_test)) self._placeholder_inputs[self.PL_BEV_ANCHORS] = bev_anchors self._placeholder_inputs[self.PL_BEV_ANCHORS_NORM] = \ self._bev_anchors_norm self._placeholder_inputs[self.PL_IMG_ANCHORS] = img_anchors self._placeholder_inputs[self.PL_IMG_ANCHORS_NORM] = \ self._img_anchors_norm
def main(): """This demo shows RPN proposals and AVOD predictions in 3D and 2D in image space. Given certain thresholds for proposals and predictions, it selects and draws the bounding boxes on the image sample. It goes through the entire proposal and prediction samples for the given dataset split. The proposals, overlaid, and prediction images can be toggled on or off separately in the options section. The prediction score and IoU with ground truth can be toggled on or off as well, shown as (score, IoU) above the detection. """ dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL) ############################## # Options ############################## dataset_config.data_split = 'val' fig_size = (10, 6.1) rpn_score_threshold = 0.1 avod_score_threshold = 0.1 # gt_classes = ['Car'] gt_classes = ['Pedestrian', 'Cyclist'] # gt_classes = ['Car', 'Pedestrian', 'Cyclist'] # Overwrite this to select a specific checkpoint global_step = None checkpoint_name = sys.argv[1] #'pyramid_cars_with_aug_example' # Drawing Toggles draw_proposals_separate = False draw_overlaid = False draw_predictions_separate = True # Show orientation for both GT and proposals/predictions draw_orientations_on_prop = False draw_orientations_on_pred = False # Draw 2D bounding boxes draw_projected_2d_boxes = True # Save images for samples with no detections save_empty_images = True draw_score = True draw_iou = True ############################## # End of Options ############################## # Get the dataset dataset = DatasetBuilder.build_kitti_dataset(dataset_config) # Setup Paths predictions_dir = avod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' proposals_and_scores_dir = predictions_dir + \ '/proposals_and_scores/' + dataset.data_split predictions_and_scores_dir = predictions_dir + \ '/final_predictions_and_scores/' + dataset.data_split # Output images directories output_dir_base = predictions_dir + '/images_2d' # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] if draw_proposals_separate: prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format( dataset.data_split, global_step, rpn_score_threshold) if not os.path.exists(prop_out_dir): os.makedirs(prop_out_dir) print('Proposal images saved to:', prop_out_dir) if draw_overlaid: overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format( dataset.data_split, global_step, avod_score_threshold) if not os.path.exists(overlaid_out_dir): os.makedirs(overlaid_out_dir) print('Overlaid images saved to:', overlaid_out_dir) if draw_predictions_separate: pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format( dataset.data_split, global_step, avod_score_threshold) if not os.path.exists(pred_out_dir): os.makedirs(pred_out_dir) print('Prediction images saved to:', pred_out_dir) # Rolling average array of times for time estimation avg_time_arr_length = 10 last_times = np.repeat(time.time(), avg_time_arr_length) + \ np.arange(avg_time_arr_length) for sample_idx in range(dataset.num_samples): # Estimate time remaining with 5 slowest times start_time = time.time() last_times = np.roll(last_times, -1) last_times[-1] = start_time avg_time = np.mean(np.sort(np.diff(last_times))[-5:]) samples_remaining = dataset.num_samples - sample_idx est_time_left = avg_time * samples_remaining # Print progress and time remaining estimate sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, ' 'Time Remaining: {:.2f}s'.format( sample_idx + 1, dataset.num_samples, avg_time, est_time_left)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] img_idx = int(sample_name) ############################## # Proposals ############################## if draw_proposals_separate or draw_overlaid: # Load proposals from files proposals_file_path = proposals_and_scores_dir + \ "/{}/{}.txt".format(global_step, sample_name) if not os.path.exists(proposals_file_path): print('Sample {}: No proposals, skipping'.format(sample_name)) continue print('Sample {}: Drawing proposals'.format(sample_name)) proposals_and_scores = np.loadtxt(proposals_file_path) proposal_boxes_3d = proposals_and_scores[:, 0:7] proposal_scores = proposals_and_scores[:, 7] # Apply score mask to proposals score_mask = proposal_scores > rpn_score_threshold proposal_boxes_3d = proposal_boxes_3d[score_mask] proposal_scores = proposal_scores[score_mask] proposal_objs = \ [box_3d_encoder.box_3d_to_object_label(proposal, obj_type='Proposal') for proposal in proposal_boxes_3d] ############################## # Predictions ############################## if draw_predictions_separate or draw_overlaid: predictions_file_path = predictions_and_scores_dir + \ "/{}/{}.txt".format(global_step, sample_name) if not os.path.exists(predictions_file_path): continue # Load predictions from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_class_indices = predictions_and_scores[:, 8] # process predictions only if we have any predictions left after # masking if len(prediction_boxes_3d) > 0: # Apply score mask avod_score_mask = prediction_scores >= avod_score_threshold prediction_boxes_3d = prediction_boxes_3d[avod_score_mask] prediction_scores = prediction_scores[avod_score_mask] prediction_class_indices = \ prediction_class_indices[avod_score_mask] # # Swap l, w for predictions where w > l # swapped_indices = \ # prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3] # prediction_boxes_3d = np.copy(prediction_boxes_3d) # prediction_boxes_3d[swapped_indices, 3] = \ # prediction_boxes_3d[swapped_indices, 4] # prediction_boxes_3d[swapped_indices, 4] = \ # prediction_boxes_3d[swapped_indices, 3] ############################## # Ground Truth ############################## # Get ground truth labels if dataset.has_labels: gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) else: gt_objects = [] # Filter objects to desired difficulty filtered_gt_objs = dataset.kitti_utils.filter_labels( gt_objects, classes=gt_classes) boxes2d, _, _ = obj_utils.build_bbs_from_objects( filtered_gt_objs, class_needed=gt_classes) image_path = dataset.get_rgb_image_path(sample_name) image = Image.open(image_path) image_size = image.size # Read the stereo calibration matrix for visualization stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx) calib_p2 = stereo_calib.p2 ############################## # Reformat and prepare to draw ############################## if draw_proposals_separate or draw_overlaid: proposals_as_anchors = box_3d_encoder.box_3d_to_anchor( proposal_boxes_3d) proposal_boxes, _ = anchor_projector.project_to_image_space( proposals_as_anchors, calib_p2, image_size) num_of_proposals = proposal_boxes_3d.shape[0] prop_fig, prop_2d_axes, prop_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False) draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals, proposal_objs, proposal_boxes, prop_2d_axes, prop_3d_axes, draw_orientations_on_prop) if draw_proposals_separate: # Save just the proposals filename = prop_out_dir + '/' + sample_name + '.png' plt.savefig(filename) if not draw_overlaid: plt.close(prop_fig) if draw_overlaid or draw_predictions_separate: if len(prediction_boxes_3d) > 0: # Project the 3D box predictions to image space image_filter = [] final_boxes_2d = [] for i in range(len(prediction_boxes_3d)): box_3d = prediction_boxes_3d[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, calib_p2, truncate=True, image_size=image_size, discard_before_truncation=False) if img_box is not None: image_filter.append(True) final_boxes_2d.append(img_box) else: image_filter.append(False) final_boxes_2d = np.asarray(final_boxes_2d) final_prediction_boxes_3d = prediction_boxes_3d[image_filter] final_scores = prediction_scores[image_filter] final_class_indices = prediction_class_indices[image_filter] num_of_predictions = final_boxes_2d.shape[0] # Convert to objs final_prediction_objs = \ [box_3d_encoder.box_3d_to_object_label( prediction, obj_type='Prediction') for prediction in final_prediction_boxes_3d] for (obj, score) in zip(final_prediction_objs, final_scores): obj.score = score else: if save_empty_images: pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) filename = pred_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(pred_fig) continue if draw_overlaid: # Overlay prediction boxes on image draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, prop_2d_axes, prop_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred) filename = overlaid_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(prop_fig) if draw_predictions_separate: # Now only draw prediction boxes on images # on a new figure handler if draw_projected_2d_boxes: pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_2d_axes, pred_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred) else: pred_fig, pred_3d_axes = \ vis_utils.visualize_single_plot( dataset.rgb_image_dir, img_idx, display=False) draw_3d_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred) filename = pred_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(pred_fig) print('\nDone')
def main(): """ Visualization of 3D grid anchor generation, showing 2D projections in BEV and image space, and a 3D display of the anchors """ dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_TRAIN) dataset_config.num_clusters[0] = 1 dataset = DatasetBuilder.build_kitti_dataset(dataset_config) label_cluster_utils = LabelClusterUtils(dataset) clusters, _ = label_cluster_utils.get_clusters() # Options img_idx = 1 # fake_clusters = np.array([[5, 4, 3], [6, 5, 4]]) # fake_clusters = np.array([[3, 3, 3], [4, 4, 4]]) fake_clusters = np.array([[4, 2, 3]]) fake_anchor_stride = [5.0, 5.0] ground_plane = [0, -1, 0, 1.72] anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() area_extents = np.array([[-40, 40], [-5, 5], [0, 70]]) # Generate anchors for cars only start_time = time.time() anchor_boxes_3d = anchor_3d_generator.generate( area_3d=dataset.kitti_utils.area_extents, anchor_3d_sizes=fake_clusters, anchor_stride=fake_anchor_stride, ground_plane=ground_plane) all_anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) end_time = time.time() print("Anchors generated in {} s".format(end_time - start_time)) # Project into bev bev_boxes, bev_normalized_boxes = \ anchor_projector.project_to_bev(all_anchors, area_extents[[0, 2]]) bev_fig, (bev_axes, bev_normalized_axes) = \ plt.subplots(1, 2, figsize=(16, 7)) bev_axes.set_xlim(0, 80) bev_axes.set_ylim(70, 0) bev_normalized_axes.set_xlim(0, 1.0) bev_normalized_axes.set_ylim(1, 0.0) plt.show(block=False) for box in bev_boxes: box_w = box[2] - box[0] box_h = box[3] - box[1] rect = patches.Rectangle((box[0], box[1]), box_w, box_h, linewidth=2, edgecolor='b', facecolor='none') bev_axes.add_patch(rect) for normalized_box in bev_normalized_boxes: box_w = normalized_box[2] - normalized_box[0] box_h = normalized_box[3] - normalized_box[1] rect = patches.Rectangle((normalized_box[0], normalized_box[1]), box_w, box_h, linewidth=2, edgecolor='b', facecolor='none') bev_normalized_axes.add_patch(rect) rgb_fig, rgb_2d_axes, rgb_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx) plt.show(block=False) image_path = dataset.get_rgb_image_path(dataset.sample_names[img_idx]) image_shape = np.array(Image.open(image_path)).shape stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 start_time = time.time() rgb_boxes, rgb_normalized_boxes = \ anchor_projector.project_to_image_space(all_anchors, stereo_calib_p2, image_shape) end_time = time.time() print("Anchors projected in {} s".format(end_time - start_time)) # Read the stereo calibration matrix for visualization stereo_calib = calib_utils.read_calibration(dataset.calib_dir, 0) p = stereo_calib.p2 # Overlay boxes on images for anchor_idx in range(len(anchor_boxes_3d)): anchor_box_3d = anchor_boxes_3d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor_box_3d) # Draw 3D boxes vis_utils.draw_box_3d(rgb_3d_axes, obj_label, p) # Draw 2D boxes rgb_box_2d = rgb_boxes[anchor_idx] box_x1 = rgb_box_2d[0] box_y1 = rgb_box_2d[1] box_w = rgb_box_2d[2] - box_x1 box_h = rgb_box_2d[3] - box_y1 rect = patches.Rectangle((box_x1, box_y1), box_w, box_h, linewidth=2, edgecolor='b', facecolor='none') rgb_2d_axes.add_patch(rect) if anchor_idx % 32 == 0: rgb_fig.canvas.draw() plt.show(block=True)