def set_up_model_test_mode(pipeline_config_path, data_split): """Returns the model and its config in test mode.""" model_config, _, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( pipeline_config_path, is_training=False) dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) # Use the validation set dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' # Remove augmentation when in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test='test', dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test='test', dataset=dataset) else: raise ValueError('Invalid model_name') return model, model_config
def main(): dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST) dataset_config.data_split = "trainval" unittest_dataset = DatasetBuilder.build_kitti_dataset(dataset_config) gen_label_clusters.main(unittest_dataset) gen_mini_batches.main(unittest_dataset)
def setUpClass(cls): # Initialize the Kitti dataset test_dir = tests.test_path() # Get the unittest-kitti dataset dataset_builder = DatasetBuilder() cls.dataset = dataset_builder.build_kitti_dataset( dataset_builder.KITTI_UNITTEST) cls.log_dir = test_dir + '/logs' cls.bev_vgg_cls = vgg.BevVggClassification()
def get_fake_dataset(self, data_split, directory): dataset_config = DatasetBuilder.copy_config( DatasetBuilder.KITTI_UNITTEST) # Overwrite config values dataset_config.data_split = data_split dataset_config.dataset_dir = directory dataset = DatasetBuilder.build_kitti_dataset(dataset_config) return dataset
def test_project_to_image_space_tensors(self): anchors = np.asarray([[0, 0, 3, 2, 0, 6], [3, 0, 3, 2, 0, 2]], dtype=np.float64) img_idx = int('000217') img_shape = [375, 1242] dataset_config = DatasetBuilder.copy_config( DatasetBuilder.KITTI_UNITTEST) dataset_config.data_split = 'train' dataset_config.dataset_dir = tests.test_path() + \ "/datasets/Kitti/object" dataset = DatasetBuilder().build_kitti_dataset(dataset_config) stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 # Project the 3D points in numpy space img_corners, img_corners_norm = anchor_projector.project_to_image_space( anchors, stereo_calib_p2, img_shape) # convert the required params to tensors tf_stereo_calib_p2 = tf.convert_to_tensor(stereo_calib_p2, dtype=tf.float32) tf_anchors = tf.convert_to_tensor(anchors, dtype=tf.float32) tf_img_shape = tf.convert_to_tensor(img_shape, dtype=tf.float32) # Project the 3D points in tensor space img_corners_tensor, img_corners_norm_tensor = \ anchor_projector.tf_project_to_image_space(tf_anchors, tf_stereo_calib_p2, tf_img_shape) sess = tf.Session() with sess.as_default(): img_corners_out = img_corners_tensor.eval() img_corners_norm_out = img_corners_norm_tensor.eval() np.testing.assert_allclose(img_corners, img_corners_out, atol=1e-04, err_msg='Incorrect corner projection') np.testing.assert_allclose( img_corners_norm, img_corners_norm_out, atol=1e-04, err_msg='Incorrect normalized corner projection')
def set_up_model(): test_pipeline_config_path = mlod.root_dir() + \ '/configs/mlod_exp_example.config' model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) dataset_config = config_builder.proto_to_obj(dataset_config) train_val_test = 'test' dataset_config.data_split = 'test' dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset_config.aug_list = [] dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=train_val_test, dataset=dataset) else: raise ValueError('Invalid model_name') return model
def main(): dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN) image_path = dataset.get_rgb_image_path('000000') # cv2 cv2_image1 = cv2.imread(image_path) cv2_image2 = cv2.imread(image_path) cv2_image3 = cv2.imread(image_path) cv2_image1_shape = cv2_image1.shape cv2_image2_shape = cv2_image2.shape cv2_image3_shape = cv2_image3.shape print(cv2_image1_shape) print(cv2_image2_shape) print(cv2_image3_shape) # PIL pil_image1 = Image.open(image_path) pil_image2 = Image.open(image_path) pil_image3 = Image.open(image_path) pil_image1_shape = pil_image1.size pil_image2_shape = pil_image2.size pil_image3_shape = pil_image3.size print(pil_image1_shape) print(pil_image2_shape) print(pil_image3_shape) pil_image1 = np.asarray(pil_image1) pil_image2 = np.asarray(pil_image2) pil_image3 = np.asarray(pil_image3)
def main(): test_pipeline_config_path = mlod.root_dir() + \ '/data/configs/official/cars/cars_000_vanilla.config' model_config, train_config, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) # train_val_test = 'val' # dataset_config.data_split = 'val' train_val_test = 'test' dataset_config.data_split = 'trainval' dataset_config.data_split_dir = 'training' dataset_config.has_labels = False # dataset_config.cache_config.cache_images = True # dataset_config.cache_config.cache_depth_maps = True dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) kitti_utils = dataset.kitti_utils bev_source = 'lidar' # sample_name = '000000' # img_idx = np.random.randint(0, 1000) # sample_name = '{:06d}'.format(img_idx) num_samples = 200 all_load_times = [] all_bev_times = [] for sample_idx in range(num_samples): sys.stdout.write('\rSample {} / {}'.format(sample_idx, num_samples - 1)) img_idx = sample_idx sample_name = '{:06d}'.format(sample_idx) loading_start_time = time.time() # Load image image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = image.shape[0:2] calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx) point_cloud = kitti_utils.get_point_cloud(bev_source, int(sample_name), image_shape) ground_plane = kitti_utils.get_ground_plane(sample_name) all_load_times.append(time.time() - loading_start_time) bev_start_time = time.time() bev_maps = kitti_utils.create_bev_maps(point_cloud, ground_plane) bev_end_time = time.time() all_bev_times.append(bev_end_time - bev_start_time) print('') print('Load mean:', np.mean(all_load_times)) print('Load median:', np.median(all_load_times)) print('BEV mean:', np.mean(all_bev_times)) print('BEV median:', np.median(all_bev_times))
def test_data_splits(self): bad_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST) # Test invalid splits bad_config.data_split = "bad" self.assertRaises(ValueError, KittiDataset, bad_config) # Should be "train" bad_config.data_split = "training" self.assertRaises(ValueError, KittiDataset, bad_config) # Should be "val" bad_config.data_split = "validation" self.assertRaises(ValueError, KittiDataset, bad_config) # Should be "test" bad_config.data_split = "testing" self.assertRaises(ValueError, KittiDataset, bad_config) # Train split train_dataset = self.get_fake_dataset('train', self.fake_kitti_dir) self.assertEqual(train_dataset.num_samples, 7) # Validation split validation_dataset = self.get_fake_dataset('val', self.fake_kitti_dir) self.assertEqual(validation_dataset.num_samples, 6) # Train + validation split trainval_dataset = self.get_fake_dataset('trainval', self.fake_kitti_dir) self.assertEqual(trainval_dataset.num_samples, 13) # Test split test_dataset = self.get_fake_dataset('test', self.fake_kitti_dir) self.assertEqual(test_dataset.num_samples, 10)
def main(): dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL) # get proposals proposal_output_dir = mlod.root_dir() + \ "/data/predictions/rpn_model/proposals_and_scores/" + \ dataset.data_split global_steps = os.listdir(proposal_output_dir) print('Checkpoints found ', global_steps) all_recalls = [] for step in global_steps: for sample_name in dataset.sample_list: img_idx = int(sample_name) # ------------------------------------ # Load proposals and scores from files # ------------------------------------ proposals_scores_dir = proposal_output_dir + \ "{}/{}/{}.txt".format(dataset.data_split, step, sample_name) if not os.path.exists(proposals_scores_dir): print('File {} not found, skipping'.format(sample_name)) continue proposals_scores = np.loadtxt(proposals_scores_dir) proposals = proposals_scores[:, 0:-1] proposal_iou_format = \ box_3d_encoder.box_3d_to_3d_iou_format(proposals) # scores are in the last column scores = proposals_scores[:, -1] # ----------------------- # Get ground truth labels # ----------------------- gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) _, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects( gt_objects, ['Car', 'car']) score_thresholds = np.array([0.3]) iou_threshold = 0.0025 # calculate RPN recall and precision precision, recall = evaluation.evaluate_3d([gt_3d_bbs], [proposal_iou_format], [scores], score_thresholds, iou_threshold) print('Recall ', recall[0]) print('Precision ', precision[0]) all_recalls.append(recall)
def setUpClass(cls): pipeline_config = pipeline_pb2.NetworkPipelineConfig() dataset_config = pipeline_config.dataset_config config_path = mlod.root_dir() + '/configs/unittest_model.config' cls.model_config = config_build.get_model_config_from_file(config_path) input_config = cls.model_config.input_config input_config.bev_depth = 5 # unittest-kitti has [2, 1, 1] clusters dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST) cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def inference(model_config, eval_config, dataset_config, data_split, ckpt_indices): # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' eval_config.eval_mode = 'test' eval_config.evaluate_repeatedly = False dataset_config.has_labels = False # Enable this to see the actually memory being used eval_config.allow_gpu_mem_growth = True eval_config = config_builder.proto_to_obj(eval_config) # Grab the checkpoint indices to evaluate eval_config.ckpt_indices = ckpt_indices # Remove augmentation during evaluation in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Overwrite repeated field model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] with tf.Graph().as_default(): if model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) else: raise ValueError('Invalid model name {}'.format(model_name)) model_evaluator = Evaluator(model, dataset_config, eval_config) model_evaluator.run_latest_checkpoints()
def set_up_video_dataset(dataset_config, dataset_dir, data_split, data_split_dir): # Overwrite fields dataset_config.name = 'kitti_video' dataset_config.dataset_dir = dataset_dir dataset_config.data_split = data_split dataset_config.data_split_dir = data_split_dir dataset_config.has_labels = False # Overwrite repeated fields dataset_config = config_builder_util.proto_to_obj(dataset_config) dataset_config.aug_list = [] dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) return dataset
def setUp(self): tf.test.TestCase.setUp(self) test_pipeline_config_path = mlod.root_dir() + \ '/configs/unittest_stagewise.config' self.model_config, self.train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) # unittest-kitti has [2, 1, 1] clusters input_config = self.model_config.input_config input_config.bev_depth = 5 # Generate dataset self.dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_UNITTEST, use_defaults=False, new_cfg=dataset_config)
def train(rpn_model_config, mlod_model_config, rpn_train_config, mlod_train_config, dataset_config): train_val_test = 'train' dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) paths_config = rpn_model_config.paths_config rpn_checkpoint_dir = paths_config.checkpoint_dir with tf.Graph().as_default(): model = RpnModel(rpn_model_config, train_val_test=train_val_test, dataset=dataset) trainer.train(model, rpn_train_config) # load the weights back in saver = tf.train.Saver() init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) trainer_utils.load_checkpoints(rpn_checkpoint_dir, saver) checkpoint_to_restore = saver.last_checkpoints[-1] trainer_utils.load_model_weights(model, sess, checkpoint_to_restore) # Merge RPN configs with MLOD - This will overwrite # the appropriate configs set for MLOD while keeping # the common configs the same. rpn_model_config.MergeFrom(mlod_model_config) rpn_train_config.MergeFrom(mlod_train_config) mlod_model_merged = deepcopy(rpn_model_config) mlod_train_merged = deepcopy(rpn_train_config) with tf.Graph().as_default(): model = MlodModel(mlod_model_merged, train_val_test=train_val_test, dataset=dataset) trainer.train(model, mlod_train_merged, stagewise_training=True, init_checkpoint_dir=rpn_checkpoint_dir)
def train(model_config, train_config, dataset_config): dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) train_val_test = 'train' model_name = model_config.model_name with tf.Graph().as_default(): if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=train_val_test, dataset=dataset) else: raise ValueError('Invalid model_name') trainer.train(model, train_config)
def set_up_model_train_mode(pipeline_config_path, data_split): """Returns the model and its train_op.""" model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( pipeline_config_path, is_training=True) dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=data_split, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=data_split, dataset=dataset) else: raise ValueError('Invalid model_name') prediction_dict = model.build() losses_dict, total_loss = model.loss(prediction_dict) # These parameters are required to set up the optimizer global_summaries = set([]) global_step_tensor = tf.Variable(0, trainable=False) training_optimizer = optimizer_builder.build(train_config.optimizer, global_summaries, global_step_tensor) # Set up the train op train_op = slim.learning.create_train_op(total_loss, training_optimizer) return model, train_op
def main(): """ Calculates clusters for each class Returns: all_clusters: list of clusters for each class all_std_devs: list of cluster standard deviations for each class """ dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN) # Calculate the remaining clusters # Load labels corresponding to the sample list for clustering sample_list = dataset.load_sample_names(dataset.cluster_split) all_dims = [] num_samples = len(sample_list) for sample_idx in range(num_samples): sys.stdout.write("\rClustering labels {} / {}".format( sample_idx + 1, num_samples)) sys.stdout.flush() sample_name = sample_list[sample_idx] img_idx = int(sample_name) obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx) filtered_lwh = LabelClusterUtils._filter_labels_by_class( obj_labels, dataset.classes) if filtered_lwh[0]: all_dims.extend(filtered_lwh[0]) all_dims = np.array(all_dims) print("\nFinished reading labels, clustering data...\n") # Print 3 decimal places np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)}) # Calculate average cluster k_means = KMeans(n_clusters=1, random_state=0).fit(all_dims) cluster_centre = k_means.cluster_centers_[0] # Calculate std. dev std_dev = np.std(all_dims, axis=0) # Calculate 2 and 3 standard deviations below the mean two_sigma_length_lo = cluster_centre[0] - 2 * std_dev[0] three_sigma_length_lo = cluster_centre[0] - 3 * std_dev[0] # Remove all labels with length above two std dev # from the mean and re-cluster small_mask_2 = all_dims[:, 0] < two_sigma_length_lo small_dims_2 = all_dims[small_mask_2] small_mask_3 = all_dims[:, 0] < three_sigma_length_lo small_dims_3 = all_dims[small_mask_3] small_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(small_dims_2) small_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(small_dims_3) small_std_dev_2 = np.std(small_dims_2, axis=0) small_std_dev_3 = np.std(small_dims_3, axis=0) print('small_k_means_2:', small_k_means_2.cluster_centers_) print('small_k_means_3:', small_k_means_3.cluster_centers_) print('small_std_dev_2:', small_std_dev_2) print('small_std_dev_3:', small_std_dev_3) # Calculate 2 and 3 standard deviations above the mean two_sigma_length_hi = cluster_centre[0] + 2 * std_dev[0] three_sigma_length_hi = cluster_centre[0] + 3 * std_dev[0] # Remove all labels with length above two std dev # from the mean and re-cluster large_mask_2 = all_dims[:, 0] > two_sigma_length_hi large_dims_2 = all_dims[large_mask_2] large_mask_3 = all_dims[:, 0] > three_sigma_length_hi large_dims_3 = all_dims[large_mask_3] large_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(large_dims_2) large_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(large_dims_3) large_std_dev_2 = np.std(large_dims_2, axis=0) large_std_dev_3 = np.std(large_dims_3, axis=0) print('large_k_means_2:', large_k_means_2.cluster_centers_) print('large_k_means_3:', large_k_means_3.cluster_centers_) print('large_std_dev_2:', large_std_dev_2) print('large_std_dev_3:', large_std_dev_3)
def setUpClass(cls): dataset_config = DatasetBuilder.copy_config( DatasetBuilder.KITTI_UNITTEST) cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config) cls.label_dir = cls.dataset.label_dir
def main(): """Shows a flipped sample in 3D """ # Create Dataset dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL) ############################## # Options ############################## # sample_name = "000191" sample_name = "000104" img_idx = int(sample_name) print("Showing anchors for sample {}".format(sample_name)) ############################## # Load Sample Data ############################## ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = [image.shape[1], image.shape[0]] # Get point cloud point_cloud = obj_utils.get_depth_map_point_cloud(img_idx, dataset.calib_dir, dataset.depth_dir, image_shape) points = np.array(point_cloud).T # Ground truth gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx) # Filter ground truth gt_labels = dataset.kitti_utils.filter_labels(gt_labels) ############################## # Flip stuff ############################## image_flipped = np.fliplr(image) # Flip ground plane coeff (x) ground_plane_flipped = np.copy(ground_plane) ground_plane_flipped[0] = -ground_plane_flipped[0] # Flip 3D points points_flipped = kitti_aug.flip_points(points) # Get point cloud colours point_colours_flipped = project_flipped_img_to_point_cloud( points_flipped, image_flipped, dataset.calib_dir, img_idx) # Flip ground truth boxes gt_labels_flipped = [ kitti_aug.flip_label_in_3d_only(obj) for obj in gt_labels ] ############################## # VTK Visualization ############################## # Axes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Point cloud vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points_flipped, point_colours=point_colours_flipped) # # Ground Truth Boxes vtk_boxes = VtkBoxes() vtk_boxes.set_objects(gt_labels_flipped, VtkBoxes.COLOUR_SCHEME_KITTI, show_orientations=True) # Renderer vtk_renderer = vtk.vtkRenderer() vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Add Actors to Rendered vtk_renderer.AddActor(axes) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_boxes.vtk_actor) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(170.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("Anchors") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_point_cloud.vtk_actor, ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def main(): """This demo shows RPN proposals and MLOD predictions in the 3D point cloud. Keys: F1: Toggle proposals F2: Toggle predictions F3: Toggle 3D voxel grid F4: Toggle point cloud F5: Toggle easy ground truth objects (Green) F6: Toggle medium ground truth objects (Orange) F7: Toggle hard ground truth objects (Red) F8: Toggle all ground truth objects (default off) F9: Toggle ground slice filter (default off) F10: Toggle offset slice filter (default off) """ ############################## # Options ############################## rpn_score_threshold = 0.1 mlod_score_threshold = 0.1 proposals_line_width = 1.0 predictions_line_width = 3.0 show_orientations = True point_cloud_source = 'lidar' # Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>) config_dir = None checkpoint_name = 'mlod_fpn_people_n_m' global_step = 135000 # Latest checkpoint #data_split = 'val_half' #data_split = 'val' data_split = 'test' # Show 3D iou text draw_ious_3d = False sample_name = '000031' # # # Cars # # # # sample_name = '000050' # sample_name = '000104' # sample_name = '000169' # sample_name = '000191' # sample_name = '000360' # sample_name = '001783' # sample_name = '001820' # val split # sample_name = '000181' # sample_name = '000751' # sample_name = '000843' # sample_name = '000944' # sample_name = '006338' # # # People # # # # val_half split # sample_name = '000001' # Hard, 1 far cyc # sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # val split # sample_name = '000015' # sample_name = '000048' # sample_name = '000058' # sample_name = '000076' # Medium, few ped, 1 cyc # sample_name = '000108' # sample_name = '000118' # sample_name = '000145' # sample_name = '000153' # sample_name = '000186' # sample_name = '000195' # sample_name = '000199' # sample_name = '000397' # sample_name = '004425' # sample_name = '004474' # Hard, many ped, 1 cyc # sample_name = '004657' # Hard, Few cycl, few ped # sample_name = '006071' # sample_name = '006828' # Hard, Few cycl, few ped # sample_name = '006908' # Hard, Few cycl, few ped # sample_name = '007412' # sample_name = '007318' # Hard, Few cycl, few ped ############################## # End of Options ############################## if data_split == 'test': draw_ious_3d = False if config_dir is None: config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name # Parse experiment config pipeline_config_file = \ config_dir + '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) dataset_config.data_split = data_split if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_names[sample_idx] ############################## # Setup Paths ############################## img_idx = int(sample_name) # Text files directory proposals_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/proposals_and_scores/' + dataset.data_split predictions_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/final_predictions_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] # Output images directory img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/predictions/images_3d/{}/{}/{}'.format(dataset.data_split, global_step, rpn_score_threshold) if not os.path.exists(img_out_dir): os.makedirs(img_out_dir) ############################## # Proposals ############################## # Load proposals from files proposals_and_scores = np.loadtxt( proposals_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) proposals = proposals_and_scores[:, 0:7] proposal_scores = proposals_and_scores[:, 7] rpn_score_mask = proposal_scores > rpn_score_threshold proposals = proposals[rpn_score_mask] proposal_scores = proposal_scores[rpn_score_mask] print('Proposals:', len(proposal_scores), proposal_scores) proposal_objs = \ [box_3d_encoder.box_3d_to_object_label(proposal, obj_type='Proposal') for proposal in proposals] ############################## # Predictions ############################## # Load proposals from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32) mlod_score_mask = prediction_scores >= mlod_score_threshold prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask] prediction_scores = prediction_scores[mlod_score_mask] print('Predictions: ', len(prediction_scores), prediction_scores) final_predictions = np.copy(prediction_boxes_3d) # # Swap l, w for predictions where w > l # swapped_indices = predictions[:, 4] > predictions[:, 3] # final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4] # final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3] prediction_objs = [] dataset.classes = ['Pedestrian', 'Cyclist', 'Car'] for pred_idx in range(len(final_predictions)): prediction_box_3d = final_predictions[pred_idx] prediction_type = dataset.classes[prediction_types[pred_idx]] prediction_obj = box_3d_encoder.box_3d_to_object_label( prediction_box_3d, obj_type=prediction_type) prediction_objs.append(prediction_obj) ############################## # Ground Truth ############################## dataset.has_labels = False if dataset.has_labels: # Get ground truth labels easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = \ demo_utils.get_gts_based_on_difficulty(dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] ############################## # 3D IoU ############################## if draw_ious_3d: # Convert to box_3d all_gt_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in all_gt_objs ] pred_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(pred_obj) for pred_obj in prediction_objs ] max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d, pred_boxes_3d) ############################## # Point Cloud ############################## image_path = dataset.get_rgb_image_path(sample_name) image = cv2.imread(image_path) point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source, img_idx, image_shape=image.shape) point_cloud = np.asarray(point_cloud) # Filter point cloud to extents area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]]) bev_extents = area_extents[[0, 2]] points = point_cloud.T point_filter = obj_utils.get_point_filter(point_cloud, area_extents) points = points[point_filter] point_colours = vis_utils.project_img_to_point_cloud( points, image, dataset.calib_dir, img_idx) # Voxelize the point cloud for visualization voxel_grid = VoxelGrid() voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False) # Ground plane ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) ############################## # Visualization ############################## # Create VtkVoxelGrid vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(voxel_grid) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # Create VtkAxes vtk_axes = vtk.vtkAxesActor() vtk_axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for proposal boxes vtk_proposal_boxes = VtkBoxes() vtk_proposal_boxes.set_line_width(proposals_line_width) vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS) # Create VtkBoxes for prediction boxes vtk_prediction_boxes = VtkPyramidBoxes() vtk_prediction_boxes.set_line_width(predictions_line_width) vtk_prediction_boxes.set_objects(prediction_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) # Create VtkBoxes for ground truth vtk_hard_gt_boxes = VtkBoxes() vtk_medium_gt_boxes = VtkBoxes() vtk_easy_gt_boxes = VtkBoxes() vtk_all_gt_boxes = VtkBoxes() vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI, show_orientations) # Create VtkTextLabels for 3D ious vtk_text_labels = VtkTextLabels() if draw_ious_3d and len(all_gt_boxes_3d) > 0: gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3] vtk_text_labels.set_text_labels( gt_positions_3d, ['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d]) # Create VtkGroundPlane vtk_ground_plane = VtkGroundPlane() vtk_slice_bot_plane = VtkGroundPlane() vtk_slice_top_plane = VtkGroundPlane() vtk_ground_plane.set_plane(ground_plane, bev_extents) vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents) vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_text_labels.vtk_actor) # Add ground plane and slice planes vtk_renderer.AddActor(vtk_ground_plane.vtk_actor) vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor) vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor) #vtk_renderer.AddActor(vtk_axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Set initial properties for some actors vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3) vtk_proposal_boxes.vtk_actor.SetVisibility(0) vtk_voxel_grid.vtk_actor.SetVisibility(0) vtk_all_gt_boxes.vtk_actor.SetVisibility(0) vtk_ground_plane.vtk_actor.SetVisibility(0) vtk_slice_bot_plane.vtk_actor.SetVisibility(0) vtk_slice_top_plane.vtk_actor.SetVisibility(0) vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9) vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9) vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(3.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName( "Predictions: Step {}, Sample {}, Min Score {}".format( global_step, sample_name, mlod_score_threshold, )) vtk_render_window.SetSize(900, 600) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) # Add custom interactor to toggle actor visibilities custom_interactor = vis_utils.CameraInfoInteractorStyle([ vtk_proposal_boxes.vtk_actor, vtk_prediction_boxes.vtk_actor, vtk_voxel_grid.vtk_actor, vtk_point_cloud.vtk_actor, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_ground_plane.vtk_actor, vtk_slice_bot_plane.vtk_actor, vtk_slice_top_plane.vtk_actor, vtk_text_labels.vtk_actor, ]) vtk_render_window_interactor.SetInteractorStyle(custom_interactor) # Render in VTK vtk_render_window.Render() # Take a screenshot window_to_image_filter = vtk.vtkWindowToImageFilter() window_to_image_filter.SetInput(vtk_render_window) window_to_image_filter.Update() png_writer = vtk.vtkPNGWriter() file_name = img_out_dir + "/{}.png".format(sample_name) png_writer.SetFileName(file_name) png_writer.SetInputData(window_to_image_filter.GetOutput()) png_writer.Write() print('Screenshot saved to ', file_name) vtk_render_window_interactor.Start() # Blocking
dataset_config.data_split = 'trainval' dataset_config.data_split_dir = 'training' dataset_config.has_labels = True # Set CUDA device id os.environ['CUDA_VISIBLE_DEVICES'] = args.device # Convert to object to overwrite repeated fields dataset_config = config_builder.proto_to_obj(dataset_config) # Remove augmentation during evaluation dataset_config.aug_list = ['flipping'] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Convert to object to overwrite repeated fields model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] index = 76 samples = dataset.load_samples([index]) depth_map= samples[0].get(constants.KEY_DPT_INPUT) print(depth_map.shape) image_input = samples[0].get(constants.KEY_IMAGE_INPUT)
def setUpClass(cls): cls.dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_UNITTEST) cls.mb_utils = cls.dataset.kitti_utils.mini_batch_utils
def main(): """This demo visualizes box 8C format predicted by MLOD, before getting converted to Box 3D. Keys: F1: Toggle predictions F2: Toggle easy ground truth objects (Green) F3: Toggle medium ground truth objects (Orange) F4: Toggle hard ground truth objects (Red) F5: Toggle all ground truth objects (default off) F6: Toggle 3D voxel grid F7: Toggle point cloud """ ############################## # Options ############################## mlod_score_threshold = 0.1 show_orientations = True checkpoint_name = 'mlod_exp_8c' global_step = None sample_name = None dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF) dataset = DatasetBuilder.build_kitti_dataset(dataset_config) ############################## # Setup Paths ############################## # # # Cars # # # # sample_name = '000050' # sample_name = '000104' # sample_name = '000169' # sample_name = '000191' # sample_name = '000360' # sample_name = '001783' # sample_name = '001820' # sample_name = '006338' # # # People # # # # val_half split # sample_name = '000001' sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_list[sample_idx] img_idx = int(sample_name) # Text files directory predictions_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/final_boxes_8c_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(predictions_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] ############################## # predictions ############################## # Load predictions from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) predictions_boxes_8c = predictions_and_scores[:, 0:24] prediction_scores = predictions_and_scores[:, 24] score_mask = prediction_scores >= mlod_score_threshold predictions_boxes_8c = predictions_boxes_8c[score_mask] all_vtk_box_corners = [] predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8]) for i in range(len(predictions_boxes_8c)): box_8c = predictions_boxes_8c[i, :, :] vtk_box_corners = VtkBox8c() vtk_box_corners.set_objects(box_8c) all_vtk_box_corners.append(vtk_box_corners) ############################## # Ground Truth ############################## if dataset.has_labels: easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = \ demo_utils.get_gts_based_on_difficulty(dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] ############################## # Point Cloud ############################## image_path = dataset.get_rgb_image_path(sample_name) image = cv2.imread(image_path) img_idx = int(sample_name) points, point_colours = demo_utils.get_filtered_pc_and_colours( dataset, image, img_idx) # Voxelize the point cloud for visualization voxel_grid = VoxelGrid() voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False) ############################## # Visualization ############################## # Create VtkVoxelGrid vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(voxel_grid) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for ground truth vtk_easy_gt_boxes, vtk_medium_gt_boxes, \ vtk_hard_gt_boxes, vtk_all_gt_boxes = \ demo_utils.create_gt_vtk_boxes(easy_gt_objs, medium_gt_objs, hard_gt_objs, all_gt_objs, show_orientations) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_box_actors = vtk.vtkAssembly() # Create VtkBoxes for prediction boxes for i in range(len(all_vtk_box_corners)): # Adding labels, slows down rendering # vtk_renderer.AddActor(all_vtk_box_corners[i]. # vtk_text_labels.vtk_actor) vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.SetBackground(0.2, 0.3, 0.4) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_box_actors) vtk_renderer.AddActor(axes) # Set initial properties for some actors vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2) vtk_voxel_grid.vtk_actor.SetVisibility(0) vtk_all_gt_boxes.vtk_actor.SetVisibility(0) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName( "Predictions: Step {}, Sample {}, Min Score {}".format( global_step, sample_name, mlod_score_threshold, )) vtk_render_window.SetSize(900, 600) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_box_actors, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor, vtk_point_cloud.vtk_actor ])) vtk_render_window_interactor.Start()
def main(): """ Visualization of the mini batch anchors for RpnModel training. Keys: F1: Toggle mini batch anchors F2: Toggle positive/negative proposal anchors F3: Toggle easy ground truth objects (Green) F4: Toggle medium ground truth objects (Orange) F5: Toggle hard ground truth objects (Red) F6: Toggle all ground truth objects (default off) F7: Toggle ground-plane """ anchor_colour_scheme = { "Car": (255, 0, 0), # Red "Pedestrian": (255, 150, 50), # Orange "Cyclist": (150, 50, 100), # Purple "DontCare": (255, 255, 255), # White "Anchor": (150, 150, 150), # Gray "Positive": (0, 255, 255), # Teal "Negative": (255, 0, 255) # Bright Purple } ############################## # Options ############################## show_orientations = True # Classes name config_name = 'car' # config_name = 'ped' # config_name = 'cyc' # config_name = 'ppl' # # # Random sample # # # sample_name = None # Small cars # sample_name = '000008' # sample_name = '000639' # # # Cars # # # # sample_name = "000001" # sample_name = "000050" # sample_name = "000112" # sample_name = "000169" # sample_name = "000191" # # # People # # # # sample_name = '000000' # val_half # sample_name = '000001' # Hard, 1 far cyc # sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # sample_name = "000000" # sample_name = "000011" # sample_name = "000015" # sample_name = "000028" # sample_name = "000035" # sample_name = "000134" # sample_name = "000167" # sample_name = '000379' # sample_name = '000381' # sample_name = '000397' # sample_name = '000398' # sample_name = '000401' # sample_name = '000407' # sample_name = '000486' # sample_name = '000509' # # Cyclists # # # # sample_name = '000122' # sample_name = '000448' # # # Multiple classes # # # # sample_name = "000764" ############################## # End of Options ############################## # Dataset config dataset_config_path = mlod.top_dir() + \ '/demos/configs/mb_rpn_{}.config'.format(config_name) # Create Dataset dataset = DatasetBuilder.load_dataset_from_config( dataset_config_path) # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_list[sample_idx].name anchor_strides = dataset.kitti_utils.anchor_strides img_idx = int(sample_name) print("Showing mini batch for sample {}".format(sample_name)) image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = [image.shape[1], image.shape[0]] # KittiUtils class dataset_utils = dataset.kitti_utils ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) point_cloud = obj_utils.get_depth_map_point_cloud(img_idx, dataset.calib_dir, dataset.depth_dir, image_shape) points = point_cloud.T point_colours = vis_utils.project_img_to_point_cloud(points, image, dataset.calib_dir, img_idx) clusters, _ = dataset.get_cluster_info() anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Read mini batch info anchors_info = dataset_utils.get_anchors_info( dataset.classes_name, anchor_strides, sample_name) if not anchors_info: # Exit early if anchors_info is empty print("Anchors info is empty, please try a different sample") return # Generate anchors for all classes all_anchor_boxes_3d = [] for class_idx in range(len(dataset.classes)): anchor_boxes_3d = anchor_generator.generate( area_3d=dataset.kitti_utils.area_extents, anchor_3d_sizes=clusters[class_idx], anchor_stride=anchor_strides[class_idx], ground_plane=ground_plane) all_anchor_boxes_3d.extend(anchor_boxes_3d) all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d) # Use anchors info indices, ious, offsets, classes = anchors_info # Get non empty anchors from the indices anchor_boxes_3d = all_anchor_boxes_3d[indices] # Sample an RPN mini batch from the non empty anchors mini_batch_utils = dataset.kitti_utils.mini_batch_utils mb_mask_tf, _ = mini_batch_utils.sample_rpn_mini_batch(ious) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) mb_mask = sess.run(mb_mask_tf) mb_anchor_boxes_3d = anchor_boxes_3d[mb_mask] mb_anchor_ious = ious[mb_mask] # ObjectLabel list that hold all boxes to visualize obj_list = [] num_positives = 0 # Convert the mini_batch anchors to object list mini_batch_size = mini_batch_utils.rpn_mini_batch_size for i in range(mini_batch_size): if mb_anchor_ious[i] > mini_batch_utils.rpn_pos_iou_range[0]: obj_type = "Positive" num_positives += 1 else: obj_type = "Negative" obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i], obj_type) obj_list.append(obj) print('Num positives', num_positives) # Convert all non-empty anchors to object list non_empty_anchor_objs = \ [box_3d_encoder.box_3d_to_object_label( anchor_box_3d, obj_type='Anchor') for anchor_box_3d in anchor_boxes_3d] ############################## # Ground Truth ############################## if dataset.has_labels: easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = demo_utils.get_gts_based_on_difficulty( dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] # Visualize 2D image vis_utils.visualization(dataset.rgb_image_dir, img_idx) plt.show(block=False) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for mini batch anchors vtk_pos_anchor_boxes = VtkBoxes() vtk_pos_anchor_boxes.set_objects(obj_list, anchor_colour_scheme) # VtkBoxes for non empty anchors vtk_non_empty_anchors = VtkBoxes() vtk_non_empty_anchors.set_objects(non_empty_anchor_objs, anchor_colour_scheme) vtk_non_empty_anchors.set_line_width(0.1) # Create VtkBoxes for ground truth vtk_easy_gt_boxes, vtk_medium_gt_boxes, \ vtk_hard_gt_boxes, vtk_all_gt_boxes = \ demo_utils.create_gt_vtk_boxes(easy_gt_objs, medium_gt_objs, hard_gt_objs, all_gt_objs, show_orientations) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2) vtk_ground_plane = VtkGroundPlane() vtk_ground_plane.set_plane(ground_plane, dataset.kitti_utils.bev_extents) # vtk_voxel_grid = VtkVoxelGrid() # vtk_voxel_grid.set_voxels(vx_grid) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_ground_plane.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) # vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_non_empty_anchors.vtk_actor) vtk_renderer.AddActor(vtk_pos_anchor_boxes.vtk_actor) vtk_renderer.AddActor(axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() mb_iou_thresholds = np.round( [mini_batch_utils.rpn_neg_iou_range[1], mini_batch_utils.rpn_pos_iou_range[0]], 3) vtk_render_window.SetWindowName( 'Sample {} RPN Mini Batch {}/{}, ' 'Num Positives {}'.format( sample_name, mb_iou_thresholds[0], mb_iou_thresholds[1], num_positives)) vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_non_empty_anchors.vtk_actor, vtk_pos_anchor_boxes.vtk_actor, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_ground_plane.vtk_actor ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def main(): """ Converts a set of network predictions into text files required for KITTI evaluation. """ ############################## # Options ############################## checkpoint_name = 'mlod_exp_example' # data_split = 'val' data_split = 'val_half' global_steps = None # global_steps = [28000, 19000, 33000, 34000] score_threshold = 0.1 save_2d = False # Save 2D predictions save_3d = True # Save 2D and 3D predictions together # Checkpoints below this are skipped min_step = 20000 # Object Type obj_type = 'obj' ############################## # End of Options ############################## # Parse experiment config pipeline_config_file = \ mlod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) # Overwrite defaults dataset_config = config_builder_util.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.aug_list = [] if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Get available prediction folders predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' proposals_root_dir = predictions_root_dir + \ '/proposals_and_scores/' + dataset.data_split print('Converting proposals from', proposals_root_dir) if not global_steps: global_steps = os.listdir(proposals_root_dir) global_steps.sort(key=int) print('Checkpoints found ', global_steps) for step_idx in range(len(global_steps)): global_step = global_steps[step_idx] # Skip first checkpoint if int(global_step) < min_step: continue final_predictions_dir = proposals_root_dir + \ '/' + str(global_step) # 2D and 3D prediction directories kitti_predictions_2d_dir = predictions_root_dir + \ '/kitti_proposals_2d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' kitti_proposals_3d_dir = predictions_root_dir + \ '/kitti_proposals_3d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' if save_2d and not os.path.exists(kitti_predictions_2d_dir): os.makedirs(kitti_predictions_2d_dir) if save_3d and not os.path.exists(kitti_proposals_3d_dir): os.makedirs(kitti_proposals_3d_dir) # Do conversion num_samples = dataset.num_samples num_valid_samples = 0 print('\nGlobal step:', global_step) print('Converting proposals from:', final_predictions_dir) if save_2d: print('2D Detections saved to:', kitti_predictions_2d_dir) if save_3d: print('Proposals saved to:', kitti_proposals_3d_dir) for sample_idx in range(num_samples): # Print progress sys.stdout.write('\rConverting {} / {}'.format( sample_idx + 1, num_samples)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] prediction_file = sample_name + '.txt' kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \ '/' + prediction_file kitti_predictions_3d_file_path = kitti_proposals_3d_dir + \ '/' + prediction_file predictions_file_path = final_predictions_dir + \ '/' + prediction_file # If no predictions, skip to next file if not os.path.exists(predictions_file_path): if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue all_predictions = np.loadtxt(predictions_file_path) # Swap l, w for predictions where w > l swapped_indices = all_predictions[:, 4] > all_predictions[:, 3] fixed_predictions = np.copy(all_predictions) fixed_predictions[swapped_indices, 3] = all_predictions[swapped_indices, 4] fixed_predictions[swapped_indices, 4] = all_predictions[swapped_indices, 3] fixed_predictions[swapped_indices, 6] = np.pi / 2 score_filter = all_predictions[:, 7] >= score_threshold all_predictions = fixed_predictions[score_filter] # If no predictions, skip to next file if len(all_predictions) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue # Project to image space sample_name = prediction_file.split('.')[0] img_idx = int(sample_name) # Load image for truncation image = Image.open(dataset.get_rgb_image_path(sample_name)) stereo_calib_p2 = calib_utils.read_calibration( dataset.calib_dir, img_idx).p2 boxes = [] image_filter = [] for i in range(len(all_predictions)): box_3d = all_predictions[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, stereo_calib_p2, truncate=True, image_size=image.size, discard_before_truncation=False) # Skip invalid boxes (outside image space) if img_box is None: image_filter.append(False) else: image_filter.append(True) boxes.append(img_box) boxes = np.asarray(boxes) all_predictions = all_predictions[image_filter] # If no predictions, skip to next file if len(boxes) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue num_valid_samples += 1 # To keep each value in its appropriate position, an array of zeros # (N, 16) is allocated but only values [4:16] are used kitti_predictions = np.zeros([len(boxes), 16]) # Truncation and Occlusion are always empty (see below) # Alpha (Not computed) kitti_predictions[:, 3] = -10 * np.ones( (len(kitti_predictions)), dtype=np.int32) # 2D predictions kitti_predictions[:, 4:8] = boxes[:, 0:4] # 3D predictions # (l, w, h) kitti_predictions[:, 8] = all_predictions[:, 5] kitti_predictions[:, 9] = all_predictions[:, 4] kitti_predictions[:, 10] = all_predictions[:, 3] # (x, y, z) kitti_predictions[:, 11:14] = all_predictions[:, 0:3] # (ry, score) kitti_predictions[:, 14:16] = all_predictions[:, 6:8] # Round detections to 3 decimal places kitti_predictions = np.round(kitti_predictions, 3) # Empty Truncation, Occlusion kitti_empty_1 = -1 * np.ones( (len(kitti_predictions), 2), dtype=np.int32) # Empty 3D (x, y, z) kitti_empty_2 = -1 * np.ones( (len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (h, w, l) kitti_empty_3 = -1000 * np.ones( (len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (ry) kitti_empty_4 = -10 * np.ones( (len(kitti_predictions), 1), dtype=np.int32) # Create Type Array obj_types = [obj_type for i in range(len(kitti_predictions))] # Stack 2D predictions text kitti_text_2d = np.column_stack([ obj_types, kitti_empty_1, kitti_predictions[:, 3:8], kitti_empty_2, kitti_empty_3, kitti_empty_4, kitti_predictions[:, 15] ]) # Stack 3D predictions text kitti_text_3d = np.column_stack( [obj_types, kitti_empty_1, kitti_predictions[:, 3:16]]) # Save to text files if save_2d: np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d, newline='\r\n', fmt='%s') if save_3d: np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d, newline='\r\n', fmt='%s') print('\nNum valid:', num_valid_samples) print('Num samples:', num_samples)
def main(): """ Visualization for comparison of anchor filtering with 2D vs 3D integral images Keys: F1: Toggle 3D integral image filtered anchors F2: Toggle 2D integral image filtered anchors F3: Toggle 2D integral image empty anchors """ anchor_2d_colour_scheme = {"Anchor": (0, 0, 255)} # Blue anchor_3d_colour_scheme = {"Anchor": (0, 255, 0)} # Green anchor_unfiltered_colour_scheme = {"Anchor": (255, 0, 255)} # Purple # Create Dataset dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_TRAINVAL) sample_name = "000001" img_idx = int(sample_name) print("Showing anchors for sample {}".format(sample_name)) # Options # These clusters are from the trainval set and give more 2D anchors than 3D clusters = np.array([[3.55, 1.835, 1.525], [4.173, 1.69, 1.49]]) anchor_stride = [3.0, 3.0] ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) area_extents = np.array([[-40, 40], [-5, 3], [0, 70]]) anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Generate anchors start_time = time.time() anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents, anchor_3d_sizes=clusters, anchor_stride=anchor_stride, ground_plane=ground_plane) end_time = time.time() print("Anchors generated in {} s".format(end_time - start_time)) # Get point cloud point_cloud = obj_utils.get_stereo_point_cloud(img_idx, dataset.calib_dir, dataset.disp_dir) ground_offset_dist = 0.2 offset_dist = 2.0 # Filter points within certain xyz range and offset from ground plane # Filter points within 0.2m of the road plane slice_filter = dataset.kitti_utils.create_slice_filter(point_cloud, area_extents, ground_plane, ground_offset_dist, offset_dist) points = np.array(point_cloud).T points = points[slice_filter] anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) # Create 2D voxel grid vx_grid_2d = voxel_grid_2d.VoxelGrid2D() vx_grid_2d.voxelize_2d(points, 0.1, area_extents) # Create 3D voxel grid vx_grid_3d = voxel_grid.VoxelGrid() vx_grid_3d.voxelize(points, 0.1, area_extents) # Filter the boxes here! start_time = time.time() empty_filter_2d = anchor_filter.get_empty_anchor_filter_2d( anchors=anchors, voxel_grid_2d=vx_grid_2d, density_threshold=1) anchors_2d = anchor_boxes_3d[empty_filter_2d] end_time = time.time() print("2D Anchors filtered in {} s".format(end_time - start_time)) print("Number of 2D anchors remaining: %d" % (anchors_2d.shape[0])) unfiltered_anchors_2d = anchor_boxes_3d[np.logical_not(empty_filter_2d)] # 3D filtering start_time = time.time() empty_filter_3d = anchor_filter.get_empty_anchor_filter( anchors=anchors, voxel_grid_3d=vx_grid_3d, density_threshold=1) anchor_boxes_3d = anchor_boxes_3d[empty_filter_3d] end_time = time.time() print("3D Anchors filtered in {} s".format(end_time - start_time)) print("Number of 3D anchors remaining: %d" % (anchor_boxes_3d.shape[0])) anchor_2d_objects = [] for anchor_idx in range(len(anchors_2d)): anchor = anchors_2d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor') # Append to a list for visualization in VTK later anchor_2d_objects.append(obj_label) anchor_3d_objects = [] for anchor_idx in range(len(anchor_boxes_3d)): anchor = anchor_boxes_3d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor') # Append to a list for visualization in VTK later anchor_3d_objects.append(obj_label) unfiltered_anchor_objects = [] for anchor_idx in range(len(unfiltered_anchors_2d)): anchor = unfiltered_anchors_2d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor') # Append to a list for visualization in VTK later unfiltered_anchor_objects.append(obj_label) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for boxes vtk_2d_boxes = VtkBoxes() vtk_2d_boxes.set_objects(anchor_2d_objects, anchor_2d_colour_scheme) vtk_3d_boxes = VtkBoxes() vtk_3d_boxes.set_objects(anchor_3d_objects, anchor_3d_colour_scheme) vtk_unfiltered_boxes = VtkBoxes() vtk_unfiltered_boxes.set_objects(unfiltered_anchor_objects, anchor_unfiltered_colour_scheme) vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(vx_grid_3d) vtk_voxel_grid_2d = VtkVoxelGrid() vtk_voxel_grid_2d.set_voxels(vx_grid_2d) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_2d_boxes.vtk_actor) vtk_renderer.AddActor(vtk_3d_boxes.vtk_actor) vtk_renderer.AddActor(vtk_unfiltered_boxes.vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid_2d.vtk_actor) vtk_renderer.AddActor(axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(170.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("Anchors") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_2d_boxes.vtk_actor, vtk_3d_boxes.vtk_actor, vtk_unfiltered_boxes.vtk_actor, ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def main(): """ Displays the bird's eye view maps for a KITTI sample. """ ############################## # Options ############################## # one of ['height_priors', 'slices'] bev_generator = 'slices' height_priors_config = \ """ height_priors { ground_filter_offset: 0.2 offset_filter_distance: 2.0 std_dev_multiplier: 2.0 } """ slices_config = \ """ slices { height_lo: -0.2 height_hi: 2.3 num_slices: 5 } """ # Use None for a random image img_idx = None # img_idx = 142 # img_idx = 191 show_ground_truth = True # Whether to overlay ground_truth boxes point_cloud_source = 'lidar' ############################## # End of Options ############################## dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL) dataset_config = DatasetBuilder.merge_defaults(dataset_config) # Overwrite bev_generator if bev_generator == 'height_priors': text_format.Merge(height_priors_config, dataset_config.kitti_utils_config.bev_generator) elif bev_generator == 'slices': text_format.Merge(slices_config, dataset_config.kitti_utils_config.bev_generator) else: raise ValueError('Invalid bev_generator') dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) if img_idx is None: img_idx = int(random.random() * dataset.num_samples) sample_name = "{:06}".format(img_idx) print('=== Showing BEV maps for image: {}.png ==='.format(sample_name)) # Load image image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = image.shape[0:2] kitti_utils = dataset.kitti_utils point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx, image_shape) ground_plane = kitti_utils.get_ground_plane(sample_name) bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane) height_maps = np.array(bev_images.get("height_maps")) density_map = np.array(bev_images.get("density_map")) box_points, box_points_norm = [None, None] if show_ground_truth: # Get projected boxes obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx) filtered_objs = obj_labels label_boxes = [] for label in filtered_objs: box = box_3d_encoder.object_label_to_box_3d(label) label_boxes.append(box) label_boxes = np.array(label_boxes) box_points, box_points_norm = box_3d_projector.project_to_bev( label_boxes, [[-40, 40], [0, 70]]) rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16) img_x_start = 60 img_y_start = 330 img_x = img_x_start img_y = img_y_start img_w = 400 img_h = 350 img_titlebar_h = 20 # Show images vis_utils.cv2_show_image("Image", image, size_wh=rgb_img_size, location_xy=(img_x, 0)) # Height maps for map_idx in range(len(height_maps)): height_map = height_maps[map_idx] height_map = draw_boxes(height_map, box_points_norm) vis_utils.cv2_show_image("Height Map {}".format(map_idx), height_map, size_wh=(img_w, img_h), location_xy=(img_x, img_y)) img_x += img_w # Wrap around if (img_x + img_w) > 1920: img_x = img_x_start img_y += img_h + img_titlebar_h # Density map density_map = draw_boxes(density_map, box_points_norm) vis_utils.cv2_show_image("Density Map", density_map, size_wh=(img_w, img_h), location_xy=(img_x, img_y)) cv2.waitKey()
def main(): # Create Dataset dataset_config_path = mlod.root_dir() + \ '/configs/mb_preprocessing/rpn_cars.config' dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path) # Random sample sample_name = '000169' anchor_strides = dataset.kitti_utils.anchor_strides img_idx = int(sample_name) print("Showing mini batch for sample {}".format(sample_name)) image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = [image.shape[1], image.shape[0]] # KittiUtils class dataset_utils = dataset.kitti_utils ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) point_cloud = obj_utils.get_depth_map_point_cloud(img_idx, dataset.calib_dir, dataset.depth_dir, image_shape) # Grab ground truth ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx) ground_truth_list = dataset_utils.filter_labels(ground_truth_list) stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 ############################## # Flip sample info ############################## start_time = time.time() flipped_image = kitti_aug.flip_image(image) flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud) flipped_gt_list = [kitti_aug.flip_label_in_3d_only(obj) for obj in ground_truth_list] flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane) flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2( stereo_calib_p2, image_shape) flipped_points = flipped_point_cloud.T print('flip sample', time.time() - start_time) ############################## # Generate anchors ############################## clusters, _ = dataset.get_cluster_info() anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Read mini batch info anchors_info = dataset_utils.get_anchors_info( dataset.classes_name, anchor_strides, sample_name) all_anchor_boxes_3d = [] all_ious = [] for class_idx in range(len(dataset.classes)): anchor_boxes_3d = anchor_generator.generate( area_3d=dataset.kitti_utils.area_extents, anchor_3d_sizes=clusters[class_idx], anchor_stride=anchor_strides[class_idx], ground_plane=ground_plane) if anchors_info: indices, ious, offsets, classes = anchors_info # Get non empty anchors from the indices non_empty_anchor_boxes_3d = anchor_boxes_3d[indices] all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d) all_ious.extend(ious) if not len(all_anchor_boxes_3d) > 0: # Exit early if anchors_info is empty print("No anchors, Please try a different sample") return # Convert to ndarrays all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d) all_ious = np.asarray(all_ious) ############################## # Flip anchors ############################## start_time = time.time() flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d, flip_ry=False) print('flip anchors', time.time() - start_time) # Overwrite with flipped things all_anchor_boxes_3d = flipped_anchor_boxes_3d points = flipped_points ground_truth_list = flipped_gt_list ground_plane = flipped_ground_plane
def main(): """Plots detection errors for xyz, lwh, ry, and shows 3D IoU with ground truth boxes """ dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL_HALF, use_defaults=True) difficulty = 2 # Full path to kitti predictions # (e.g. '.../data/outputs/mlod_exp_example/predictions/' # 'kitti_predictions_3d/val/0.1/100000/data' predictions_data_path = 'path_to_detections/data' # Loop through split and save ious and errors all_3d_ious = [] all_errors = [] for sample_idx in range(dataset.num_samples): sys.stdout.write('\r{} / {}'.format(sample_idx + 1, dataset.num_samples)) sample_name = dataset.sample_names[sample_idx] img_idx = int(sample_name) # Get filtered ground truth all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx) all_gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs, difficulty=difficulty) pred_objs = obj_utils.read_labels(predictions_data_path, img_idx) ############################## # Check IoUs ############################## if len(all_gt_objs) > 0 and \ pred_objs is not None and len(pred_objs) > 0: all_gt_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in all_gt_objs ] pred_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(pred_obj) for pred_obj in pred_objs ] # Convert to iou format gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( all_gt_boxes_3d) pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( pred_boxes_3d) max_ious_3d = np.zeros(len(all_gt_objs)) max_iou_pred_indices = -np.ones(len(all_gt_objs)) for gt_obj_idx in range(len(all_gt_objs)): gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx] ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt, pred_objs_iou_fmt) max_iou_3d = np.amax(ious_3d) max_ious_3d[gt_obj_idx] = max_iou_3d if max_iou_3d > 0.0: max_iou_pred_indices[gt_obj_idx] = np.argmax(ious_3d) for gt_obj_idx in range(len(all_gt_objs)): max_iou_pred_idx = int(max_iou_pred_indices[gt_obj_idx]) if max_iou_pred_idx >= 0: error = all_gt_boxes_3d[gt_obj_idx] - \ pred_boxes_3d[max_iou_pred_idx] all_errors.append(error) all_3d_ious.extend(max_ious_3d) print('Done') all_errors = np.asarray(all_errors) # Plot Data Histograms f, ax_arr = plt.subplots(3, 3) xyzlwh_bins = 51 ry_bins = 31 iou_bins = 51 # xyz ax_arr[0, 0].hist(all_errors[:, 0], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[0, 1].hist(all_errors[:, 1], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[0, 2].hist(all_errors[:, 2], xyzlwh_bins, facecolor='green', alpha=0.75) # lwh ax_arr[1, 0].hist(all_errors[:, 3], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[1, 1].hist(all_errors[:, 4], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[1, 2].hist(all_errors[:, 5], xyzlwh_bins, facecolor='green', alpha=0.75) # orientation ax_arr[2, 0].hist(all_errors[:, 6], ry_bins, facecolor='green', alpha=0.75) # iou ax_arr[2, 2].hist(all_3d_ious, iou_bins, facecolor='green', alpha=0.75) plt.show()