def main(): test_pipeline_config_path = mlod.root_dir() + \ '/data/configs/official/cars/cars_000_vanilla.config' model_config, train_config, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) # train_val_test = 'val' # dataset_config.data_split = 'val' train_val_test = 'test' dataset_config.data_split = 'trainval' dataset_config.data_split_dir = 'training' dataset_config.has_labels = False # dataset_config.cache_config.cache_images = True # dataset_config.cache_config.cache_depth_maps = True dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) kitti_utils = dataset.kitti_utils bev_source = 'lidar' # sample_name = '000000' # img_idx = np.random.randint(0, 1000) # sample_name = '{:06d}'.format(img_idx) num_samples = 200 all_load_times = [] all_bev_times = [] for sample_idx in range(num_samples): sys.stdout.write('\rSample {} / {}'.format(sample_idx, num_samples - 1)) img_idx = sample_idx sample_name = '{:06d}'.format(sample_idx) loading_start_time = time.time() # Load image image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = image.shape[0:2] calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx) point_cloud = kitti_utils.get_point_cloud(bev_source, int(sample_name), image_shape) ground_plane = kitti_utils.get_ground_plane(sample_name) all_load_times.append(time.time() - loading_start_time) bev_start_time = time.time() bev_maps = kitti_utils.create_bev_maps(point_cloud, ground_plane) bev_end_time = time.time() all_bev_times.append(bev_end_time - bev_start_time) print('') print('Load mean:', np.mean(all_load_times)) print('Load median:', np.median(all_load_times)) print('BEV mean:', np.mean(all_bev_times)) print('BEV median:', np.median(all_bev_times))
def setUp(self): tf.test.TestCase.setUp(self) img_feature_extractor = img_vgg.ImgVgg16() # dummy imageplaceholder img_input_placeholder = tf.placeholder(tf.float32, [480, 1590, 3]) img_input_batches = tf.expand_dims(img_input_placeholder, axis=0) img_preprocessed = img_feature_extractor.preprocess_input( img_input_batches, [224, 224]) _, img_end_points = img_feature_extractor.build(img_preprocessed) # download the vgg weights url = "http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz" self.vgg_checkpoint_dir = mlod.root_dir() + '/checkpoints/vgg_16' if not tf.gfile.Exists(self.vgg_checkpoint_dir): tf.gfile.MakeDirs(self.vgg_checkpoint_dir) # Download vgg16 if it hasn't been downloaded yet if not os.path.isfile(self.vgg_checkpoint_dir + "/vgg_16_2016_08_28.tar.gz"): dataset_utils.download_and_uncompress_tarball( url, self.vgg_checkpoint_dir) else: print("Already downloaded")
def run_kitti_native_script_with_05_iou(checkpoint_name, score_threshold, global_step, branch_idx=None): """Runs the kitti native code script.""" eval_script_dir = mlod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' make_script = eval_script_dir + \ '/kitti_native_eval/run_eval_05_iou.sh' script_folder = eval_script_dir + \ '/kitti_native_eval/' results_dir = mlod.top_dir() + '/scripts/offline_eval/results_05_iou/' # Round this because protobuf encodes default values as full decimal score_threshold = round(score_threshold, 3) if branch_idx is None: branch_idx_str = 'main' else: branch_idx_str = 'br_' + str(branch_idx) subprocess.call([ make_script, script_folder, str(score_threshold), str(global_step), str(checkpoint_name), str(results_dir), branch_idx_str ])
def set_up_model(): test_pipeline_config_path = mlod.root_dir() + \ '/configs/mlod_exp_example.config' model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) dataset_config = config_builder.proto_to_obj(dataset_config) train_val_test = 'test' dataset_config.data_split = 'test' dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset_config.aug_list = [] dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=train_val_test, dataset=dataset) else: raise ValueError('Invalid model_name') return model
def test_get_clusters(self): # classes = ['Car', 'Pedestrian', 'Cyclist'] num_clusters = [2, 1, 1] label_cluster_utils = LabelClusterUtils(self.dataset) clusters, std_devs = label_cluster_utils.get_clusters() # Check that correct number of clusters are returned clusters_per_class = [len(cls_clusters) for cls_clusters in clusters] std_devs_per_class = [len(cls_std_devs) for cls_std_devs in std_devs] self.assertEqual(clusters_per_class, num_clusters) self.assertEqual(std_devs_per_class, num_clusters) # Check that text files were saved txt_folder_exists = os.path.isdir( mlod.root_dir() + "/data/label_clusters/unittest-kitti") self.assertTrue(txt_folder_exists) # Calling get_clusters again should read from files read_clusters, read_std_devs = label_cluster_utils.get_clusters() # Check that read values are the same as generated ones np.testing.assert_allclose(np.vstack(clusters), np.vstack(read_clusters)) np.testing.assert_allclose(np.vstack(std_devs), np.vstack(read_std_devs))
def copy_kitti_native_code(checkpoint_name): """Copies and compiles kitti native code. It also creates neccessary directories for storing the results of the kitti native evaluation code. """ mlod_root_dir = mlod.root_dir() kitti_native_code_copy = mlod_root_dir + '/data/outputs/' + \ checkpoint_name + '/predictions/kitti_native_eval/' # Only copy if the code has not been already copied over if not os.path.exists(kitti_native_code_copy): os.makedirs(kitti_native_code_copy) original_kitti_native_code = mlod.top_dir() + \ '/scripts/offline_eval/kitti_native_eval/' predictions_dir = mlod_root_dir + '/data/outputs/' + \ checkpoint_name + '/predictions/' # create dir for it first dir_util.copy_tree(original_kitti_native_code, kitti_native_code_copy) # run the script to compile the c++ code script_folder = predictions_dir + \ '/kitti_native_eval/' make_script = script_folder + 'run_make.sh' subprocess.call([make_script, script_folder]) # Set up the results folders if they don't exist results_dir = mlod.top_dir() + '/scripts/offline_eval/results' results_05_dir = mlod.top_dir() + '/scripts/offline_eval/results_05_iou' if not os.path.exists(results_dir): os.makedirs(results_dir) if not os.path.exists(results_05_dir): os.makedirs(results_05_dir)
def __init__(self, dataset): self._dataset = dataset self.cluster_split = dataset.cluster_split self.data_dir = mlod.root_dir() + "/data/label_clusters" self.clusters = [] self.std_devs = []
def main(_): parser = argparse.ArgumentParser() # Example usage # --checkpoint_name='mlod_exp_example' # --data_split='test' # --ckpt_indices=50 100 112 # Optional arg: # --device=0 parser.add_argument('--checkpoint_name', type=str, dest='checkpoint_name', required=True, help='Checkpoint name must be specified as a str\ and must match the experiment config file name.') parser.add_argument('--data_split', type=str, dest='data_split', required=True, help='Data split must be specified e.g. val or test') parser.add_argument('--ckpt_indices', type=int, nargs='+', dest='ckpt_indices', required=True, help='Checkpoint indices must be a set of \ integers with space in between -> 0 10 20 etc') parser.add_argument('--device', type=str, dest='device', default='0', help='CUDA device id') args = parser.parse_args() if len(sys.argv) == 1: parser.print_help() sys.exit(1) experiment_config = args.checkpoint_name + '.config' # Read the config from the experiment folder experiment_config_path = mlod.root_dir() + '/data/outputs/' +\ args.checkpoint_name + '/' + experiment_config model_config, _, eval_config, dataset_config = \ config_builder.get_configs_from_pipeline_file( experiment_config_path, is_training=False) os.environ['CUDA_VISIBLE_DEVICES'] = args.device inference(model_config, eval_config, dataset_config, args.data_split, args.ckpt_indices)
def main(): dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL) # get proposals proposal_output_dir = mlod.root_dir() + \ "/data/predictions/rpn_model/proposals_and_scores/" + \ dataset.data_split global_steps = os.listdir(proposal_output_dir) print('Checkpoints found ', global_steps) all_recalls = [] for step in global_steps: for sample_name in dataset.sample_list: img_idx = int(sample_name) # ------------------------------------ # Load proposals and scores from files # ------------------------------------ proposals_scores_dir = proposal_output_dir + \ "{}/{}/{}.txt".format(dataset.data_split, step, sample_name) if not os.path.exists(proposals_scores_dir): print('File {} not found, skipping'.format(sample_name)) continue proposals_scores = np.loadtxt(proposals_scores_dir) proposals = proposals_scores[:, 0:-1] proposal_iou_format = \ box_3d_encoder.box_3d_to_3d_iou_format(proposals) # scores are in the last column scores = proposals_scores[:, -1] # ----------------------- # Get ground truth labels # ----------------------- gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) _, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects( gt_objects, ['Car', 'car']) score_thresholds = np.array([0.3]) iou_threshold = 0.0025 # calculate RPN recall and precision precision, recall = evaluation.evaluate_3d([gt_3d_bbs], [proposal_iou_format], [scores], score_thresholds, iou_threshold) print('Recall ', recall[0]) print('Precision ', precision[0]) all_recalls.append(recall)
def inference(sess, image): # Download VGG16 url = "http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz" # checkpoints_dir = '/tmp/mlod/checkpoints' checkpoints_dir = mlod.root_dir() + '/checkpoints/vgg_16' if not tf.gfile.Exists(checkpoints_dir): tf.gfile.MakeDirs(checkpoints_dir) # Download vgg16 if it hasn't been downloaded yet if not os.path.isfile(checkpoints_dir + "/vgg_16_2016_08_28.tar.gz"): dataset_utils.download_and_uncompress_tarball(url, checkpoints_dir) else: print("Already downloaded") # Set the image size to [224, 224] image_size = vgg.vgg_16.default_image_size # Pre-process input with tf.name_scope('input_reshape'): processed_image = vgg_preprocessing.preprocess_image( image, image_size, image_size, is_training=False) image_summary = tf.expand_dims(image, 0) processed_images = tf.expand_dims(processed_image, 0) tf.summary.image('images', image_summary, max_outputs=5) tf.summary.image('processed_images', processed_images, max_outputs=5) # Create the model, use the default arg scope to configure the # batch norm parameters. with slim.arg_scope(vgg.vgg_arg_scope()): # 1000 classes instead of 1001. logits, end_points = vgg.vgg_16( processed_images, num_classes=1000, is_training=False) probabilities = tf.nn.softmax(logits) # Add some images for feature maps from conv layers vis_feature_maps(end_points, 'vgg_16/conv1/conv1_1') vis_feature_maps(end_points, 'vgg_16/conv2/conv2_1') vis_feature_maps(end_points, 'vgg_16/conv3/conv3_1') vis_feature_maps(end_points, 'vgg_16/conv4/conv4_1') vis_feature_maps(end_points, 'vgg_16/conv5/conv5_1') vis_feature_maps(end_points, 'vgg_16/conv5/conv5_3') # Initialize vgg16 weights init_fn = slim.assign_from_checkpoint_fn( os.path.join(checkpoints_dir, 'vgg_16.ckpt'), slim.get_model_variables('vgg_16')) init_fn(sess) return logits, probabilities
def setUpClass(cls): pipeline_config = pipeline_pb2.NetworkPipelineConfig() dataset_config = pipeline_config.dataset_config config_path = mlod.root_dir() + '/configs/unittest_model.config' cls.model_config = config_build.get_model_config_from_file(config_path) input_config = cls.model_config.input_config input_config.bev_depth = 5 # unittest-kitti has [2, 1, 1] clusters dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST) cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def main(): # Profile large csv csv_file_path = mlod.root_dir() + \ '/data/mini_batches/iou_2d/kitti/train/depth/People[ 0.3 0.3]/007463.csv' ################### # Numpy loadtxt ################### np_data = np.loadtxt(csv_file_path, delimiter=',') ################### # Numpy genfromtxt ################### np_data = np.genfromtxt(csv_file_path, delimiter=',') ################### # Pandas read_csv ################### start_time = time.time() pandas_data = pandas.read_csv(csv_file_path).values print('pandas.read_csv took :', time.time() - start_time) start_time = time.time() # Profile small csv calib_dir = mlod.root_dir() + \ '/tests/datasets/Kitti/object/training/calib/000001.txt' start_time = time.time() data_file = open(calib_dir, 'r') csv_reader = csv.reader(data_file, delimiter=' ') data = [] for row in csv_reader: data.append(row) print('csv.reader took :', time.time() - start_time) pandas_data = pandas.read_csv(csv_file_path).values
def main(_): parser = argparse.ArgumentParser() default_rpn_config_path = mlod.root_dir() + \ '/configs/stagewise_rpn_example.config' parser.add_argument('--pipeline_rpn_config', type=str, dest='rpn_config_path', default=default_rpn_config_path, help='Path to the rpn config') default_mlod_config_path = mlod.root_dir() + \ '/configs/stagewise_mlod_example.config' parser.add_argument('--mlod_config_path', type=str, dest='mlod_config_path', default=default_mlod_config_path, help='Path to the mlod config') args = parser.parse_args() rpn_model_config, rpn_train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( args.rpn_config_path, is_training=True, stagewise_training=True) mlod_model_config, mlod_train_config, _, _ = \ config_builder.get_configs_from_pipeline_file( args.mlod_config_path, is_training=True, stagewise_training=True) os.environ['CUDA_VISIBLE_DEVICES'] = '1' train(rpn_model_config, mlod_model_config, rpn_train_config, mlod_train_config, dataset_config)
def fill_feed_dict(input_pl): image_name = 'cat.jpg' # image_name = 'school_bus.jpg' # image_name = 'table.jpg' cv2_bgr_image = cv2.imread(mlod.root_dir() + '/tests/images/' + image_name) # BGR -> RGB rgb_image = cv2_bgr_image[..., ::-1] feed_dict = { # input_pl: bev_image, input_pl: rgb_image, } return feed_dict
def setUp(self): tf.test.TestCase.setUp(self) test_pipeline_config_path = mlod.root_dir() + \ '/configs/unittest_stagewise.config' self.model_config, self.train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) # unittest-kitti has [2, 1, 1] clusters input_config = self.model_config.input_config input_config.bev_depth = 5 # Generate dataset self.dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_UNITTEST, use_defaults=False, new_cfg=dataset_config)
def main(_): os.environ['CUDA_VISIBLE_DEVICES'] = '1' parser = argparse.ArgumentParser() default_train_config_path = mlod.root_dir() + \ '/configs/mlod_exp_example.config' parser.add_argument('--pipeline_config', type=str, dest='pipeline_config_path', default=default_train_config_path, help='Path to the train config') args = parser.parse_args() model_config, train_config, eval_config, dataset_config = \ config_builder.get_configs_from_pipeline_file( args.pipeline_config_path, is_training=True) train_and_eval(model_config, train_config, eval_config, dataset_config)
def main(_): parser = argparse.ArgumentParser() # Defaults default_pipeline_config_path = mlod.root_dir() + \ '/configs/mlod_exp_example.config' default_data_split = 'train' default_device = '0' parser.add_argument('--pipeline_config', type=str, dest='pipeline_config_path', default=default_pipeline_config_path, help='Path to the pipeline config') parser.add_argument('--data_split', type=str, dest='data_split', default=default_data_split, help='Data split for training') parser.add_argument('--device', type=str, dest='device', default=default_device, help='CUDA device id') args = parser.parse_args() # Parse pipeline config model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( args.pipeline_config_path, is_training=True) # Overwrite data split dataset_config.data_split = args.data_split # Set CUDA device id os.environ['CUDA_VISIBLE_DEVICES'] = args.device train(model_config, train_config, dataset_config)
def run_kitti_native_script(checkpoint_name, score_threshold, global_step, sub_score_threshold): """Runs the kitti native code script.""" eval_script_dir = mlod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' make_script = eval_script_dir + \ '/kitti_native_eval/run_eval.sh' script_folder = eval_script_dir + \ '/kitti_native_eval/' results_dir = mlod.top_dir() + '/scripts/offline_eval/results/' # Round this because protobuf encodes default values as full decimal score_threshold = round(score_threshold, 3) print('evaluating....') subprocess.call([ make_script, script_folder, str(score_threshold), str(global_step), str(checkpoint_name), str(results_dir), sub_score_threshold ])
def main(): """This demo shows RPN proposals and MLOD predictions in the 3D point cloud. Keys: F1: Toggle proposals F2: Toggle predictions F3: Toggle 3D voxel grid F4: Toggle point cloud F5: Toggle easy ground truth objects (Green) F6: Toggle medium ground truth objects (Orange) F7: Toggle hard ground truth objects (Red) F8: Toggle all ground truth objects (default off) F9: Toggle ground slice filter (default off) F10: Toggle offset slice filter (default off) """ ############################## # Options ############################## rpn_score_threshold = 0.1 mlod_score_threshold = 0.1 proposals_line_width = 1.0 predictions_line_width = 3.0 show_orientations = True point_cloud_source = 'lidar' # Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>) config_dir = None checkpoint_name = 'mlod_fpn_people_n_m' global_step = 135000 # Latest checkpoint #data_split = 'val_half' #data_split = 'val' data_split = 'test' # Show 3D iou text draw_ious_3d = False sample_name = '000031' # # # Cars # # # # sample_name = '000050' # sample_name = '000104' # sample_name = '000169' # sample_name = '000191' # sample_name = '000360' # sample_name = '001783' # sample_name = '001820' # val split # sample_name = '000181' # sample_name = '000751' # sample_name = '000843' # sample_name = '000944' # sample_name = '006338' # # # People # # # # val_half split # sample_name = '000001' # Hard, 1 far cyc # sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # val split # sample_name = '000015' # sample_name = '000048' # sample_name = '000058' # sample_name = '000076' # Medium, few ped, 1 cyc # sample_name = '000108' # sample_name = '000118' # sample_name = '000145' # sample_name = '000153' # sample_name = '000186' # sample_name = '000195' # sample_name = '000199' # sample_name = '000397' # sample_name = '004425' # sample_name = '004474' # Hard, many ped, 1 cyc # sample_name = '004657' # Hard, Few cycl, few ped # sample_name = '006071' # sample_name = '006828' # Hard, Few cycl, few ped # sample_name = '006908' # Hard, Few cycl, few ped # sample_name = '007412' # sample_name = '007318' # Hard, Few cycl, few ped ############################## # End of Options ############################## if data_split == 'test': draw_ious_3d = False if config_dir is None: config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name # Parse experiment config pipeline_config_file = \ config_dir + '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) dataset_config.data_split = data_split if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_names[sample_idx] ############################## # Setup Paths ############################## img_idx = int(sample_name) # Text files directory proposals_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/proposals_and_scores/' + dataset.data_split predictions_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/final_predictions_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] # Output images directory img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/predictions/images_3d/{}/{}/{}'.format(dataset.data_split, global_step, rpn_score_threshold) if not os.path.exists(img_out_dir): os.makedirs(img_out_dir) ############################## # Proposals ############################## # Load proposals from files proposals_and_scores = np.loadtxt( proposals_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) proposals = proposals_and_scores[:, 0:7] proposal_scores = proposals_and_scores[:, 7] rpn_score_mask = proposal_scores > rpn_score_threshold proposals = proposals[rpn_score_mask] proposal_scores = proposal_scores[rpn_score_mask] print('Proposals:', len(proposal_scores), proposal_scores) proposal_objs = \ [box_3d_encoder.box_3d_to_object_label(proposal, obj_type='Proposal') for proposal in proposals] ############################## # Predictions ############################## # Load proposals from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32) mlod_score_mask = prediction_scores >= mlod_score_threshold prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask] prediction_scores = prediction_scores[mlod_score_mask] print('Predictions: ', len(prediction_scores), prediction_scores) final_predictions = np.copy(prediction_boxes_3d) # # Swap l, w for predictions where w > l # swapped_indices = predictions[:, 4] > predictions[:, 3] # final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4] # final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3] prediction_objs = [] dataset.classes = ['Pedestrian', 'Cyclist', 'Car'] for pred_idx in range(len(final_predictions)): prediction_box_3d = final_predictions[pred_idx] prediction_type = dataset.classes[prediction_types[pred_idx]] prediction_obj = box_3d_encoder.box_3d_to_object_label( prediction_box_3d, obj_type=prediction_type) prediction_objs.append(prediction_obj) ############################## # Ground Truth ############################## dataset.has_labels = False if dataset.has_labels: # Get ground truth labels easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = \ demo_utils.get_gts_based_on_difficulty(dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] ############################## # 3D IoU ############################## if draw_ious_3d: # Convert to box_3d all_gt_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in all_gt_objs ] pred_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(pred_obj) for pred_obj in prediction_objs ] max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d, pred_boxes_3d) ############################## # Point Cloud ############################## image_path = dataset.get_rgb_image_path(sample_name) image = cv2.imread(image_path) point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source, img_idx, image_shape=image.shape) point_cloud = np.asarray(point_cloud) # Filter point cloud to extents area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]]) bev_extents = area_extents[[0, 2]] points = point_cloud.T point_filter = obj_utils.get_point_filter(point_cloud, area_extents) points = points[point_filter] point_colours = vis_utils.project_img_to_point_cloud( points, image, dataset.calib_dir, img_idx) # Voxelize the point cloud for visualization voxel_grid = VoxelGrid() voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False) # Ground plane ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) ############################## # Visualization ############################## # Create VtkVoxelGrid vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(voxel_grid) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # Create VtkAxes vtk_axes = vtk.vtkAxesActor() vtk_axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for proposal boxes vtk_proposal_boxes = VtkBoxes() vtk_proposal_boxes.set_line_width(proposals_line_width) vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS) # Create VtkBoxes for prediction boxes vtk_prediction_boxes = VtkPyramidBoxes() vtk_prediction_boxes.set_line_width(predictions_line_width) vtk_prediction_boxes.set_objects(prediction_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) # Create VtkBoxes for ground truth vtk_hard_gt_boxes = VtkBoxes() vtk_medium_gt_boxes = VtkBoxes() vtk_easy_gt_boxes = VtkBoxes() vtk_all_gt_boxes = VtkBoxes() vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI, show_orientations) # Create VtkTextLabels for 3D ious vtk_text_labels = VtkTextLabels() if draw_ious_3d and len(all_gt_boxes_3d) > 0: gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3] vtk_text_labels.set_text_labels( gt_positions_3d, ['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d]) # Create VtkGroundPlane vtk_ground_plane = VtkGroundPlane() vtk_slice_bot_plane = VtkGroundPlane() vtk_slice_top_plane = VtkGroundPlane() vtk_ground_plane.set_plane(ground_plane, bev_extents) vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents) vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_text_labels.vtk_actor) # Add ground plane and slice planes vtk_renderer.AddActor(vtk_ground_plane.vtk_actor) vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor) vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor) #vtk_renderer.AddActor(vtk_axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Set initial properties for some actors vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3) vtk_proposal_boxes.vtk_actor.SetVisibility(0) vtk_voxel_grid.vtk_actor.SetVisibility(0) vtk_all_gt_boxes.vtk_actor.SetVisibility(0) vtk_ground_plane.vtk_actor.SetVisibility(0) vtk_slice_bot_plane.vtk_actor.SetVisibility(0) vtk_slice_top_plane.vtk_actor.SetVisibility(0) vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9) vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9) vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(3.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName( "Predictions: Step {}, Sample {}, Min Score {}".format( global_step, sample_name, mlod_score_threshold, )) vtk_render_window.SetSize(900, 600) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) # Add custom interactor to toggle actor visibilities custom_interactor = vis_utils.CameraInfoInteractorStyle([ vtk_proposal_boxes.vtk_actor, vtk_prediction_boxes.vtk_actor, vtk_voxel_grid.vtk_actor, vtk_point_cloud.vtk_actor, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_ground_plane.vtk_actor, vtk_slice_bot_plane.vtk_actor, vtk_slice_top_plane.vtk_actor, vtk_text_labels.vtk_actor, ]) vtk_render_window_interactor.SetInteractorStyle(custom_interactor) # Render in VTK vtk_render_window.Render() # Take a screenshot window_to_image_filter = vtk.vtkWindowToImageFilter() window_to_image_filter.SetInput(vtk_render_window) window_to_image_filter.Update() png_writer = vtk.vtkPNGWriter() file_name = img_out_dir + "/{}.png".format(sample_name) png_writer.SetFileName(file_name) png_writer.SetInputData(window_to_image_filter.GetOutput()) png_writer.Write() print('Screenshot saved to ', file_name) vtk_render_window_interactor.Start() # Blocking
def main(): """This demo visualizes box 8C format predicted by MLOD, before getting converted to Box 3D. Keys: F1: Toggle predictions F2: Toggle easy ground truth objects (Green) F3: Toggle medium ground truth objects (Orange) F4: Toggle hard ground truth objects (Red) F5: Toggle all ground truth objects (default off) F6: Toggle 3D voxel grid F7: Toggle point cloud """ ############################## # Options ############################## mlod_score_threshold = 0.1 show_orientations = True checkpoint_name = 'mlod_exp_8c' global_step = None sample_name = None dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF) dataset = DatasetBuilder.build_kitti_dataset(dataset_config) ############################## # Setup Paths ############################## # # # Cars # # # # sample_name = '000050' # sample_name = '000104' # sample_name = '000169' # sample_name = '000191' # sample_name = '000360' # sample_name = '001783' # sample_name = '001820' # sample_name = '006338' # # # People # # # # val_half split # sample_name = '000001' sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_list[sample_idx] img_idx = int(sample_name) # Text files directory predictions_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/final_boxes_8c_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(predictions_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] ############################## # predictions ############################## # Load predictions from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) predictions_boxes_8c = predictions_and_scores[:, 0:24] prediction_scores = predictions_and_scores[:, 24] score_mask = prediction_scores >= mlod_score_threshold predictions_boxes_8c = predictions_boxes_8c[score_mask] all_vtk_box_corners = [] predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8]) for i in range(len(predictions_boxes_8c)): box_8c = predictions_boxes_8c[i, :, :] vtk_box_corners = VtkBox8c() vtk_box_corners.set_objects(box_8c) all_vtk_box_corners.append(vtk_box_corners) ############################## # Ground Truth ############################## if dataset.has_labels: easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = \ demo_utils.get_gts_based_on_difficulty(dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] ############################## # Point Cloud ############################## image_path = dataset.get_rgb_image_path(sample_name) image = cv2.imread(image_path) img_idx = int(sample_name) points, point_colours = demo_utils.get_filtered_pc_and_colours( dataset, image, img_idx) # Voxelize the point cloud for visualization voxel_grid = VoxelGrid() voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False) ############################## # Visualization ############################## # Create VtkVoxelGrid vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(voxel_grid) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for ground truth vtk_easy_gt_boxes, vtk_medium_gt_boxes, \ vtk_hard_gt_boxes, vtk_all_gt_boxes = \ demo_utils.create_gt_vtk_boxes(easy_gt_objs, medium_gt_objs, hard_gt_objs, all_gt_objs, show_orientations) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_box_actors = vtk.vtkAssembly() # Create VtkBoxes for prediction boxes for i in range(len(all_vtk_box_corners)): # Adding labels, slows down rendering # vtk_renderer.AddActor(all_vtk_box_corners[i]. # vtk_text_labels.vtk_actor) vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.SetBackground(0.2, 0.3, 0.4) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_box_actors) vtk_renderer.AddActor(axes) # Set initial properties for some actors vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2) vtk_voxel_grid.vtk_actor.SetVisibility(0) vtk_all_gt_boxes.vtk_actor.SetVisibility(0) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName( "Predictions: Step {}, Sample {}, Min Score {}".format( global_step, sample_name, mlod_score_threshold, )) vtk_render_window.SetSize(900, 600) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_box_actors, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor, vtk_point_cloud.vtk_actor ])) vtk_render_window_interactor.Start()
def main(): """ Converts a set of network predictions into text files required for KITTI evaluation. """ ############################## # Options ############################## checkpoint_name = 'mlod_exp_example' # data_split = 'val' data_split = 'val_half' global_steps = None # global_steps = [28000, 19000, 33000, 34000] score_threshold = 0.1 save_2d = False # Save 2D predictions save_3d = True # Save 2D and 3D predictions together # Checkpoints below this are skipped min_step = 20000 # Object Type obj_type = 'obj' ############################## # End of Options ############################## # Parse experiment config pipeline_config_file = \ mlod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) # Overwrite defaults dataset_config = config_builder_util.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.aug_list = [] if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Get available prediction folders predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' proposals_root_dir = predictions_root_dir + \ '/proposals_and_scores/' + dataset.data_split print('Converting proposals from', proposals_root_dir) if not global_steps: global_steps = os.listdir(proposals_root_dir) global_steps.sort(key=int) print('Checkpoints found ', global_steps) for step_idx in range(len(global_steps)): global_step = global_steps[step_idx] # Skip first checkpoint if int(global_step) < min_step: continue final_predictions_dir = proposals_root_dir + \ '/' + str(global_step) # 2D and 3D prediction directories kitti_predictions_2d_dir = predictions_root_dir + \ '/kitti_proposals_2d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' kitti_proposals_3d_dir = predictions_root_dir + \ '/kitti_proposals_3d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' if save_2d and not os.path.exists(kitti_predictions_2d_dir): os.makedirs(kitti_predictions_2d_dir) if save_3d and not os.path.exists(kitti_proposals_3d_dir): os.makedirs(kitti_proposals_3d_dir) # Do conversion num_samples = dataset.num_samples num_valid_samples = 0 print('\nGlobal step:', global_step) print('Converting proposals from:', final_predictions_dir) if save_2d: print('2D Detections saved to:', kitti_predictions_2d_dir) if save_3d: print('Proposals saved to:', kitti_proposals_3d_dir) for sample_idx in range(num_samples): # Print progress sys.stdout.write('\rConverting {} / {}'.format( sample_idx + 1, num_samples)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] prediction_file = sample_name + '.txt' kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \ '/' + prediction_file kitti_predictions_3d_file_path = kitti_proposals_3d_dir + \ '/' + prediction_file predictions_file_path = final_predictions_dir + \ '/' + prediction_file # If no predictions, skip to next file if not os.path.exists(predictions_file_path): if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue all_predictions = np.loadtxt(predictions_file_path) # Swap l, w for predictions where w > l swapped_indices = all_predictions[:, 4] > all_predictions[:, 3] fixed_predictions = np.copy(all_predictions) fixed_predictions[swapped_indices, 3] = all_predictions[swapped_indices, 4] fixed_predictions[swapped_indices, 4] = all_predictions[swapped_indices, 3] fixed_predictions[swapped_indices, 6] = np.pi / 2 score_filter = all_predictions[:, 7] >= score_threshold all_predictions = fixed_predictions[score_filter] # If no predictions, skip to next file if len(all_predictions) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue # Project to image space sample_name = prediction_file.split('.')[0] img_idx = int(sample_name) # Load image for truncation image = Image.open(dataset.get_rgb_image_path(sample_name)) stereo_calib_p2 = calib_utils.read_calibration( dataset.calib_dir, img_idx).p2 boxes = [] image_filter = [] for i in range(len(all_predictions)): box_3d = all_predictions[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, stereo_calib_p2, truncate=True, image_size=image.size, discard_before_truncation=False) # Skip invalid boxes (outside image space) if img_box is None: image_filter.append(False) else: image_filter.append(True) boxes.append(img_box) boxes = np.asarray(boxes) all_predictions = all_predictions[image_filter] # If no predictions, skip to next file if len(boxes) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue num_valid_samples += 1 # To keep each value in its appropriate position, an array of zeros # (N, 16) is allocated but only values [4:16] are used kitti_predictions = np.zeros([len(boxes), 16]) # Truncation and Occlusion are always empty (see below) # Alpha (Not computed) kitti_predictions[:, 3] = -10 * np.ones( (len(kitti_predictions)), dtype=np.int32) # 2D predictions kitti_predictions[:, 4:8] = boxes[:, 0:4] # 3D predictions # (l, w, h) kitti_predictions[:, 8] = all_predictions[:, 5] kitti_predictions[:, 9] = all_predictions[:, 4] kitti_predictions[:, 10] = all_predictions[:, 3] # (x, y, z) kitti_predictions[:, 11:14] = all_predictions[:, 0:3] # (ry, score) kitti_predictions[:, 14:16] = all_predictions[:, 6:8] # Round detections to 3 decimal places kitti_predictions = np.round(kitti_predictions, 3) # Empty Truncation, Occlusion kitti_empty_1 = -1 * np.ones( (len(kitti_predictions), 2), dtype=np.int32) # Empty 3D (x, y, z) kitti_empty_2 = -1 * np.ones( (len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (h, w, l) kitti_empty_3 = -1000 * np.ones( (len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (ry) kitti_empty_4 = -10 * np.ones( (len(kitti_predictions), 1), dtype=np.int32) # Create Type Array obj_types = [obj_type for i in range(len(kitti_predictions))] # Stack 2D predictions text kitti_text_2d = np.column_stack([ obj_types, kitti_empty_1, kitti_predictions[:, 3:8], kitti_empty_2, kitti_empty_3, kitti_empty_4, kitti_predictions[:, 15] ]) # Stack 3D predictions text kitti_text_3d = np.column_stack( [obj_types, kitti_empty_1, kitti_predictions[:, 3:16]]) # Save to text files if save_2d: np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d, newline='\r\n', fmt='%s') if save_3d: np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d, newline='\r\n', fmt='%s') print('\nNum valid:', num_valid_samples) print('Num samples:', num_samples)
class DatasetBuilder(object): """ Static class to return preconfigured dataset objects """ KITTI_UNITTEST = KittiDatasetConfig( name="unittest-kitti", dataset_dir=mlod.root_dir() + "/tests/datasets/Kitti/object", data_split="train", data_split_dir="training", has_labels=True, cluster_split="train", classes=["Car", "Pedestrian", "Cyclist"], num_clusters=[2, 1, 1], ) KITTI_TRAIN = KittiDatasetConfig(name="kitti", data_split="train", data_split_dir="training", has_labels=True, cluster_split="train", classes=["Car"], num_clusters=[2]) KITTI_VAL = KittiDatasetConfig( name="kitti", data_split="val", data_split_dir="training", has_labels=True, cluster_split="train", classes=["Car"], num_clusters=[2], ) KITTI_TEST = KittiDatasetConfig( name="kitti", data_split="test", data_split_dir="testing", has_labels=False, cluster_split="train", classes=["Car"], num_clusters=[2], ) KITTI_TRAINVAL = KittiDatasetConfig( name="kitti", data_split="trainval", data_split_dir="training", has_labels=True, cluster_split="trainval", classes=["Car"], num_clusters=[2], ) KITTI_TRAIN_MINI = KittiDatasetConfig( name="kitti", data_split="train_mini", data_split_dir="training", has_labels=True, cluster_split="train", classes=["Car"], num_clusters=[2], ) KITTI_VAL_MINI = KittiDatasetConfig( name="kitti", data_split="val_mini", data_split_dir="training", has_labels=True, cluster_split="train", classes=["Car"], num_clusters=[2], ) KITTI_TEST_MINI = KittiDatasetConfig( name="kitti", data_split="test_mini", data_split_dir="testing", has_labels=False, cluster_split="train", classes=["Car"], num_clusters=[2], ) KITTI_VAL_HALF = KittiDatasetConfig( name="kitti", data_split="val_half", data_split_dir="training", has_labels=True, cluster_split="train", classes=["Car"], num_clusters=[2], ) CONFIG_DEFAULTS_PROTO = \ """ bev_source: 'stereo' kitti_utils_config { area_extents: [-40, 40, -5, 3, 0, 70] voxel_size: 0.1 anchor_strides: [0.5, 0.5, 0.2, 0.2, 0.3, 0.3] bev_generator { height_priors { ground_filter_offset: 0.2 offset_filter_distance: 2.0 } # slices { # height_lo: -0.5 # height_hi: 2.5 # num_slices: 5 # } } mini_batch_config { density_threshold: 1 rpn_config { iou_2d_thresholds { neg_iou_lo: 0.001 neg_iou_hi: 0.05 pos_iou_lo: 0.3 pos_iou_hi: 1.0 } # iou_3d_thresholds { # neg_iou_lo: 0.001 # neg_iou_hi: 0.005 # pos_iou_lo: 0.1 # pos_iou_hi: 1.0 # } mini_batch_size: 512 } mlod_config { iou_2d_thresholds { neg_iou_lo: 0.0 neg_iou_hi: 0.5 pos_iou_lo: 0.7 pos_iou_hi: 1.0 } mini_batch_size: 2000 } } } """ @staticmethod def load_dataset_from_config(dataset_config_path): dataset_config = kitti_dataset_pb2.KittiDatasetConfig() with open(dataset_config_path, 'r') as f: text_format.Merge(f.read(), dataset_config) return DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) @staticmethod def copy_config(cfg): return deepcopy(cfg) @staticmethod def merge_defaults(cfg): cfg_copy = DatasetBuilder.copy_config(cfg) text_format.Merge(DatasetBuilder.CONFIG_DEFAULTS_PROTO, cfg_copy) return cfg_copy @staticmethod def build_kitti_dataset(base_cfg, use_defaults=True, new_cfg=None) -> KittiDataset: """Builds a KittiDataset object using the provided configurations Args: base_cfg: a base dataset configuration use_defaults: whether to use the default config values new_cfg: (optional) a custom dataset configuration, no default values will be used, all config values must be provided Returns: KittiDataset object """ cfg_copy = DatasetBuilder.copy_config(base_cfg) if use_defaults: # Use default values text_format.Merge(DatasetBuilder.CONFIG_DEFAULTS_PROTO, cfg_copy) if new_cfg: # Use new config values if provided cfg_copy.MergeFrom(new_cfg) return KittiDataset(cfg_copy)
def main(dataset=None): """Generates anchors info which is used for mini batch sampling. Processing on 'Cars' can be split into multiple processes, see the Options section for configuration. Args: dataset: KittiDataset (optional) If dataset is provided, only generate info for that dataset. If no dataset provided, generates info for all 3 classes. """ if dataset is not None: do_preprocessing(dataset, None) return car_dataset_config_path = mlod.root_dir() + \ '/configs/mb_preprocessing/rpn_cars.config' ped_dataset_config_path = mlod.root_dir() + \ '/configs/mb_preprocessing/rpn_pedestrians.config' cyc_dataset_config_path = mlod.root_dir() + \ '/configs/mb_preprocessing/rpn_cyclists.config' ppl_dataset_config_path = mlod.root_dir() + \ '/configs/mb_preprocessing/rpn_people.config' ############################## # Options ############################## # Serial vs parallel processing in_parallel = True process_car = True # Cars process_ped = False # Pedestrians process_cyc = False # Cyclists process_ppl = True # People (Pedestrians + Cyclists) # Number of child processes to fork, samples will # be divided evenly amongst the processes (in_parallel must be True) num_car_children = 8 num_ped_children = 8 num_cyc_children = 8 num_ppl_children = 8 ############################## # Dataset setup ############################## if process_car: car_dataset = DatasetBuilder.load_dataset_from_config( car_dataset_config_path) if process_ped: ped_dataset = DatasetBuilder.load_dataset_from_config( ped_dataset_config_path) if process_cyc: cyc_dataset = DatasetBuilder.load_dataset_from_config( cyc_dataset_config_path) if process_ppl: ppl_dataset = DatasetBuilder.load_dataset_from_config( ppl_dataset_config_path) ############################## # Serial Processing ############################## if not in_parallel: if process_car: do_preprocessing(car_dataset, None) if process_ped: do_preprocessing(ped_dataset, None) if process_cyc: do_preprocessing(cyc_dataset, None) if process_ppl: do_preprocessing(ppl_dataset, None) print('All Done (Serial)') ############################## # Parallel Processing ############################## else: # List of all child pids to wait on all_child_pids = [] # Cars if process_car: car_indices_split = split_indices(car_dataset, num_car_children) split_work(all_child_pids, car_dataset, car_indices_split, num_car_children) # Pedestrians if process_ped: ped_indices_split = split_indices(ped_dataset, num_ped_children) split_work(all_child_pids, ped_dataset, ped_indices_split, num_ped_children) # Cyclists if process_cyc: cyc_indices_split = split_indices(cyc_dataset, num_cyc_children) split_work(all_child_pids, cyc_dataset, cyc_indices_split, num_cyc_children) # People (Pedestrians + Cyclists) if process_ppl: ppl_indices_split = split_indices(ppl_dataset, num_ppl_children) split_work(all_child_pids, ppl_dataset, ppl_indices_split, num_ppl_children) # Wait to child processes to finish print('num children:', len(all_child_pids)) for i, child_pid in enumerate(all_child_pids): os.waitpid(child_pid, 0) print('All Done (Parallel)')
def main(): """ This demo shows example mini batch info for full MlodModel training. This includes ground truth, ortho rotated ground truth, negative proposal anchors, positive proposal anchors, and a sampled mini batch. The 2D iou can be modified to show the effect of changing the iou threshold for mini batch sampling. In order to let this demo run without training an RPN, the proposals shown are being read from a text file. Keys: F1: Toggle ground truth F2: Toggle ortho rotated ground truth F3: Toggle negative proposal anchors F4: Toggle positive proposal anchors F5: Toggle mini batch anchors """ ############################## # Options ############################## # Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>) config_dir = None # checkpoint_name = None checkpoint_name = 'mlod_exp_example' data_split = 'val_half' # global_step = None global_step = 100000 # # # Cars # # # # sample_name = "000050" sample_name = "000104" # sample_name = "000764" # # # People # # # # val_half # sample_name = '000001' # Hard, 1 far cyc # sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc mini_batch_size = 512 neg_proposal_2d_iou_hi = 0.6 pos_proposal_2d_iou_lo = 0.65 bkg_proposals_line_width = 0.5 neg_proposals_line_width = 0.5 mid_proposals_line_width = 0.5 pos_proposals_line_width = 1.0 ############################## # End of Options ############################## img_idx = int(sample_name) print("Showing mini batch for sample {}".format(sample_name)) # Read proposals from file if checkpoint_name is None: # Use VAL Dataset dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL) # Load demo proposals proposals_and_scores_dir = mlod.top_dir() + \ '/demos/data/predictions/' + checkpoint_name + \ '/proposals_and_scores/' + dataset.data_split else: if config_dir is None: config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name # Parse experiment config pipeline_config_file = \ config_dir + '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) dataset_config.data_split = data_split dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Overwrite mini_batch_utils = dataset.kitti_utils.mini_batch_utils mini_batch_utils.mlod_neg_iou_range[1] = neg_proposal_2d_iou_hi mini_batch_utils.mlod_pos_iou_range[0] = pos_proposal_2d_iou_lo # Load proposals from outputs folder proposals_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + \ '/predictions/proposals_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] proposals_and_scores = np.loadtxt( proposals_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) proposal_boxes_3d = proposals_and_scores[:, 0:7] proposal_anchors = box_3d_encoder.box_3d_to_anchor(proposal_boxes_3d) # Get filtered ground truth obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx) filtered_objs = dataset.kitti_utils.filter_labels(obj_labels) # Convert ground truth to anchors gt_boxes_3d = np.asarray([ box_3d_encoder.object_label_to_box_3d(obj_label) for obj_label in filtered_objs ]) gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d, ortho_rotate=True) # Ortho rotate ground truth gt_ortho_boxes_3d = box_3d_encoder.anchors_to_box_3d(gt_anchors) gt_ortho_objs = [ box_3d_encoder.box_3d_to_object_label(box_3d, obj_type='OrthoGt') for box_3d in gt_ortho_boxes_3d ] # Project gt and anchors into BEV gt_bev_anchors, _ = \ anchor_projector.project_to_bev(gt_anchors, dataset.kitti_utils.bev_extents) bev_anchors, _ = \ anchor_projector.project_to_bev(proposal_anchors, dataset.kitti_utils.bev_extents) # Reorder boxes into (y1, x1, y2, x2) order gt_bev_anchors_tf_order = anchor_projector.reorder_projected_boxes( gt_bev_anchors) bev_anchors_tf_order = anchor_projector.reorder_projected_boxes( bev_anchors) # Convert to box_list format for iou calculation gt_anchor_box_list = box_list.BoxList( tf.cast(gt_bev_anchors_tf_order, tf.float32)) anchor_box_list = box_list.BoxList( tf.cast(bev_anchors_tf_order, tf.float32)) # Get IoU for every anchor tf_all_ious = box_list_ops.iou(gt_anchor_box_list, anchor_box_list) valid_ious = True # Make sure the calculated IoUs contain values. Since its a [N, M] # tensor, if there are no gt's for instance, that entry will be zero. if tf_all_ious.shape[0] == 0 or tf_all_ious.shape[1] == 0: print('#################################################') print('Warning: This sample does not contain valid IoUs') print('#################################################') valid_ious = False if valid_ious: tf_max_ious = tf.reduce_max(tf_all_ious, axis=0) tf_max_iou_indices = tf.argmax(tf_all_ious, axis=0) # Sample an RPN mini batch from the non empty anchors mini_batch_utils = dataset.kitti_utils.mini_batch_utils # Overwrite mini batch size and sample a mini batch mini_batch_utils.mlod_mini_batch_size = mini_batch_size mb_mask_tf, _ = mini_batch_utils.sample_mlod_mini_batch(tf_max_ious) # Create a session config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) # Run the graph to calculate ious for every proposal and # to get the mini batch mask all_ious, max_ious, max_iou_indices = sess.run( [tf_all_ious, tf_max_ious, tf_max_iou_indices]) mb_mask = sess.run(mb_mask_tf) mb_anchors = proposal_anchors[mb_mask] mb_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(mb_anchors) mb_anchor_ious = max_ious[mb_mask] else: # We have no valid IoU's, so assume all IoUs are zeros # and the mini-batch contains all the anchors since we cannot # mask without IoUs. max_ious = np.zeros(proposal_boxes_3d.shape[0]) mb_anchor_ious = max_ious mb_anchors = proposal_anchors mb_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(mb_anchors) # Create list of positive/negative proposals based on iou pos_proposal_objs = [] mid_proposal_objs = [] neg_proposal_objs = [] bkg_proposal_objs = [] for i in range(len(proposal_boxes_3d)): box_3d = proposal_boxes_3d[i] if max_ious[i] == 0.0: # Background proposals bkg_proposal_objs.append( box_3d_encoder.box_3d_to_object_label( box_3d, obj_type='BackgroundProposal')) elif max_ious[i] < neg_proposal_2d_iou_hi: # Negative proposals neg_proposal_objs.append( box_3d_encoder.box_3d_to_object_label( box_3d, obj_type='NegativeProposal')) elif max_ious[i] < pos_proposal_2d_iou_lo: # Middle proposals (in between negative and positive) mid_proposal_objs.append( box_3d_encoder.box_3d_to_object_label( box_3d, obj_type='MiddleProposal')) elif max_ious[i] <= 1.0: # Positive proposals pos_proposal_objs.append( box_3d_encoder.box_3d_to_object_label( box_3d, obj_type='PositiveProposal')) else: raise ValueError('Invalid IoU > 1.0') print('{} bkg, {} neg, {} mid, {} pos proposals:'.format( len(bkg_proposal_objs), len(neg_proposal_objs), len(mid_proposal_objs), len(pos_proposal_objs))) # Convert the mini_batch anchors to object list mb_obj_list = [] for i in range(len(mb_anchor_ious)): if valid_ious and (mb_anchor_ious[i] > mini_batch_utils.mlod_pos_iou_range[0]): obj_type = "Positive" else: obj_type = "Negative" obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i], obj_type) mb_obj_list.append(obj) # Point cloud image = cv2.imread(dataset.get_rgb_image_path(sample_name)) points, point_colours = demo_utils.get_filtered_pc_and_colours( dataset, image, img_idx) # Visualize from here vis_utils.visualization(dataset.rgb_image_dir, img_idx) plt.show(block=False) # VtkPointCloud vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # VtkBoxes for ground truth vtk_gt_boxes = VtkBoxes() vtk_gt_boxes.set_objects(filtered_objs, COLOUR_SCHEME) # VtkBoxes for ortho ground truth vtk_gt_ortho_boxes = VtkBoxes() vtk_gt_ortho_boxes.set_objects(gt_ortho_objs, COLOUR_SCHEME) # VtkBoxes for background proposals vtk_bkg_proposal_boxes = VtkBoxes() vtk_bkg_proposal_boxes.set_objects(bkg_proposal_objs, COLOUR_SCHEME) vtk_bkg_proposal_boxes.set_line_width(bkg_proposals_line_width) # VtkBoxes for negative proposals vtk_neg_proposal_boxes = VtkBoxes() vtk_neg_proposal_boxes.set_objects(neg_proposal_objs, COLOUR_SCHEME) vtk_neg_proposal_boxes.set_line_width(neg_proposals_line_width) # VtkBoxes for middle proposals vtk_mid_proposal_boxes = VtkBoxes() vtk_mid_proposal_boxes.set_objects(mid_proposal_objs, COLOUR_SCHEME) vtk_mid_proposal_boxes.set_line_width(mid_proposals_line_width) # VtkBoxes for positive proposals vtk_pos_proposal_boxes = VtkBoxes() vtk_pos_proposal_boxes.set_objects(pos_proposal_objs, COLOUR_SCHEME) vtk_pos_proposal_boxes.set_line_width(pos_proposals_line_width) # Create VtkBoxes for mini batch anchors vtk_mb_boxes = VtkBoxes() vtk_mb_boxes.set_objects(mb_obj_list, COLOUR_SCHEME) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Add actors vtk_renderer.AddActor(axes) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_gt_ortho_boxes.vtk_actor) vtk_renderer.AddActor(vtk_bkg_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_neg_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_mid_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_pos_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_mb_boxes.vtk_actor) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("MLOD Mini Batch") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_gt_boxes.vtk_actor, vtk_gt_ortho_boxes.vtk_actor, vtk_bkg_proposal_boxes.vtk_actor, vtk_neg_proposal_boxes.vtk_actor, vtk_mid_proposal_boxes.vtk_actor, vtk_pos_proposal_boxes.vtk_actor, vtk_mb_boxes.vtk_actor, ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def main(): # Create Dataset dataset_config_path = mlod.root_dir() + \ '/configs/mb_preprocessing/rpn_cars.config' dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path) # Random sample sample_name = '000169' anchor_strides = dataset.kitti_utils.anchor_strides img_idx = int(sample_name) print("Showing mini batch for sample {}".format(sample_name)) image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = [image.shape[1], image.shape[0]] # KittiUtils class dataset_utils = dataset.kitti_utils ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) point_cloud = obj_utils.get_depth_map_point_cloud(img_idx, dataset.calib_dir, dataset.depth_dir, image_shape) # Grab ground truth ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx) ground_truth_list = dataset_utils.filter_labels(ground_truth_list) stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 ############################## # Flip sample info ############################## start_time = time.time() flipped_image = kitti_aug.flip_image(image) flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud) flipped_gt_list = [kitti_aug.flip_label_in_3d_only(obj) for obj in ground_truth_list] flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane) flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2( stereo_calib_p2, image_shape) flipped_points = flipped_point_cloud.T print('flip sample', time.time() - start_time) ############################## # Generate anchors ############################## clusters, _ = dataset.get_cluster_info() anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Read mini batch info anchors_info = dataset_utils.get_anchors_info( dataset.classes_name, anchor_strides, sample_name) all_anchor_boxes_3d = [] all_ious = [] for class_idx in range(len(dataset.classes)): anchor_boxes_3d = anchor_generator.generate( area_3d=dataset.kitti_utils.area_extents, anchor_3d_sizes=clusters[class_idx], anchor_stride=anchor_strides[class_idx], ground_plane=ground_plane) if anchors_info: indices, ious, offsets, classes = anchors_info # Get non empty anchors from the indices non_empty_anchor_boxes_3d = anchor_boxes_3d[indices] all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d) all_ious.extend(ious) if not len(all_anchor_boxes_3d) > 0: # Exit early if anchors_info is empty print("No anchors, Please try a different sample") return # Convert to ndarrays all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d) all_ious = np.asarray(all_ious) ############################## # Flip anchors ############################## start_time = time.time() flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d, flip_ry=False) print('flip anchors', time.time() - start_time) # Overwrite with flipped things all_anchor_boxes_3d = flipped_anchor_boxes_3d points = flipped_points ground_truth_list = flipped_gt_list ground_plane = flipped_ground_plane
import tensorflow as tf import numpy as np from tensorflow.python.framework import ops from tensorflow.python.ops import gen_nn_ops import mlod import mlod.builders.config_builder_util as config_builder from mlod.builders.dataset_builder import DatasetBuilder from wavedata.tools.obj_detection import obj_utils from mlod.core import constants from occlusion_mask_layer import OccMaskLayer parser = argparse.ArgumentParser() default_pipeline_config_path = mlod.root_dir() + \ '/configs/mlod_fpn_people.config' parser.add_argument('--pipeline_config', type=str, dest='pipeline_config_path', default=default_pipeline_config_path, help='Path to the pipeline config') parser.add_argument('--device', type=str, dest='device', default='0', help='CUDA device id') checkpoint_to_restore = ''
def __init__(self, dataset): self._dataset = dataset self._mini_batch_sampler = \ balanced_positive_negative_sampler.BalancedPositiveNegativeSampler() self._mini_batch_sampler2 = \ balanced_positive_negative_sampler2.BalancedPositiveNegativeSampler() ############################## # Parse KittiUtils config ############################## self.kitti_utils_config = dataset.config.kitti_utils_config self._area_extents = self.kitti_utils_config.area_extents self._anchor_strides = np.reshape( self.kitti_utils_config.anchor_strides, (-1, 2)) ############################## # Parse MiniBatchUtils config ############################## self.config = self.kitti_utils_config.mini_batch_config self._density_threshold = self.config.density_threshold # RPN mini batches rpn_config = self.config.rpn_config rpn_iou_type = rpn_config.WhichOneof('iou_type') if rpn_iou_type == 'iou_2d_thresholds': self.rpn_iou_type = '2d' self.rpn_iou_thresholds = rpn_config.iou_2d_thresholds elif rpn_iou_type == 'iou_3d_thresholds': self.rpn_iou_type = '3d' self.rpn_iou_thresholds = rpn_config.iou_3d_thresholds self.rpn_neg_iou_range = [self.rpn_iou_thresholds.neg_iou_lo, self.rpn_iou_thresholds.neg_iou_hi] self.rpn_pos_iou_range = [self.rpn_iou_thresholds.pos_iou_lo, self.rpn_iou_thresholds.pos_iou_hi] rpn_img_config = self.config.rpn_img_config self.rpn_img_iou_thresholds = rpn_img_config.iou_2d_thresholds self.rpn_img_neg_iou_range = [self.rpn_img_iou_thresholds.neg_iou_lo, self.rpn_img_iou_thresholds.neg_iou_hi] self.rpn_img_pos_iou_range = [self.rpn_img_iou_thresholds.pos_iou_lo, self.rpn_img_iou_thresholds.pos_iou_hi] self.rpn_mini_batch_size = rpn_config.mini_batch_size # MLOD mini batches mlod_config = self.config.mlod_config self.mlod_iou_type = '2d' self.mlod_iou_thresholds = mlod_config.iou_2d_thresholds self.mlod_neg_iou_range = [self.mlod_iou_thresholds.neg_iou_lo, self.mlod_iou_thresholds.neg_iou_hi] self.mlod_pos_iou_range = [self.mlod_iou_thresholds.pos_iou_lo, self.mlod_iou_thresholds.pos_iou_hi] self.mlod_mini_batch_size = mlod_config.mini_batch_size mlod_img_config = self.config.mlod_img_config self.mlod_img_iou_thresholds = mlod_img_config.iou_2d_thresholds self.mlod_img_neg_iou_range = [self.mlod_img_iou_thresholds.neg_iou_lo, self.mlod_img_iou_thresholds.neg_iou_hi] self.mlod_img_pos_iou_range = [self.mlod_img_iou_thresholds.pos_iou_lo, self.mlod_img_iou_thresholds.pos_iou_hi] self.mlod_img_mini_batch_size = mlod_img_config.mini_batch_size # Setup paths self.mini_batch_dir = mlod.root_dir() + '/data/mini_batches/' + \ 'iou_{}/'.format(self.rpn_iou_type) + \ dataset.name + '/' + dataset.cluster_split + '/' + \ dataset.bev_source # Array column indices for saving to files self.col_length = 15 self.col_anchor_indices = 0 self.col_ious = 1 self.col_offsets_lo = 2 self.col_offsets_hi = 8 self.col_class_idx = 8 self.col_img_ious = 9 self.col_img_offsets_lo = 10 self.col_img_offsets_hi = 14 self.col_img_class_idx = 14
def save_predictions_in_kitti_format(sample_names, checkpoint_name, final_predictions_root_dir, branch_predictions_root_dirs, kitti_predictions_3d_dir, score_threshold, sub_score_threshold, global_step): """ Converts a set of network predictions into text files required for KITTI evaluation. """ # Round this because protobuf encodes default values as full decimal score_threshold = round(score_threshold, 3) # Get available prediction folders predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' final_predictions_dir = final_predictions_root_dir + \ '/' + str(global_step) branch_prediction_dir_list = [ branch_dir + '/' + str(global_step) for branch_dir in branch_predictions_root_dirs ] if not os.path.exists(kitti_predictions_3d_dir): os.makedirs(kitti_predictions_3d_dir) # Do conversion num_samples = len(sample_names) num_valid_samples = 0 print('\nGlobal step:', global_step) print('Converting detections from:', final_predictions_dir) print('3D Detections being saved to:', kitti_predictions_3d_dir) for sample_idx, sample_name in enumerate(sample_names): # Print progress sys.stdout.write('\rConverting {} / {}'.format(sample_idx + 1, num_samples)) sys.stdout.flush() prediction_file = sample_name + '.txt' kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \ '/' + prediction_file predictions_file_path = final_predictions_dir + \ '/' + prediction_file # If no predictions, skip to next file if not os.path.exists(predictions_file_path): np.savetxt(kitti_predictions_3d_file_path, []) continue all_predictions = np.loadtxt(predictions_file_path) sub_predictions_list = [] for branch_prediction_dir in branch_prediction_dir_list: sub_predictions_file_path = branch_prediction_dir + \ '/' + prediction_file sub_predictions = np.loadtxt(sub_predictions_file_path) sub_predictions_list.append(sub_predictions) # # Swap l, w for predictions where w > l # swapped_indices = all_predictions[:, 4] > all_predictions[:, 3] # fixed_predictions = np.copy(all_predictions) # fixed_predictions[swapped_indices, 3] = all_predictions[ # swapped_indices, 4] # fixed_predictions[swapped_indices, 4] = all_predictions[ # swapped_indices, 3] score_filter = np.array(all_predictions[:, 7] >= score_threshold) #print('main',score_filter) score_filter1 = np.array( sub_predictions_list[0][:, 7] >= sub_score_threshold[0]) #print('br0',score_filter1) score_filter2 = np.array( sub_predictions_list[1][:, 7] >= sub_score_threshold[1]) #print('br1',score_filter2) all_predictions = all_predictions[score_filter & score_filter1 & score_filter2] # If no predictions, skip to next file if len(all_predictions) == 0: np.savetxt(kitti_predictions_3d_file_path, []) continue # Project to image space sample_name = prediction_file.split('.')[0] img_idx = int(sample_name) # Load image for truncation image = Image.open(get_rgb_image_path(sample_name)) calib_dir = '/home/j7deng/Kitti/object/training/calib' stereo_calib_p2 = calib_utils.read_calibration(calib_dir, img_idx).p2 boxes = [] image_filter = [] for i in range(len(all_predictions)): box_3d = all_predictions[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, stereo_calib_p2, truncate=True, image_size=image.size) # Skip invalid boxes (outside image space) if img_box is None: image_filter.append(False) continue image_filter.append(True) boxes.append(img_box) boxes = np.asarray(boxes) all_predictions = all_predictions[image_filter] # If no predictions, skip to next file if len(boxes) == 0: np.savetxt(kitti_predictions_3d_file_path, []) continue num_valid_samples += 1 # To keep each value in its appropriate position, an array of zeros # (N, 16) is allocated but only values [4:16] are used kitti_predictions = np.zeros([len(boxes), 16]) # Get object types all_pred_classes = all_predictions[:, 8].astype(np.int32) dataset_classes = ['Pedestrian', 'Cyclist'] obj_types = [ dataset_classes[class_idx] for class_idx in all_pred_classes ] # Truncation and Occlusion are always empty (see below) # Alpha (Not computed) kitti_predictions[:, 3] = -10 * np.ones( (len(kitti_predictions)), dtype=np.int32) # 2D predictions kitti_predictions[:, 4:8] = boxes[:, 0:4] # 3D predictions # (l, w, h) kitti_predictions[:, 8] = all_predictions[:, 5] kitti_predictions[:, 9] = all_predictions[:, 4] kitti_predictions[:, 10] = all_predictions[:, 3] # (x, y, z) kitti_predictions[:, 11:14] = all_predictions[:, 0:3] # (ry, score) kitti_predictions[:, 14:16] = all_predictions[:, 6:8] # Round detections to 3 decimal places kitti_predictions = np.round(kitti_predictions, 3) # Empty Truncation, Occlusion kitti_empty_1 = -1 * np.ones( (len(kitti_predictions), 2), dtype=np.int32) # Stack 3D predictions text kitti_text_3d = np.column_stack( [obj_types, kitti_empty_1, kitti_predictions[:, 3:16]]) # Save to text files np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d, newline='\r\n', fmt='%s') print('\nNum valid:', num_valid_samples) print('Num samples:', num_samples)
def main(): """Flip RPN Mini Batch Visualization of the mini batch anchors for RpnModel training. Keys: F1: Toggle mini batch anchors F2: Flipped """ anchor_colour_scheme = { "Car": (255, 0, 0), # Red "Pedestrian": (255, 150, 50), # Orange "Cyclist": (150, 50, 100), # Purple "DontCare": (255, 255, 255), # White "Anchor": (150, 150, 150), # Gray "Regressed Anchor": (255, 255, 0), # Yellow "Positive": (0, 255, 255), # Teal "Negative": (255, 0, 255) # Purple } dataset_config_path = mlod.root_dir() + \ '/configs/mb_rpn_demo_cars.config' # dataset_config_path = mlod.root_dir() + \ # '/configs/mb_rpn_demo_people.config' ############################## # Options ############################## # # # Random sample # # # sample_name = None # # # Cars # # # # sample_name = "000001" # sample_name = "000050" # sample_name = "000104" # sample_name = "000112" # sample_name = "000169" # sample_name = "000191" sample_name = "003801" # # # Pedestrians # # # # sample_name = "000000" # sample_name = "000011" # sample_name = "000015" # sample_name = "000028" # sample_name = "000035" # sample_name = "000134" # sample_name = "000167" # sample_name = '000379' # sample_name = '000381' # sample_name = '000397' # sample_name = '000398' # sample_name = '000401' # sample_name = '000407' # sample_name = '000486' # sample_name = '000509' # # Cyclists # # # # sample_name = '000122' # sample_name = '000448' # # # Multiple classes # # # # sample_name = "000764" ############################## # End of Options ############################## # Create Dataset dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path) # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_list[sample_idx] anchor_strides = dataset.kitti_utils.anchor_strides img_idx = int(sample_name) print("Showing mini batch for sample {}".format(sample_name)) image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = [image.shape[1], image.shape[0]] # KittiUtils class dataset_utils = dataset.kitti_utils ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) point_cloud = obj_utils.get_depth_map_point_cloud(img_idx, dataset.calib_dir, dataset.depth_dir, image_shape) points = point_cloud.T # Grab ground truth ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx) ground_truth_list = dataset_utils.filter_labels(ground_truth_list) stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 ############################## # Flip sample info ############################## start_time = time.time() flipped_image = kitti_aug.flip_image(image) flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud) flipped_gt_list = [ kitti_aug.flip_label_in_3d_only(obj) for obj in ground_truth_list ] flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane) flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2(stereo_calib_p2, image_shape) print('flip sample', time.time() - start_time) flipped_points = flipped_point_cloud.T point_colours = vis_utils.project_img_to_point_cloud( points, image, dataset.calib_dir, img_idx) ############################## # Generate anchors ############################## clusters, _ = dataset.get_cluster_info() anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Read mini batch info anchors_info = dataset_utils.get_anchors_info(sample_name) all_anchor_boxes_3d = [] all_ious = [] all_offsets = [] for class_idx in range(len(dataset.classes)): anchor_boxes_3d = anchor_generator.generate( area_3d=dataset.kitti_utils.area_extents, anchor_3d_sizes=clusters[class_idx], anchor_stride=anchor_strides[class_idx], ground_plane=ground_plane) if len(anchors_info[class_idx]) > 0: indices, ious, offsets, classes = anchors_info[class_idx] # Get non empty anchors from the indices non_empty_anchor_boxes_3d = anchor_boxes_3d[indices] all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d) all_ious.extend(ious) all_offsets.extend(offsets) if not len(all_anchor_boxes_3d) > 0: # Exit early if anchors_info is empty print("No anchors, Please try a different sample") return # Convert to ndarrays all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d) all_ious = np.asarray(all_ious) all_offsets = np.asarray(all_offsets) ############################## # Flip anchors ############################## start_time = time.time() # Flip anchors and offsets flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d, flip_ry=False) all_offsets[:, 0] = -all_offsets[:, 0] print('flip anchors and offsets', time.time() - start_time) # Overwrite with flipped things all_anchor_boxes_3d = flipped_anchor_boxes_3d points = flipped_points ground_truth_list = flipped_gt_list ground_plane = flipped_ground_plane ############################## # Mini batch sampling ############################## # Sample an RPN mini batch from the non empty anchors mini_batch_utils = dataset.kitti_utils.mini_batch_utils mb_mask_tf, _ = mini_batch_utils.sample_rpn_mini_batch(all_ious) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) mb_mask = sess.run(mb_mask_tf) mb_anchor_boxes_3d = all_anchor_boxes_3d[mb_mask] mb_anchor_ious = all_ious[mb_mask] mb_anchor_offsets = all_offsets[mb_mask] # ObjectLabel list that hold all boxes to visualize obj_list = [] # Convert the mini_batch anchors to object list for i in range(len(mb_anchor_boxes_3d)): if mb_anchor_ious[i] > mini_batch_utils.rpn_pos_iou_range[0]: obj_type = "Positive" else: obj_type = "Negative" obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i], obj_type) obj_list.append(obj) # Convert all non-empty anchors to object list non_empty_anchor_objs = \ [box_3d_encoder.box_3d_to_object_label( anchor_box_3d, obj_type='Anchor') for anchor_box_3d in all_anchor_boxes_3d] ############################## # Regress Positive Anchors ############################## # Convert anchor_boxes_3d to anchors and apply offsets mb_pos_mask = mb_anchor_ious > mini_batch_utils.rpn_pos_iou_range[0] mb_pos_anchor_boxes_3d = mb_anchor_boxes_3d[mb_pos_mask] mb_pos_anchor_offsets = mb_anchor_offsets[mb_pos_mask] mb_pos_anchors = box_3d_encoder.box_3d_to_anchor(mb_pos_anchor_boxes_3d) regressed_pos_anchors = anchor_encoder.offset_to_anchor( mb_pos_anchors, mb_pos_anchor_offsets) # Convert regressed anchors to ObjectLabels for visualization regressed_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d( regressed_pos_anchors, fix_lw=True) regressed_anchor_objs = \ [box_3d_encoder.box_3d_to_object_label( box_3d, obj_type='Regressed Anchor') for box_3d in regressed_anchor_boxes_3d] ############################## # Visualization ############################## cv2.imshow('{} flipped'.format(sample_name), flipped_image) cv2.waitKey() # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for mini batch anchors vtk_pos_anchor_boxes = VtkBoxes() vtk_pos_anchor_boxes.set_objects(obj_list, anchor_colour_scheme) # VtkBoxes for non empty anchors vtk_non_empty_anchors = VtkBoxes() vtk_non_empty_anchors.set_objects(non_empty_anchor_objs, anchor_colour_scheme) vtk_non_empty_anchors.set_line_width(0.1) # VtkBoxes for regressed anchors vtk_regressed_anchors = VtkBoxes() vtk_regressed_anchors.set_objects(regressed_anchor_objs, anchor_colour_scheme) vtk_regressed_anchors.set_line_width(5.0) # Create VtkBoxes for ground truth vtk_gt_boxes = VtkBoxes() vtk_gt_boxes.set_objects(ground_truth_list, anchor_colour_scheme, show_orientations=True) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) vtk_ground_plane = VtkGroundPlane() vtk_ground_plane.set_plane(ground_plane, dataset.kitti_utils.bev_extents) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_non_empty_anchors.vtk_actor) vtk_renderer.AddActor(vtk_pos_anchor_boxes.vtk_actor) vtk_renderer.AddActor(vtk_regressed_anchors.vtk_actor) vtk_renderer.AddActor(vtk_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_ground_plane.vtk_actor) vtk_renderer.AddActor(axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("RPN Mini Batch") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_non_empty_anchors.vtk_actor, vtk_pos_anchor_boxes.vtk_actor, vtk_regressed_anchors.vtk_actor, vtk_ground_plane.vtk_actor, ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def main(): """This demo shows RPN proposals and MLOD predictions in 3D and 2D in image space. Given certain thresholds for proposals and predictions, it selects and draws the bounding boxes on the image sample. It goes through the entire proposal and prediction samples for the given dataset split. The proposals, overlaid, and prediction images can be toggled on or off separately in the options section. The prediction score and IoU with ground truth can be toggled on or off as well, shown as (score, IoU) above the detection. """ dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL) ############################## # Options ############################## dataset_config.data_split = 'val' fig_size = (10, 6.1) rpn_score_threshold = 0.1 mlod_score_threshold = 0.1 # Flag for projecting the 3D boxes to image space # in tensor format (for testing purposes) test_img_tensor_projection = False gt_classes = ['Pedestrian', 'Cyclist'] # gt_classes = ['Pedestrian', 'Cyclist'] # Overwrite this to select a specific checkpoint global_step = 44000 checkpoint_name = 'mlod_fpn_people' # Drawing Toggles draw_proposals_separate = False draw_overlaid = False draw_predictions_separate = True # Show orientation for both GT and proposals/predictions draw_orientations_on_prop = False draw_orientations_on_pred = False # Draw 2D bounding boxes draw_projected_2d_boxes = False # Save images for samples with no detections save_empty_images = True draw_score = True draw_iou = False iou_3d = False ############################## # End of Options ############################## # Get the dataset dataset = DatasetBuilder.build_kitti_dataset(dataset_config) # Setup Paths predictions_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' proposals_and_scores_dir = predictions_dir + \ '/proposals_and_scores/' + dataset.data_split predictions_and_scores_dir = predictions_dir + \ '/final_predictions_and_scores/' + dataset.data_split # Output images directories output_dir_base = predictions_dir + '/images_2d' # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] if draw_proposals_separate: prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format( dataset.data_split, global_step, rpn_score_threshold) if not os.path.exists(prop_out_dir): os.makedirs(prop_out_dir) print('Proposal images saved to:', prop_out_dir) if draw_overlaid: overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format( dataset.data_split, global_step, mlod_score_threshold) if not os.path.exists(overlaid_out_dir): os.makedirs(overlaid_out_dir) print('Overlaid images saved to:', overlaid_out_dir) if draw_predictions_separate: pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format( dataset.data_split, global_step, mlod_score_threshold) if not os.path.exists(pred_out_dir): os.makedirs(pred_out_dir) print('Prediction images saved to:', pred_out_dir) # Rolling average array of times for time estimation avg_time_arr_length = 10 last_times = np.repeat(time.time(), avg_time_arr_length) + \ np.arange(avg_time_arr_length) for sample_idx in range(dataset.num_samples): # Estimate time remaining with 5 slowest times start_time = time.time() last_times = np.roll(last_times, -1) last_times[-1] = start_time avg_time = np.mean(np.sort(np.diff(last_times))[-5:]) samples_remaining = dataset.num_samples - sample_idx est_time_left = avg_time * samples_remaining # Print progress and time remaining estimate sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, ' 'Time Remaining: {:.2f}s'. format( sample_idx + 1, dataset.num_samples, avg_time, est_time_left)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] img_idx = int(sample_name) ############################## # Proposals ############################## if draw_proposals_separate or draw_overlaid: # Load proposals from files proposals_file_path = proposals_and_scores_dir + \ "/{}/{}.txt".format(global_step, sample_name) if not os.path.exists(proposals_file_path): print('Sample {}: No proposals, skipping'.format(sample_name)) continue print('Sample {}: Drawing proposals'.format(sample_name)) proposals_and_scores = np.loadtxt(proposals_file_path) proposal_boxes_3d = proposals_and_scores[:, 0:7] proposal_scores = proposals_and_scores[:, 7] # Apply score mask to proposals score_mask = proposal_scores > rpn_score_threshold proposal_boxes_3d = proposal_boxes_3d[score_mask] proposal_scores = proposal_scores[score_mask] proposal_objs = \ [box_3d_encoder.box_3d_to_object_label(proposal, obj_type='Proposal') for proposal in proposal_boxes_3d] ############################## # Predictions ############################## if draw_predictions_separate or draw_overlaid: predictions_file_path = predictions_and_scores_dir + \ "/{}/{}.txt".format(global_step, sample_name) if not os.path.exists(predictions_file_path): continue # Load predictions from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_class_indices = predictions_and_scores[:, 8] # process predictions only if we have any predictions left after # masking if len(prediction_boxes_3d) > 0: # Apply score mask mlod_score_mask = prediction_scores >= 0.1 mlod_show_mask = mlod_score_mask prediction_boxes_3d = prediction_boxes_3d[mlod_show_mask] prediction_scores = prediction_scores[mlod_show_mask] prediction_class_indices = \ prediction_class_indices[mlod_show_mask] # # Swap l, w for predictions where w > l # swapped_indices = \ # prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3] # prediction_boxes_3d = np.copy(prediction_boxes_3d) # prediction_boxes_3d[swapped_indices, 3] = \ # prediction_boxes_3d[swapped_indices, 4] # prediction_boxes_3d[swapped_indices, 4] = \ # prediction_boxes_3d[swapped_indices, 3] ############################## # Ground Truth ############################## # Get ground truth labels dataset.has_labels = False if dataset.has_labels: gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) else: gt_objects = [] # Filter objects to desired difficulty filtered_gt_objs = dataset.kitti_utils.filter_labels( gt_objects, classes=gt_classes) boxes2d, _, _ = obj_utils.build_bbs_from_objects( filtered_gt_objs, class_needed=gt_classes) image_path = dataset.get_rgb_image_path(sample_name) image = Image.open(image_path) image_size = image.size # Read the stereo calibration matrix for visualization stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx) calib_p2 = stereo_calib.p2 ############################## # Reformat and prepare to draw ############################## if draw_proposals_separate or draw_overlaid: proposals_as_anchors = box_3d_encoder.box_3d_to_anchor( proposal_boxes_3d) if test_img_tensor_projection: proposal_boxes = demo_utils.tf_project_to_image_space( proposals_as_anchors, calib_p2, image_size, img_idx) else: proposal_boxes, _ = anchor_projector.project_to_image_space( proposals_as_anchors, calib_p2, image_size) num_of_proposals = proposal_boxes_3d.shape[0] prop_fig, prop_2d_axes, prop_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False) draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals, proposal_objs, proposal_boxes, prop_2d_axes, prop_3d_axes, draw_orientations_on_prop) if draw_proposals_separate: # Save just the proposals filename = prop_out_dir + '/' + sample_name + '.png' plt.savefig(filename) if not draw_overlaid: plt.close(prop_fig) if draw_overlaid or draw_predictions_separate: if len(prediction_boxes_3d) > 0: # Project the 3D box predictions to image space image_filter = [] final_boxes_2d = [] for i in range(len(prediction_boxes_3d)): box_3d = prediction_boxes_3d[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, calib_p2, truncate=True, image_size=image_size, discard_before_truncation=False) if img_box is not None: image_filter.append(True) final_boxes_2d.append(img_box) else: image_filter.append(False) final_boxes_2d = np.asarray(final_boxes_2d) final_prediction_boxes_3d = prediction_boxes_3d[image_filter] final_scores = prediction_scores[image_filter] final_class_indices = prediction_class_indices[image_filter] num_of_predictions = final_boxes_2d.shape[0] # Convert to objs final_prediction_objs = \ [box_3d_encoder.box_3d_to_object_label( prediction, obj_type='Prediction') for prediction in final_prediction_boxes_3d] for (obj, score) in zip(final_prediction_objs, final_scores): obj.score = score else: if save_empty_images: pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) filename = pred_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(pred_fig) continue if draw_overlaid: # Overlay prediction boxes on image draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, prop_2d_axes, prop_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred, iou_3d) filename = overlaid_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(prop_fig) if draw_predictions_separate: # Now only draw prediction boxes on images # on a new figure handler if draw_projected_2d_boxes: pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_2d_axes, pred_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred, iou_3d) else: pred_fig, pred_3d_axes = \ vis_utils.visualize_single_plot( dataset.rgb_image_dir, img_idx, display=False) draw_3d_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred) filename = pred_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(pred_fig) print('\nDone')