def set_up_model_test_mode(pipeline_config_path, data_split): """Returns the model and its config in test mode.""" model_config, _, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( pipeline_config_path, is_training=False) dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) # Use the validation set dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' # Remove augmentation when in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test='test', dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test='test', dataset=dataset) else: raise ValueError('Invalid model_name') return model, model_config
def set_up_model(): test_pipeline_config_path = mlod.root_dir() + \ '/configs/mlod_exp_example.config' model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) dataset_config = config_builder.proto_to_obj(dataset_config) train_val_test = 'test' dataset_config.data_split = 'test' dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset_config.aug_list = [] dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=train_val_test, dataset=dataset) else: raise ValueError('Invalid model_name') return model
def main(): dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN) image_path = dataset.get_rgb_image_path('000000') # cv2 cv2_image1 = cv2.imread(image_path) cv2_image2 = cv2.imread(image_path) cv2_image3 = cv2.imread(image_path) cv2_image1_shape = cv2_image1.shape cv2_image2_shape = cv2_image2.shape cv2_image3_shape = cv2_image3.shape print(cv2_image1_shape) print(cv2_image2_shape) print(cv2_image3_shape) # PIL pil_image1 = Image.open(image_path) pil_image2 = Image.open(image_path) pil_image3 = Image.open(image_path) pil_image1_shape = pil_image1.size pil_image2_shape = pil_image2.size pil_image3_shape = pil_image3.size print(pil_image1_shape) print(pil_image2_shape) print(pil_image3_shape) pil_image1 = np.asarray(pil_image1) pil_image2 = np.asarray(pil_image2) pil_image3 = np.asarray(pil_image3)
def main(): test_pipeline_config_path = mlod.root_dir() + \ '/data/configs/official/cars/cars_000_vanilla.config' model_config, train_config, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) # train_val_test = 'val' # dataset_config.data_split = 'val' train_val_test = 'test' dataset_config.data_split = 'trainval' dataset_config.data_split_dir = 'training' dataset_config.has_labels = False # dataset_config.cache_config.cache_images = True # dataset_config.cache_config.cache_depth_maps = True dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) kitti_utils = dataset.kitti_utils bev_source = 'lidar' # sample_name = '000000' # img_idx = np.random.randint(0, 1000) # sample_name = '{:06d}'.format(img_idx) num_samples = 200 all_load_times = [] all_bev_times = [] for sample_idx in range(num_samples): sys.stdout.write('\rSample {} / {}'.format(sample_idx, num_samples - 1)) img_idx = sample_idx sample_name = '{:06d}'.format(sample_idx) loading_start_time = time.time() # Load image image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = image.shape[0:2] calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx) point_cloud = kitti_utils.get_point_cloud(bev_source, int(sample_name), image_shape) ground_plane = kitti_utils.get_ground_plane(sample_name) all_load_times.append(time.time() - loading_start_time) bev_start_time = time.time() bev_maps = kitti_utils.create_bev_maps(point_cloud, ground_plane) bev_end_time = time.time() all_bev_times.append(bev_end_time - bev_start_time) print('') print('Load mean:', np.mean(all_load_times)) print('Load median:', np.median(all_load_times)) print('BEV mean:', np.mean(all_bev_times)) print('BEV median:', np.median(all_bev_times))
def main(): dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST) dataset_config.data_split = "trainval" unittest_dataset = DatasetBuilder.build_kitti_dataset(dataset_config) gen_label_clusters.main(unittest_dataset) gen_mini_batches.main(unittest_dataset)
def main(): dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL) # get proposals proposal_output_dir = mlod.root_dir() + \ "/data/predictions/rpn_model/proposals_and_scores/" + \ dataset.data_split global_steps = os.listdir(proposal_output_dir) print('Checkpoints found ', global_steps) all_recalls = [] for step in global_steps: for sample_name in dataset.sample_list: img_idx = int(sample_name) # ------------------------------------ # Load proposals and scores from files # ------------------------------------ proposals_scores_dir = proposal_output_dir + \ "{}/{}/{}.txt".format(dataset.data_split, step, sample_name) if not os.path.exists(proposals_scores_dir): print('File {} not found, skipping'.format(sample_name)) continue proposals_scores = np.loadtxt(proposals_scores_dir) proposals = proposals_scores[:, 0:-1] proposal_iou_format = \ box_3d_encoder.box_3d_to_3d_iou_format(proposals) # scores are in the last column scores = proposals_scores[:, -1] # ----------------------- # Get ground truth labels # ----------------------- gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) _, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects( gt_objects, ['Car', 'car']) score_thresholds = np.array([0.3]) iou_threshold = 0.0025 # calculate RPN recall and precision precision, recall = evaluation.evaluate_3d([gt_3d_bbs], [proposal_iou_format], [scores], score_thresholds, iou_threshold) print('Recall ', recall[0]) print('Precision ', precision[0]) all_recalls.append(recall)
def get_fake_dataset(self, data_split, directory): dataset_config = DatasetBuilder.copy_config( DatasetBuilder.KITTI_UNITTEST) # Overwrite config values dataset_config.data_split = data_split dataset_config.dataset_dir = directory dataset = DatasetBuilder.build_kitti_dataset(dataset_config) return dataset
def setUpClass(cls): # Initialize the Kitti dataset test_dir = tests.test_path() # Get the unittest-kitti dataset dataset_builder = DatasetBuilder() cls.dataset = dataset_builder.build_kitti_dataset( dataset_builder.KITTI_UNITTEST) cls.log_dir = test_dir + '/logs' cls.bev_vgg_cls = vgg.BevVggClassification()
def setUpClass(cls): pipeline_config = pipeline_pb2.NetworkPipelineConfig() dataset_config = pipeline_config.dataset_config config_path = mlod.root_dir() + '/configs/unittest_model.config' cls.model_config = config_build.get_model_config_from_file(config_path) input_config = cls.model_config.input_config input_config.bev_depth = 5 # unittest-kitti has [2, 1, 1] clusters dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST) cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def inference(model_config, eval_config, dataset_config, data_split, ckpt_indices): # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' eval_config.eval_mode = 'test' eval_config.evaluate_repeatedly = False dataset_config.has_labels = False # Enable this to see the actually memory being used eval_config.allow_gpu_mem_growth = True eval_config = config_builder.proto_to_obj(eval_config) # Grab the checkpoint indices to evaluate eval_config.ckpt_indices = ckpt_indices # Remove augmentation during evaluation in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Overwrite repeated field model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] with tf.Graph().as_default(): if model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) else: raise ValueError('Invalid model name {}'.format(model_name)) model_evaluator = Evaluator(model, dataset_config, eval_config) model_evaluator.run_latest_checkpoints()
def set_up_video_dataset(dataset_config, dataset_dir, data_split, data_split_dir): # Overwrite fields dataset_config.name = 'kitti_video' dataset_config.dataset_dir = dataset_dir dataset_config.data_split = data_split dataset_config.data_split_dir = data_split_dir dataset_config.has_labels = False # Overwrite repeated fields dataset_config = config_builder_util.proto_to_obj(dataset_config) dataset_config.aug_list = [] dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) return dataset
def setUp(self): tf.test.TestCase.setUp(self) test_pipeline_config_path = mlod.root_dir() + \ '/configs/unittest_stagewise.config' self.model_config, self.train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( test_pipeline_config_path, is_training=True) # unittest-kitti has [2, 1, 1] clusters input_config = self.model_config.input_config input_config.bev_depth = 5 # Generate dataset self.dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_UNITTEST, use_defaults=False, new_cfg=dataset_config)
def train(rpn_model_config, mlod_model_config, rpn_train_config, mlod_train_config, dataset_config): train_val_test = 'train' dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) paths_config = rpn_model_config.paths_config rpn_checkpoint_dir = paths_config.checkpoint_dir with tf.Graph().as_default(): model = RpnModel(rpn_model_config, train_val_test=train_val_test, dataset=dataset) trainer.train(model, rpn_train_config) # load the weights back in saver = tf.train.Saver() init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) trainer_utils.load_checkpoints(rpn_checkpoint_dir, saver) checkpoint_to_restore = saver.last_checkpoints[-1] trainer_utils.load_model_weights(model, sess, checkpoint_to_restore) # Merge RPN configs with MLOD - This will overwrite # the appropriate configs set for MLOD while keeping # the common configs the same. rpn_model_config.MergeFrom(mlod_model_config) rpn_train_config.MergeFrom(mlod_train_config) mlod_model_merged = deepcopy(rpn_model_config) mlod_train_merged = deepcopy(rpn_train_config) with tf.Graph().as_default(): model = MlodModel(mlod_model_merged, train_val_test=train_val_test, dataset=dataset) trainer.train(model, mlod_train_merged, stagewise_training=True, init_checkpoint_dir=rpn_checkpoint_dir)
def train(model_config, train_config, dataset_config): dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) train_val_test = 'train' model_name = model_config.model_name with tf.Graph().as_default(): if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=train_val_test, dataset=dataset) else: raise ValueError('Invalid model_name') trainer.train(model, train_config)
def set_up_model_train_mode(pipeline_config_path, data_split): """Returns the model and its train_op.""" model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( pipeline_config_path, is_training=True) dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=data_split, dataset=dataset) elif model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=data_split, dataset=dataset) else: raise ValueError('Invalid model_name') prediction_dict = model.build() losses_dict, total_loss = model.loss(prediction_dict) # These parameters are required to set up the optimizer global_summaries = set([]) global_step_tensor = tf.Variable(0, trainable=False) training_optimizer = optimizer_builder.build(train_config.optimizer, global_summaries, global_step_tensor) # Set up the train op train_op = slim.learning.create_train_op(total_loss, training_optimizer) return model, train_op
def setUpClass(cls): cls.dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_UNITTEST) cls.mb_utils = cls.dataset.kitti_utils.mini_batch_utils
def main(): """This demo shows RPN proposals and MLOD predictions in the 3D point cloud. Keys: F1: Toggle proposals F2: Toggle predictions F3: Toggle 3D voxel grid F4: Toggle point cloud F5: Toggle easy ground truth objects (Green) F6: Toggle medium ground truth objects (Orange) F7: Toggle hard ground truth objects (Red) F8: Toggle all ground truth objects (default off) F9: Toggle ground slice filter (default off) F10: Toggle offset slice filter (default off) """ ############################## # Options ############################## rpn_score_threshold = 0.1 mlod_score_threshold = 0.1 proposals_line_width = 1.0 predictions_line_width = 3.0 show_orientations = True point_cloud_source = 'lidar' # Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>) config_dir = None checkpoint_name = 'mlod_fpn_people_n_m' global_step = 135000 # Latest checkpoint #data_split = 'val_half' #data_split = 'val' data_split = 'test' # Show 3D iou text draw_ious_3d = False sample_name = '000031' # # # Cars # # # # sample_name = '000050' # sample_name = '000104' # sample_name = '000169' # sample_name = '000191' # sample_name = '000360' # sample_name = '001783' # sample_name = '001820' # val split # sample_name = '000181' # sample_name = '000751' # sample_name = '000843' # sample_name = '000944' # sample_name = '006338' # # # People # # # # val_half split # sample_name = '000001' # Hard, 1 far cyc # sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # val split # sample_name = '000015' # sample_name = '000048' # sample_name = '000058' # sample_name = '000076' # Medium, few ped, 1 cyc # sample_name = '000108' # sample_name = '000118' # sample_name = '000145' # sample_name = '000153' # sample_name = '000186' # sample_name = '000195' # sample_name = '000199' # sample_name = '000397' # sample_name = '004425' # sample_name = '004474' # Hard, many ped, 1 cyc # sample_name = '004657' # Hard, Few cycl, few ped # sample_name = '006071' # sample_name = '006828' # Hard, Few cycl, few ped # sample_name = '006908' # Hard, Few cycl, few ped # sample_name = '007412' # sample_name = '007318' # Hard, Few cycl, few ped ############################## # End of Options ############################## if data_split == 'test': draw_ious_3d = False if config_dir is None: config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name # Parse experiment config pipeline_config_file = \ config_dir + '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) dataset_config.data_split = data_split if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_names[sample_idx] ############################## # Setup Paths ############################## img_idx = int(sample_name) # Text files directory proposals_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/proposals_and_scores/' + dataset.data_split predictions_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/final_predictions_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] # Output images directory img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/predictions/images_3d/{}/{}/{}'.format(dataset.data_split, global_step, rpn_score_threshold) if not os.path.exists(img_out_dir): os.makedirs(img_out_dir) ############################## # Proposals ############################## # Load proposals from files proposals_and_scores = np.loadtxt( proposals_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) proposals = proposals_and_scores[:, 0:7] proposal_scores = proposals_and_scores[:, 7] rpn_score_mask = proposal_scores > rpn_score_threshold proposals = proposals[rpn_score_mask] proposal_scores = proposal_scores[rpn_score_mask] print('Proposals:', len(proposal_scores), proposal_scores) proposal_objs = \ [box_3d_encoder.box_3d_to_object_label(proposal, obj_type='Proposal') for proposal in proposals] ############################## # Predictions ############################## # Load proposals from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32) mlod_score_mask = prediction_scores >= mlod_score_threshold prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask] prediction_scores = prediction_scores[mlod_score_mask] print('Predictions: ', len(prediction_scores), prediction_scores) final_predictions = np.copy(prediction_boxes_3d) # # Swap l, w for predictions where w > l # swapped_indices = predictions[:, 4] > predictions[:, 3] # final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4] # final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3] prediction_objs = [] dataset.classes = ['Pedestrian', 'Cyclist', 'Car'] for pred_idx in range(len(final_predictions)): prediction_box_3d = final_predictions[pred_idx] prediction_type = dataset.classes[prediction_types[pred_idx]] prediction_obj = box_3d_encoder.box_3d_to_object_label( prediction_box_3d, obj_type=prediction_type) prediction_objs.append(prediction_obj) ############################## # Ground Truth ############################## dataset.has_labels = False if dataset.has_labels: # Get ground truth labels easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = \ demo_utils.get_gts_based_on_difficulty(dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] ############################## # 3D IoU ############################## if draw_ious_3d: # Convert to box_3d all_gt_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in all_gt_objs ] pred_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(pred_obj) for pred_obj in prediction_objs ] max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d, pred_boxes_3d) ############################## # Point Cloud ############################## image_path = dataset.get_rgb_image_path(sample_name) image = cv2.imread(image_path) point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source, img_idx, image_shape=image.shape) point_cloud = np.asarray(point_cloud) # Filter point cloud to extents area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]]) bev_extents = area_extents[[0, 2]] points = point_cloud.T point_filter = obj_utils.get_point_filter(point_cloud, area_extents) points = points[point_filter] point_colours = vis_utils.project_img_to_point_cloud( points, image, dataset.calib_dir, img_idx) # Voxelize the point cloud for visualization voxel_grid = VoxelGrid() voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False) # Ground plane ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) ############################## # Visualization ############################## # Create VtkVoxelGrid vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(voxel_grid) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # Create VtkAxes vtk_axes = vtk.vtkAxesActor() vtk_axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for proposal boxes vtk_proposal_boxes = VtkBoxes() vtk_proposal_boxes.set_line_width(proposals_line_width) vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS) # Create VtkBoxes for prediction boxes vtk_prediction_boxes = VtkPyramidBoxes() vtk_prediction_boxes.set_line_width(predictions_line_width) vtk_prediction_boxes.set_objects(prediction_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) # Create VtkBoxes for ground truth vtk_hard_gt_boxes = VtkBoxes() vtk_medium_gt_boxes = VtkBoxes() vtk_easy_gt_boxes = VtkBoxes() vtk_all_gt_boxes = VtkBoxes() vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS, show_orientations) vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI, show_orientations) # Create VtkTextLabels for 3D ious vtk_text_labels = VtkTextLabels() if draw_ious_3d and len(all_gt_boxes_3d) > 0: gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3] vtk_text_labels.set_text_labels( gt_positions_3d, ['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d]) # Create VtkGroundPlane vtk_ground_plane = VtkGroundPlane() vtk_slice_bot_plane = VtkGroundPlane() vtk_slice_top_plane = VtkGroundPlane() vtk_ground_plane.set_plane(ground_plane, bev_extents) vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents) vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor) vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_text_labels.vtk_actor) # Add ground plane and slice planes vtk_renderer.AddActor(vtk_ground_plane.vtk_actor) vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor) vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor) #vtk_renderer.AddActor(vtk_axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Set initial properties for some actors vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3) vtk_proposal_boxes.vtk_actor.SetVisibility(0) vtk_voxel_grid.vtk_actor.SetVisibility(0) vtk_all_gt_boxes.vtk_actor.SetVisibility(0) vtk_ground_plane.vtk_actor.SetVisibility(0) vtk_slice_bot_plane.vtk_actor.SetVisibility(0) vtk_slice_top_plane.vtk_actor.SetVisibility(0) vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9) vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9) vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(3.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName( "Predictions: Step {}, Sample {}, Min Score {}".format( global_step, sample_name, mlod_score_threshold, )) vtk_render_window.SetSize(900, 600) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) # Add custom interactor to toggle actor visibilities custom_interactor = vis_utils.CameraInfoInteractorStyle([ vtk_proposal_boxes.vtk_actor, vtk_prediction_boxes.vtk_actor, vtk_voxel_grid.vtk_actor, vtk_point_cloud.vtk_actor, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_ground_plane.vtk_actor, vtk_slice_bot_plane.vtk_actor, vtk_slice_top_plane.vtk_actor, vtk_text_labels.vtk_actor, ]) vtk_render_window_interactor.SetInteractorStyle(custom_interactor) # Render in VTK vtk_render_window.Render() # Take a screenshot window_to_image_filter = vtk.vtkWindowToImageFilter() window_to_image_filter.SetInput(vtk_render_window) window_to_image_filter.Update() png_writer = vtk.vtkPNGWriter() file_name = img_out_dir + "/{}.png".format(sample_name) png_writer.SetFileName(file_name) png_writer.SetInputData(window_to_image_filter.GetOutput()) png_writer.Write() print('Screenshot saved to ', file_name) vtk_render_window_interactor.Start() # Blocking
def main(): """ Visualization of anchor filtering using 3D integral images """ anchor_colour_scheme = { "Car": (0, 255, 0), # Green "Pedestrian": (255, 150, 50), # Orange "Cyclist": (150, 50, 100), # Purple "DontCare": (255, 0, 0), # Red "Anchor": (0, 0, 255), # Blue } # Create Dataset dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL) # Options clusters, _ = dataset.get_cluster_info() sample_name = "000000" img_idx = int(sample_name) anchor_stride = [0.5, 0.5] ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator( anchor_3d_sizes=clusters, anchor_stride=anchor_stride) area_extents = np.array([[-40, 40], [-5, 3], [0, 70]]) # Generate anchors in box_3d format start_time = time.time() anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents, ground_plane=ground_plane) end_time = time.time() print("Anchors generated in {} s".format(end_time - start_time)) point_cloud = obj_utils.get_lidar_point_cloud(img_idx, dataset.calib_dir, dataset.velo_dir) offset_dist = 2.0 # Filter points within certain xyz range and offset from ground plane offset_filter = obj_utils.get_point_filter(point_cloud, area_extents, ground_plane, offset_dist) # Filter points within 0.2m of the road plane road_filter = obj_utils.get_point_filter(point_cloud, area_extents, ground_plane, 0.1) slice_filter = np.logical_xor(offset_filter, road_filter) point_cloud = point_cloud.T[slice_filter] # Generate Voxel Grid vx_grid_3d = voxel_grid.VoxelGrid() vx_grid_3d.voxelize(point_cloud, 0.1, area_extents) # Anchors in anchor format all_anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) # Filter the boxes here! start_time = time.time() empty_filter = \ anchor_filter.get_empty_anchor_filter(anchors=all_anchors, voxel_grid_3d=vx_grid_3d, density_threshold=1) anchor_boxes_3d = anchor_boxes_3d[empty_filter] end_time = time.time() print("Anchors filtered in {} s".format(end_time - start_time)) # Visualize GT boxes # Grab ground truth ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx) # ---------- # Test Sample extraction # Visualize from here vis_utils.visualization(dataset.rgb_image_dir, img_idx) plt.show(block=False) image_path = dataset.get_rgb_image_path(sample_name) image_shape = np.array(Image.open(image_path)).shape rgb_boxes, rgb_normalized_boxes = \ anchor_projector.project_to_image_space(all_anchors, dataset, image_shape, img_idx) # Overlay boxes on images anchor_objects = [] for anchor_idx in range(len(anchor_boxes_3d)): anchor_box_3d = anchor_boxes_3d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label( anchor_box_3d, 'Anchor') # Append to a list for visualization in VTK later anchor_objects.append(obj_label) for idx in range(len(ground_truth_list)): ground_truth_obj = ground_truth_list[idx] # Append to a list for visualization in VTK later anchor_objects.append(ground_truth_obj) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for boxes vtk_boxes = VtkBoxes() vtk_boxes.set_objects(anchor_objects, anchor_colour_scheme) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(point_cloud) vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(vx_grid_3d) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_boxes.vtk_actor) # vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(170.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("Anchors") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vtk.vtkInteractorStyleTrackballCamera()) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start() # Blocking
def train_and_eval(model_config, train_config, eval_config, dataset_config): # Dataset Configuration dataset_config_train = DatasetBuilder.copy_config(dataset_config) dataset_config_eval = DatasetBuilder.copy_config(dataset_config) dataset_train = DatasetBuilder.build_kitti_dataset(dataset_config_train, use_defaults=False) dataset_eval = DatasetBuilder.build_kitti_dataset(dataset_config_eval, use_defaults=False) model_name = model_config.model_name train_val_test = 'train' eval_mode = eval_config.eval_mode if eval_mode == 'train': raise ValueError('Evaluation mode can only be set to `val` or `test`.') # keep a copy as this will be overwritten inside # the training loop below max_train_iter = train_config.max_iterations checkpoint_interval = train_config.checkpoint_interval eval_interval = eval_config.eval_interval if eval_interval < checkpoint_interval or \ (eval_interval % checkpoint_interval) != 0: raise ValueError( 'Checkpoint interval (given {}) must be greater than and' 'divisible by the evaluation interval (given {}).'.format( eval_interval, checkpoint_interval)) # Use the evaluation losses file to continue from the latest # checkpoint already_evaluated_ckpts = evaluator.get_evaluated_ckpts( model_config, model_name) if len(already_evaluated_ckpts) != 0: current_train_iter = already_evaluated_ckpts[-1] else: current_train_iter = eval_interval # while training is not finished while current_train_iter <= max_train_iter: # Train with tf.Graph().as_default(): if model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=train_val_test, dataset=dataset_train) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset_train) else: raise ValueError('Invalid model name {}'.format(model_name)) # overwrite the training epochs train_config.max_iterations = current_train_iter print('\n*************** Training ****************\n') trainer.train(model, train_config) current_train_iter += eval_interval # Evaluate with tf.Graph().as_default(): if model_name == 'mlod_model': model = MlodModel(model_config, train_val_test=eval_mode, dataset=dataset_eval) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=eval_mode, dataset=dataset_eval) else: raise ValueError('Invalid model name {}'.format(model_name)) print('\n*************** Evaluating *****************\n') evaluator.run_latest_checkpoints(model, dataset_config_eval) print('\n************ Finished training and evaluating *************\n')
def main(): """ Converts a set of network predictions into text files required for KITTI evaluation. """ ############################## # Options ############################## checkpoint_name = 'mlod_exp_example' # data_split = 'val' data_split = 'val_half' global_steps = None # global_steps = [28000, 19000, 33000, 34000] score_threshold = 0.1 save_2d = False # Save 2D predictions save_3d = True # Save 2D and 3D predictions together # Checkpoints below this are skipped min_step = 20000 # Object Type obj_type = 'obj' ############################## # End of Options ############################## # Parse experiment config pipeline_config_file = \ mlod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) # Overwrite defaults dataset_config = config_builder_util.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.aug_list = [] if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Get available prediction folders predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' proposals_root_dir = predictions_root_dir + \ '/proposals_and_scores/' + dataset.data_split print('Converting proposals from', proposals_root_dir) if not global_steps: global_steps = os.listdir(proposals_root_dir) global_steps.sort(key=int) print('Checkpoints found ', global_steps) for step_idx in range(len(global_steps)): global_step = global_steps[step_idx] # Skip first checkpoint if int(global_step) < min_step: continue final_predictions_dir = proposals_root_dir + \ '/' + str(global_step) # 2D and 3D prediction directories kitti_predictions_2d_dir = predictions_root_dir + \ '/kitti_proposals_2d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' kitti_proposals_3d_dir = predictions_root_dir + \ '/kitti_proposals_3d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' if save_2d and not os.path.exists(kitti_predictions_2d_dir): os.makedirs(kitti_predictions_2d_dir) if save_3d and not os.path.exists(kitti_proposals_3d_dir): os.makedirs(kitti_proposals_3d_dir) # Do conversion num_samples = dataset.num_samples num_valid_samples = 0 print('\nGlobal step:', global_step) print('Converting proposals from:', final_predictions_dir) if save_2d: print('2D Detections saved to:', kitti_predictions_2d_dir) if save_3d: print('Proposals saved to:', kitti_proposals_3d_dir) for sample_idx in range(num_samples): # Print progress sys.stdout.write('\rConverting {} / {}'.format( sample_idx + 1, num_samples)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] prediction_file = sample_name + '.txt' kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \ '/' + prediction_file kitti_predictions_3d_file_path = kitti_proposals_3d_dir + \ '/' + prediction_file predictions_file_path = final_predictions_dir + \ '/' + prediction_file # If no predictions, skip to next file if not os.path.exists(predictions_file_path): if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue all_predictions = np.loadtxt(predictions_file_path) # Swap l, w for predictions where w > l swapped_indices = all_predictions[:, 4] > all_predictions[:, 3] fixed_predictions = np.copy(all_predictions) fixed_predictions[swapped_indices, 3] = all_predictions[swapped_indices, 4] fixed_predictions[swapped_indices, 4] = all_predictions[swapped_indices, 3] fixed_predictions[swapped_indices, 6] = np.pi / 2 score_filter = all_predictions[:, 7] >= score_threshold all_predictions = fixed_predictions[score_filter] # If no predictions, skip to next file if len(all_predictions) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue # Project to image space sample_name = prediction_file.split('.')[0] img_idx = int(sample_name) # Load image for truncation image = Image.open(dataset.get_rgb_image_path(sample_name)) stereo_calib_p2 = calib_utils.read_calibration( dataset.calib_dir, img_idx).p2 boxes = [] image_filter = [] for i in range(len(all_predictions)): box_3d = all_predictions[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, stereo_calib_p2, truncate=True, image_size=image.size, discard_before_truncation=False) # Skip invalid boxes (outside image space) if img_box is None: image_filter.append(False) else: image_filter.append(True) boxes.append(img_box) boxes = np.asarray(boxes) all_predictions = all_predictions[image_filter] # If no predictions, skip to next file if len(boxes) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue num_valid_samples += 1 # To keep each value in its appropriate position, an array of zeros # (N, 16) is allocated but only values [4:16] are used kitti_predictions = np.zeros([len(boxes), 16]) # Truncation and Occlusion are always empty (see below) # Alpha (Not computed) kitti_predictions[:, 3] = -10 * np.ones( (len(kitti_predictions)), dtype=np.int32) # 2D predictions kitti_predictions[:, 4:8] = boxes[:, 0:4] # 3D predictions # (l, w, h) kitti_predictions[:, 8] = all_predictions[:, 5] kitti_predictions[:, 9] = all_predictions[:, 4] kitti_predictions[:, 10] = all_predictions[:, 3] # (x, y, z) kitti_predictions[:, 11:14] = all_predictions[:, 0:3] # (ry, score) kitti_predictions[:, 14:16] = all_predictions[:, 6:8] # Round detections to 3 decimal places kitti_predictions = np.round(kitti_predictions, 3) # Empty Truncation, Occlusion kitti_empty_1 = -1 * np.ones( (len(kitti_predictions), 2), dtype=np.int32) # Empty 3D (x, y, z) kitti_empty_2 = -1 * np.ones( (len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (h, w, l) kitti_empty_3 = -1000 * np.ones( (len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (ry) kitti_empty_4 = -10 * np.ones( (len(kitti_predictions), 1), dtype=np.int32) # Create Type Array obj_types = [obj_type for i in range(len(kitti_predictions))] # Stack 2D predictions text kitti_text_2d = np.column_stack([ obj_types, kitti_empty_1, kitti_predictions[:, 3:8], kitti_empty_2, kitti_empty_3, kitti_empty_4, kitti_predictions[:, 15] ]) # Stack 3D predictions text kitti_text_3d = np.column_stack( [obj_types, kitti_empty_1, kitti_predictions[:, 3:16]]) # Save to text files if save_2d: np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d, newline='\r\n', fmt='%s') if save_3d: np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d, newline='\r\n', fmt='%s') print('\nNum valid:', num_valid_samples) print('Num samples:', num_samples)
def main(): """This demo visualizes box 8C format predicted by MLOD, before getting converted to Box 3D. Keys: F1: Toggle predictions F2: Toggle easy ground truth objects (Green) F3: Toggle medium ground truth objects (Orange) F4: Toggle hard ground truth objects (Red) F5: Toggle all ground truth objects (default off) F6: Toggle 3D voxel grid F7: Toggle point cloud """ ############################## # Options ############################## mlod_score_threshold = 0.1 show_orientations = True checkpoint_name = 'mlod_exp_8c' global_step = None sample_name = None dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF) dataset = DatasetBuilder.build_kitti_dataset(dataset_config) ############################## # Setup Paths ############################## # # # Cars # # # # sample_name = '000050' # sample_name = '000104' # sample_name = '000169' # sample_name = '000191' # sample_name = '000360' # sample_name = '001783' # sample_name = '001820' # sample_name = '006338' # # # People # # # # val_half split # sample_name = '000001' sample_name = '000005' # Easy, 1 ped # sample_name = '000122' # Easy, 1 cyc # sample_name = '000134' # Hard, lots of people # sample_name = '000167' # Medium, 1 ped, 2 cycs # sample_name = '000187' # Medium, 1 ped on left # sample_name = '000381' # Easy, 1 ped # sample_name = '000398' # Easy, 1 ped # sample_name = '000401' # Hard, obscured peds # sample_name = '000407' # Easy, 1 ped # sample_name = '000448' # Hard, several far people # sample_name = '000486' # Hard 2 obscured peds # sample_name = '000509' # Easy, 1 ped # sample_name = '000718' # Hard, lots of people # sample_name = '002216' # Easy, 1 cyc # Random sample if sample_name is None: sample_idx = np.random.randint(0, dataset.num_samples) sample_name = dataset.sample_list[sample_idx] img_idx = int(sample_name) # Text files directory predictions_and_scores_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' + \ '/final_boxes_8c_and_scores/' + dataset.data_split # Get checkpoint step steps = os.listdir(predictions_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] ############################## # predictions ############################## # Load predictions from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) predictions_boxes_8c = predictions_and_scores[:, 0:24] prediction_scores = predictions_and_scores[:, 24] score_mask = prediction_scores >= mlod_score_threshold predictions_boxes_8c = predictions_boxes_8c[score_mask] all_vtk_box_corners = [] predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8]) for i in range(len(predictions_boxes_8c)): box_8c = predictions_boxes_8c[i, :, :] vtk_box_corners = VtkBox8c() vtk_box_corners.set_objects(box_8c) all_vtk_box_corners.append(vtk_box_corners) ############################## # Ground Truth ############################## if dataset.has_labels: easy_gt_objs, medium_gt_objs, \ hard_gt_objs, all_gt_objs = \ demo_utils.get_gts_based_on_difficulty(dataset, img_idx) else: easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = [] ############################## # Point Cloud ############################## image_path = dataset.get_rgb_image_path(sample_name) image = cv2.imread(image_path) img_idx = int(sample_name) points, point_colours = demo_utils.get_filtered_pc_and_colours( dataset, image, img_idx) # Voxelize the point cloud for visualization voxel_grid = VoxelGrid() voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False) ############################## # Visualization ############################## # Create VtkVoxelGrid vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(voxel_grid) vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points, point_colours) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for ground truth vtk_easy_gt_boxes, vtk_medium_gt_boxes, \ vtk_hard_gt_boxes, vtk_all_gt_boxes = \ demo_utils.create_gt_vtk_boxes(easy_gt_objs, medium_gt_objs, hard_gt_objs, all_gt_objs, show_orientations) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_box_actors = vtk.vtkAssembly() # Create VtkBoxes for prediction boxes for i in range(len(all_vtk_box_corners)): # Adding labels, slows down rendering # vtk_renderer.AddActor(all_vtk_box_corners[i]. # vtk_text_labels.vtk_actor) vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.SetBackground(0.2, 0.3, 0.4) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor) vtk_renderer.AddActor(vtk_box_actors) vtk_renderer.AddActor(axes) # Set initial properties for some actors vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2) vtk_voxel_grid.vtk_actor.SetVisibility(0) vtk_all_gt_boxes.vtk_actor.SetVisibility(0) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(160.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName( "Predictions: Step {}, Sample {}, Min Score {}".format( global_step, sample_name, mlod_score_threshold, )) vtk_render_window.SetSize(900, 600) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_box_actors, vtk_easy_gt_boxes.vtk_actor, vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor, vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor, vtk_point_cloud.vtk_actor ])) vtk_render_window_interactor.Start()
def main(): """ Displays the bird's eye view maps for a KITTI sample. """ ############################## # Options ############################## # one of ['height_priors', 'slices'] bev_generator = 'slices' height_priors_config = \ """ height_priors { ground_filter_offset: 0.2 offset_filter_distance: 2.0 std_dev_multiplier: 2.0 } """ slices_config = \ """ slices { height_lo: -0.2 height_hi: 2.3 num_slices: 5 } """ # Use None for a random image img_idx = None # img_idx = 142 # img_idx = 191 show_ground_truth = True # Whether to overlay ground_truth boxes point_cloud_source = 'lidar' ############################## # End of Options ############################## dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL) dataset_config = DatasetBuilder.merge_defaults(dataset_config) # Overwrite bev_generator if bev_generator == 'height_priors': text_format.Merge(height_priors_config, dataset_config.kitti_utils_config.bev_generator) elif bev_generator == 'slices': text_format.Merge(slices_config, dataset_config.kitti_utils_config.bev_generator) else: raise ValueError('Invalid bev_generator') dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) if img_idx is None: img_idx = int(random.random() * dataset.num_samples) sample_name = "{:06}".format(img_idx) print('=== Showing BEV maps for image: {}.png ==='.format(sample_name)) # Load image image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = image.shape[0:2] kitti_utils = dataset.kitti_utils point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx, image_shape) ground_plane = kitti_utils.get_ground_plane(sample_name) bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane) height_maps = np.array(bev_images.get("height_maps")) density_map = np.array(bev_images.get("density_map")) box_points, box_points_norm = [None, None] if show_ground_truth: # Get projected boxes obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx) filtered_objs = obj_labels label_boxes = [] for label in filtered_objs: box = box_3d_encoder.object_label_to_box_3d(label) label_boxes.append(box) label_boxes = np.array(label_boxes) box_points, box_points_norm = box_3d_projector.project_to_bev( label_boxes, [[-40, 40], [0, 70]]) rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16) img_x_start = 60 img_y_start = 330 img_x = img_x_start img_y = img_y_start img_w = 400 img_h = 350 img_titlebar_h = 20 # Show images vis_utils.cv2_show_image("Image", image, size_wh=rgb_img_size, location_xy=(img_x, 0)) # Height maps for map_idx in range(len(height_maps)): height_map = height_maps[map_idx] height_map = draw_boxes(height_map, box_points_norm) vis_utils.cv2_show_image("Height Map {}".format(map_idx), height_map, size_wh=(img_w, img_h), location_xy=(img_x, img_y)) img_x += img_w # Wrap around if (img_x + img_w) > 1920: img_x = img_x_start img_y += img_h + img_titlebar_h # Density map density_map = draw_boxes(density_map, box_points_norm) vis_utils.cv2_show_image("Density Map", density_map, size_wh=(img_w, img_h), location_xy=(img_x, img_y)) cv2.waitKey()
def generate_fake_dataset(): return DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_UNITTEST)
def main(): """Plots detection errors for xyz, lwh, ry, and shows 3D IoU with ground truth boxes """ dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL_HALF, use_defaults=True) difficulty = 2 # Full path to kitti predictions # (e.g. '.../data/outputs/mlod_exp_example/predictions/' # 'kitti_predictions_3d/val/0.1/100000/data' predictions_data_path = 'path_to_detections/data' # Loop through split and save ious and errors all_3d_ious = [] all_errors = [] for sample_idx in range(dataset.num_samples): sys.stdout.write('\r{} / {}'.format(sample_idx + 1, dataset.num_samples)) sample_name = dataset.sample_names[sample_idx] img_idx = int(sample_name) # Get filtered ground truth all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx) all_gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs, difficulty=difficulty) pred_objs = obj_utils.read_labels(predictions_data_path, img_idx) ############################## # Check IoUs ############################## if len(all_gt_objs) > 0 and \ pred_objs is not None and len(pred_objs) > 0: all_gt_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(gt_obj) for gt_obj in all_gt_objs ] pred_boxes_3d = [ box_3d_encoder.object_label_to_box_3d(pred_obj) for pred_obj in pred_objs ] # Convert to iou format gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( all_gt_boxes_3d) pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format( pred_boxes_3d) max_ious_3d = np.zeros(len(all_gt_objs)) max_iou_pred_indices = -np.ones(len(all_gt_objs)) for gt_obj_idx in range(len(all_gt_objs)): gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx] ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt, pred_objs_iou_fmt) max_iou_3d = np.amax(ious_3d) max_ious_3d[gt_obj_idx] = max_iou_3d if max_iou_3d > 0.0: max_iou_pred_indices[gt_obj_idx] = np.argmax(ious_3d) for gt_obj_idx in range(len(all_gt_objs)): max_iou_pred_idx = int(max_iou_pred_indices[gt_obj_idx]) if max_iou_pred_idx >= 0: error = all_gt_boxes_3d[gt_obj_idx] - \ pred_boxes_3d[max_iou_pred_idx] all_errors.append(error) all_3d_ious.extend(max_ious_3d) print('Done') all_errors = np.asarray(all_errors) # Plot Data Histograms f, ax_arr = plt.subplots(3, 3) xyzlwh_bins = 51 ry_bins = 31 iou_bins = 51 # xyz ax_arr[0, 0].hist(all_errors[:, 0], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[0, 1].hist(all_errors[:, 1], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[0, 2].hist(all_errors[:, 2], xyzlwh_bins, facecolor='green', alpha=0.75) # lwh ax_arr[1, 0].hist(all_errors[:, 3], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[1, 1].hist(all_errors[:, 4], xyzlwh_bins, facecolor='green', alpha=0.75) ax_arr[1, 2].hist(all_errors[:, 5], xyzlwh_bins, facecolor='green', alpha=0.75) # orientation ax_arr[2, 0].hist(all_errors[:, 6], ry_bins, facecolor='green', alpha=0.75) # iou ax_arr[2, 2].hist(all_3d_ious, iou_bins, facecolor='green', alpha=0.75) plt.show()
def main(): """This demo shows RPN proposals and MLOD predictions in 3D and 2D in image space. Given certain thresholds for proposals and predictions, it selects and draws the bounding boxes on the image sample. It goes through the entire proposal and prediction samples for the given dataset split. The proposals, overlaid, and prediction images can be toggled on or off separately in the options section. The prediction score and IoU with ground truth can be toggled on or off as well, shown as (score, IoU) above the detection. """ dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL) ############################## # Options ############################## dataset_config.data_split = 'val' fig_size = (10, 6.1) rpn_score_threshold = 0.1 mlod_score_threshold = 0.1 # Flag for projecting the 3D boxes to image space # in tensor format (for testing purposes) test_img_tensor_projection = False gt_classes = ['Pedestrian', 'Cyclist'] # gt_classes = ['Pedestrian', 'Cyclist'] # Overwrite this to select a specific checkpoint global_step = 44000 checkpoint_name = 'mlod_fpn_people' # Drawing Toggles draw_proposals_separate = False draw_overlaid = False draw_predictions_separate = True # Show orientation for both GT and proposals/predictions draw_orientations_on_prop = False draw_orientations_on_pred = False # Draw 2D bounding boxes draw_projected_2d_boxes = False # Save images for samples with no detections save_empty_images = True draw_score = True draw_iou = False iou_3d = False ############################## # End of Options ############################## # Get the dataset dataset = DatasetBuilder.build_kitti_dataset(dataset_config) # Setup Paths predictions_dir = mlod.root_dir() + \ '/data/outputs/' + checkpoint_name + '/predictions' proposals_and_scores_dir = predictions_dir + \ '/proposals_and_scores/' + dataset.data_split predictions_and_scores_dir = predictions_dir + \ '/final_predictions_and_scores/' + dataset.data_split # Output images directories output_dir_base = predictions_dir + '/images_2d' # Get checkpoint step steps = os.listdir(proposals_and_scores_dir) steps.sort(key=int) print('Available steps: {}'.format(steps)) # Use latest checkpoint if no index provided if global_step is None: global_step = steps[-1] if draw_proposals_separate: prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format( dataset.data_split, global_step, rpn_score_threshold) if not os.path.exists(prop_out_dir): os.makedirs(prop_out_dir) print('Proposal images saved to:', prop_out_dir) if draw_overlaid: overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format( dataset.data_split, global_step, mlod_score_threshold) if not os.path.exists(overlaid_out_dir): os.makedirs(overlaid_out_dir) print('Overlaid images saved to:', overlaid_out_dir) if draw_predictions_separate: pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format( dataset.data_split, global_step, mlod_score_threshold) if not os.path.exists(pred_out_dir): os.makedirs(pred_out_dir) print('Prediction images saved to:', pred_out_dir) # Rolling average array of times for time estimation avg_time_arr_length = 10 last_times = np.repeat(time.time(), avg_time_arr_length) + \ np.arange(avg_time_arr_length) for sample_idx in range(dataset.num_samples): # Estimate time remaining with 5 slowest times start_time = time.time() last_times = np.roll(last_times, -1) last_times[-1] = start_time avg_time = np.mean(np.sort(np.diff(last_times))[-5:]) samples_remaining = dataset.num_samples - sample_idx est_time_left = avg_time * samples_remaining # Print progress and time remaining estimate sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, ' 'Time Remaining: {:.2f}s'. format( sample_idx + 1, dataset.num_samples, avg_time, est_time_left)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] img_idx = int(sample_name) ############################## # Proposals ############################## if draw_proposals_separate or draw_overlaid: # Load proposals from files proposals_file_path = proposals_and_scores_dir + \ "/{}/{}.txt".format(global_step, sample_name) if not os.path.exists(proposals_file_path): print('Sample {}: No proposals, skipping'.format(sample_name)) continue print('Sample {}: Drawing proposals'.format(sample_name)) proposals_and_scores = np.loadtxt(proposals_file_path) proposal_boxes_3d = proposals_and_scores[:, 0:7] proposal_scores = proposals_and_scores[:, 7] # Apply score mask to proposals score_mask = proposal_scores > rpn_score_threshold proposal_boxes_3d = proposal_boxes_3d[score_mask] proposal_scores = proposal_scores[score_mask] proposal_objs = \ [box_3d_encoder.box_3d_to_object_label(proposal, obj_type='Proposal') for proposal in proposal_boxes_3d] ############################## # Predictions ############################## if draw_predictions_separate or draw_overlaid: predictions_file_path = predictions_and_scores_dir + \ "/{}/{}.txt".format(global_step, sample_name) if not os.path.exists(predictions_file_path): continue # Load predictions from files predictions_and_scores = np.loadtxt( predictions_and_scores_dir + "/{}/{}.txt".format(global_step, sample_name)) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_class_indices = predictions_and_scores[:, 8] # process predictions only if we have any predictions left after # masking if len(prediction_boxes_3d) > 0: # Apply score mask mlod_score_mask = prediction_scores >= 0.1 mlod_show_mask = mlod_score_mask prediction_boxes_3d = prediction_boxes_3d[mlod_show_mask] prediction_scores = prediction_scores[mlod_show_mask] prediction_class_indices = \ prediction_class_indices[mlod_show_mask] # # Swap l, w for predictions where w > l # swapped_indices = \ # prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3] # prediction_boxes_3d = np.copy(prediction_boxes_3d) # prediction_boxes_3d[swapped_indices, 3] = \ # prediction_boxes_3d[swapped_indices, 4] # prediction_boxes_3d[swapped_indices, 4] = \ # prediction_boxes_3d[swapped_indices, 3] ############################## # Ground Truth ############################## # Get ground truth labels dataset.has_labels = False if dataset.has_labels: gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) else: gt_objects = [] # Filter objects to desired difficulty filtered_gt_objs = dataset.kitti_utils.filter_labels( gt_objects, classes=gt_classes) boxes2d, _, _ = obj_utils.build_bbs_from_objects( filtered_gt_objs, class_needed=gt_classes) image_path = dataset.get_rgb_image_path(sample_name) image = Image.open(image_path) image_size = image.size # Read the stereo calibration matrix for visualization stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx) calib_p2 = stereo_calib.p2 ############################## # Reformat and prepare to draw ############################## if draw_proposals_separate or draw_overlaid: proposals_as_anchors = box_3d_encoder.box_3d_to_anchor( proposal_boxes_3d) if test_img_tensor_projection: proposal_boxes = demo_utils.tf_project_to_image_space( proposals_as_anchors, calib_p2, image_size, img_idx) else: proposal_boxes, _ = anchor_projector.project_to_image_space( proposals_as_anchors, calib_p2, image_size) num_of_proposals = proposal_boxes_3d.shape[0] prop_fig, prop_2d_axes, prop_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False) draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals, proposal_objs, proposal_boxes, prop_2d_axes, prop_3d_axes, draw_orientations_on_prop) if draw_proposals_separate: # Save just the proposals filename = prop_out_dir + '/' + sample_name + '.png' plt.savefig(filename) if not draw_overlaid: plt.close(prop_fig) if draw_overlaid or draw_predictions_separate: if len(prediction_boxes_3d) > 0: # Project the 3D box predictions to image space image_filter = [] final_boxes_2d = [] for i in range(len(prediction_boxes_3d)): box_3d = prediction_boxes_3d[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, calib_p2, truncate=True, image_size=image_size, discard_before_truncation=False) if img_box is not None: image_filter.append(True) final_boxes_2d.append(img_box) else: image_filter.append(False) final_boxes_2d = np.asarray(final_boxes_2d) final_prediction_boxes_3d = prediction_boxes_3d[image_filter] final_scores = prediction_scores[image_filter] final_class_indices = prediction_class_indices[image_filter] num_of_predictions = final_boxes_2d.shape[0] # Convert to objs final_prediction_objs = \ [box_3d_encoder.box_3d_to_object_label( prediction, obj_type='Prediction') for prediction in final_prediction_boxes_3d] for (obj, score) in zip(final_prediction_objs, final_scores): obj.score = score else: if save_empty_images: pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) filename = pred_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(pred_fig) continue if draw_overlaid: # Overlay prediction boxes on image draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, prop_2d_axes, prop_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred, iou_3d) filename = overlaid_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(prop_fig) if draw_predictions_separate: # Now only draw prediction boxes on images # on a new figure handler if draw_projected_2d_boxes: pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_2d_axes, pred_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred, iou_3d) else: pred_fig, pred_3d_axes = \ vis_utils.visualize_single_plot( dataset.rgb_image_dir, img_idx, display=False) draw_3d_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_3d_axes, draw_score, draw_iou, gt_classes, draw_orientations_on_pred) filename = pred_out_dir + '/' + sample_name + '.png' plt.savefig(filename) plt.close(pred_fig) print('\nDone')
def main(): """ Visualization for comparison of anchor filtering with 2D vs 3D integral images Keys: F1: Toggle 3D integral image filtered anchors F2: Toggle 2D integral image filtered anchors F3: Toggle 2D integral image empty anchors """ anchor_2d_colour_scheme = {"Anchor": (0, 0, 255)} # Blue anchor_3d_colour_scheme = {"Anchor": (0, 255, 0)} # Green anchor_unfiltered_colour_scheme = {"Anchor": (255, 0, 255)} # Purple # Create Dataset dataset = DatasetBuilder.build_kitti_dataset( DatasetBuilder.KITTI_TRAINVAL) sample_name = "000001" img_idx = int(sample_name) print("Showing anchors for sample {}".format(sample_name)) # Options # These clusters are from the trainval set and give more 2D anchors than 3D clusters = np.array([[3.55, 1.835, 1.525], [4.173, 1.69, 1.49]]) anchor_stride = [3.0, 3.0] ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) area_extents = np.array([[-40, 40], [-5, 3], [0, 70]]) anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator() # Generate anchors start_time = time.time() anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents, anchor_3d_sizes=clusters, anchor_stride=anchor_stride, ground_plane=ground_plane) end_time = time.time() print("Anchors generated in {} s".format(end_time - start_time)) # Get point cloud point_cloud = obj_utils.get_stereo_point_cloud(img_idx, dataset.calib_dir, dataset.disp_dir) ground_offset_dist = 0.2 offset_dist = 2.0 # Filter points within certain xyz range and offset from ground plane # Filter points within 0.2m of the road plane slice_filter = dataset.kitti_utils.create_slice_filter(point_cloud, area_extents, ground_plane, ground_offset_dist, offset_dist) points = np.array(point_cloud).T points = points[slice_filter] anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d) # Create 2D voxel grid vx_grid_2d = voxel_grid_2d.VoxelGrid2D() vx_grid_2d.voxelize_2d(points, 0.1, area_extents) # Create 3D voxel grid vx_grid_3d = voxel_grid.VoxelGrid() vx_grid_3d.voxelize(points, 0.1, area_extents) # Filter the boxes here! start_time = time.time() empty_filter_2d = anchor_filter.get_empty_anchor_filter_2d( anchors=anchors, voxel_grid_2d=vx_grid_2d, density_threshold=1) anchors_2d = anchor_boxes_3d[empty_filter_2d] end_time = time.time() print("2D Anchors filtered in {} s".format(end_time - start_time)) print("Number of 2D anchors remaining: %d" % (anchors_2d.shape[0])) unfiltered_anchors_2d = anchor_boxes_3d[np.logical_not(empty_filter_2d)] # 3D filtering start_time = time.time() empty_filter_3d = anchor_filter.get_empty_anchor_filter( anchors=anchors, voxel_grid_3d=vx_grid_3d, density_threshold=1) anchor_boxes_3d = anchor_boxes_3d[empty_filter_3d] end_time = time.time() print("3D Anchors filtered in {} s".format(end_time - start_time)) print("Number of 3D anchors remaining: %d" % (anchor_boxes_3d.shape[0])) anchor_2d_objects = [] for anchor_idx in range(len(anchors_2d)): anchor = anchors_2d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor') # Append to a list for visualization in VTK later anchor_2d_objects.append(obj_label) anchor_3d_objects = [] for anchor_idx in range(len(anchor_boxes_3d)): anchor = anchor_boxes_3d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor') # Append to a list for visualization in VTK later anchor_3d_objects.append(obj_label) unfiltered_anchor_objects = [] for anchor_idx in range(len(unfiltered_anchors_2d)): anchor = unfiltered_anchors_2d[anchor_idx] obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor') # Append to a list for visualization in VTK later unfiltered_anchor_objects.append(obj_label) # Create VtkAxes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Create VtkBoxes for boxes vtk_2d_boxes = VtkBoxes() vtk_2d_boxes.set_objects(anchor_2d_objects, anchor_2d_colour_scheme) vtk_3d_boxes = VtkBoxes() vtk_3d_boxes.set_objects(anchor_3d_objects, anchor_3d_colour_scheme) vtk_unfiltered_boxes = VtkBoxes() vtk_unfiltered_boxes.set_objects(unfiltered_anchor_objects, anchor_unfiltered_colour_scheme) vtk_voxel_grid = VtkVoxelGrid() vtk_voxel_grid.set_voxels(vx_grid_3d) vtk_voxel_grid_2d = VtkVoxelGrid() vtk_voxel_grid_2d.set_voxels(vx_grid_2d) # Create Voxel Grid Renderer in bottom half vtk_renderer = vtk.vtkRenderer() vtk_renderer.AddActor(vtk_2d_boxes.vtk_actor) vtk_renderer.AddActor(vtk_3d_boxes.vtk_actor) vtk_renderer.AddActor(vtk_unfiltered_boxes.vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor) vtk_renderer.AddActor(vtk_voxel_grid_2d.vtk_actor) vtk_renderer.AddActor(axes) vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(170.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("Anchors") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_2d_boxes.vtk_actor, vtk_3d_boxes.vtk_actor, vtk_unfiltered_boxes.vtk_actor, ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def setUpClass(cls): dataset_config = DatasetBuilder.copy_config( DatasetBuilder.KITTI_UNITTEST) cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config) cls.label_dir = cls.dataset.label_dir
def main(): """Shows a flipped sample in 3D """ # Create Dataset dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL) ############################## # Options ############################## # sample_name = "000191" sample_name = "000104" img_idx = int(sample_name) print("Showing anchors for sample {}".format(sample_name)) ############################## # Load Sample Data ############################## ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir) image = cv2.imread(dataset.get_rgb_image_path(sample_name)) image_shape = [image.shape[1], image.shape[0]] # Get point cloud point_cloud = obj_utils.get_depth_map_point_cloud(img_idx, dataset.calib_dir, dataset.depth_dir, image_shape) points = np.array(point_cloud).T # Ground truth gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx) # Filter ground truth gt_labels = dataset.kitti_utils.filter_labels(gt_labels) ############################## # Flip stuff ############################## image_flipped = np.fliplr(image) # Flip ground plane coeff (x) ground_plane_flipped = np.copy(ground_plane) ground_plane_flipped[0] = -ground_plane_flipped[0] # Flip 3D points points_flipped = kitti_aug.flip_points(points) # Get point cloud colours point_colours_flipped = project_flipped_img_to_point_cloud( points_flipped, image_flipped, dataset.calib_dir, img_idx) # Flip ground truth boxes gt_labels_flipped = [ kitti_aug.flip_label_in_3d_only(obj) for obj in gt_labels ] ############################## # VTK Visualization ############################## # Axes axes = vtk.vtkAxesActor() axes.SetTotalLength(5, 5, 5) # Point cloud vtk_point_cloud = VtkPointCloud() vtk_point_cloud.set_points(points_flipped, point_colours=point_colours_flipped) # # Ground Truth Boxes vtk_boxes = VtkBoxes() vtk_boxes.set_objects(gt_labels_flipped, VtkBoxes.COLOUR_SCHEME_KITTI, show_orientations=True) # Renderer vtk_renderer = vtk.vtkRenderer() vtk_renderer.SetBackground(0.2, 0.3, 0.4) # Add Actors to Rendered vtk_renderer.AddActor(axes) vtk_renderer.AddActor(vtk_point_cloud.vtk_actor) vtk_renderer.AddActor(vtk_boxes.vtk_actor) # Setup Camera current_cam = vtk_renderer.GetActiveCamera() current_cam.Pitch(170.0) current_cam.Roll(180.0) # Zooms out to fit all points on screen vtk_renderer.ResetCamera() # Zoom in slightly current_cam.Zoom(2.5) # Reset the clipping range to show all points vtk_renderer.ResetCameraClippingRange() # Setup Render Window vtk_render_window = vtk.vtkRenderWindow() vtk_render_window.SetWindowName("Anchors") vtk_render_window.SetSize(900, 500) vtk_render_window.AddRenderer(vtk_renderer) # Setup custom interactor style, which handles mouse and key events vtk_render_window_interactor = vtk.vtkRenderWindowInteractor() vtk_render_window_interactor.SetRenderWindow(vtk_render_window) vtk_render_window_interactor.SetInteractorStyle( vis_utils.ToggleActorsInteractorStyle([ vtk_point_cloud.vtk_actor, ])) # Render in VTK vtk_render_window.Render() vtk_render_window_interactor.Start()
def main(): """ Calculates clusters for each class Returns: all_clusters: list of clusters for each class all_std_devs: list of cluster standard deviations for each class """ dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN) # Calculate the remaining clusters # Load labels corresponding to the sample list for clustering sample_list = dataset.load_sample_names(dataset.cluster_split) all_dims = [] num_samples = len(sample_list) for sample_idx in range(num_samples): sys.stdout.write("\rClustering labels {} / {}".format( sample_idx + 1, num_samples)) sys.stdout.flush() sample_name = sample_list[sample_idx] img_idx = int(sample_name) obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx) filtered_lwh = LabelClusterUtils._filter_labels_by_class( obj_labels, dataset.classes) if filtered_lwh[0]: all_dims.extend(filtered_lwh[0]) all_dims = np.array(all_dims) print("\nFinished reading labels, clustering data...\n") # Print 3 decimal places np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)}) # Calculate average cluster k_means = KMeans(n_clusters=1, random_state=0).fit(all_dims) cluster_centre = k_means.cluster_centers_[0] # Calculate std. dev std_dev = np.std(all_dims, axis=0) # Calculate 2 and 3 standard deviations below the mean two_sigma_length_lo = cluster_centre[0] - 2 * std_dev[0] three_sigma_length_lo = cluster_centre[0] - 3 * std_dev[0] # Remove all labels with length above two std dev # from the mean and re-cluster small_mask_2 = all_dims[:, 0] < two_sigma_length_lo small_dims_2 = all_dims[small_mask_2] small_mask_3 = all_dims[:, 0] < three_sigma_length_lo small_dims_3 = all_dims[small_mask_3] small_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(small_dims_2) small_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(small_dims_3) small_std_dev_2 = np.std(small_dims_2, axis=0) small_std_dev_3 = np.std(small_dims_3, axis=0) print('small_k_means_2:', small_k_means_2.cluster_centers_) print('small_k_means_3:', small_k_means_3.cluster_centers_) print('small_std_dev_2:', small_std_dev_2) print('small_std_dev_3:', small_std_dev_3) # Calculate 2 and 3 standard deviations above the mean two_sigma_length_hi = cluster_centre[0] + 2 * std_dev[0] three_sigma_length_hi = cluster_centre[0] + 3 * std_dev[0] # Remove all labels with length above two std dev # from the mean and re-cluster large_mask_2 = all_dims[:, 0] > two_sigma_length_hi large_dims_2 = all_dims[large_mask_2] large_mask_3 = all_dims[:, 0] > three_sigma_length_hi large_dims_3 = all_dims[large_mask_3] large_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(large_dims_2) large_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(large_dims_3) large_std_dev_2 = np.std(large_dims_2, axis=0) large_std_dev_3 = np.std(large_dims_3, axis=0) print('large_k_means_2:', large_k_means_2.cluster_centers_) print('large_k_means_3:', large_k_means_3.cluster_centers_) print('large_std_dev_2:', large_std_dev_2) print('large_std_dev_3:', large_std_dev_3)
dataset_config.data_split = 'trainval' dataset_config.data_split_dir = 'training' dataset_config.has_labels = True # Set CUDA device id os.environ['CUDA_VISIBLE_DEVICES'] = args.device # Convert to object to overwrite repeated fields dataset_config = config_builder.proto_to_obj(dataset_config) # Remove augmentation during evaluation dataset_config.aug_list = ['flipping'] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Convert to object to overwrite repeated fields model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] index = 76 samples = dataset.load_samples([index]) depth_map= samples[0].get(constants.KEY_DPT_INPUT) print(depth_map.shape) image_input = samples[0].get(constants.KEY_IMAGE_INPUT)