def __init__(self, experiment_config_path, planes_dir, calib_dir): model_config, _, eval_config, dataset_config = \ config_builder.get_configs_from_pipeline_file( experiment_config_path, is_training=False) self.dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_split_dir = 'testing' # These two lines below are necessary for KittiUtils to function properly self.cluster_split = self.dataset_config.cluster_split self.config = dataset_config self.data_split = self.config.data_split self.name = self.config.name self.dataset_dir = os.path.expanduser(self.config.dataset_dir) self.has_labels = self.config.has_labels self.cluster_split = self.config.cluster_split self.classes = list(self.config.classes) self.num_classes = len(self.classes) self.num_clusters = np.asarray(self.config.num_clusters) self.model_config = config_builder.proto_to_obj(model_config) self.paths_config = self.model_config.paths_config self.checkpoint_dir = self.paths_config.checkpoint_dir # Build the dataset object self.dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) self.dataset.train_val_test = "test" # Score threshold for drawing bounding boxes self.avod_score_threshold = 0.25
def inference(model_config, eval_config, dataset_config, data_split, ckpt_indices): # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' eval_config.eval_mode = 'test' eval_config.evaluate_repeatedly = False dataset_config.has_labels = False # Enable this to see the actually memory being used eval_config.allow_gpu_mem_growth = True eval_config = config_builder.proto_to_obj(eval_config) # Grab the checkpoint indices to evaluate eval_config.ckpt_indices = ckpt_indices # Remove augmentation during evaluation in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Overwrite repeated field model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] with tf.Graph().as_default(): if model_name == 'avod_model': model = AvodModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) elif model_name == 'bev_only_rpn_model': model = BevOnlyRpnModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) elif model_name == 'bev_only_avod_model': model = BevOnlyAvodModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) else: raise ValueError('Invalid model name {}'.format(model_name)) model_evaluator = Evaluator(model, dataset_config, eval_config) model_evaluator.run_latest_checkpoints()
def set_up_model(pipeline_config, data_split): model_config, train_config, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( pipeline_config, is_training=False) dataset_config = config_builder.proto_to_obj(dataset_config) train_val_test = data_split # Always run in test mode dataset_config.data_split = 'test' dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False dataset_config.aug_list = [] dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'avod_model': model = AvodModel(model_config, train_val_test=train_val_test, dataset=dataset) elif model_name == 'avod_ssd_model': model = AvodSSDModel(model_config, train_val_test=train_val_test, dataset=dataset) else: raise ValueError('Invalid model_name') return model
def filter_gt_labels(dataset_config, base_dir): """Plots detection errors for xyz, lwh, ry, and shows 3D IoU with ground truth boxes """ # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_dir = base_dir dataset_config.dataset_dir = base_dir dataset_config.data_split = 'train' dataset_config.data_split_dir = 'training' # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) dataset.classes = ['Car', 'Bus', 'Pedestrian', 'Cyclist', 'Truck'] dataset.aug_sample_list = [] difficulty = 2 out_dir = base_dir + dataset_config.data_split_dir + '/label_filtered_2/' if not os.path.exists(out_dir): os.makedirs(out_dir) print("Saving filtered labels to: ", out_dir) # TODO read in augmented labels to filter by 3D point count # Also changes synthetic to true in read_labels # dataset.label_dir = base_dir + '/' + dataset_config.data_split_dir + '/label_aug_2/' dataset.label_dir = base_dir + '/' + dataset_config.data_split_dir + '/label_2/' files = os.listdir(dataset.label_dir) num_files = len(files) sample_idx = 0 for file in files: sys.stdout.flush() sys.stdout.write('\r{} / {}'.format(sample_idx + 1, num_files)) sample_idx = sample_idx + 1 filepath = dataset.label_dir + '/' + file idx = int(os.path.splitext(file)[0]) # Get filtered ground truth all_gt_objs = obj_utils.read_labels(dataset.label_dir, idx, synthetic=False) if all_gt_objs != None: all_gt_objs = dataset.kitti_utils.filter_labels( all_gt_objs, max_forward=dataset.kitti_utils.area_extents[2, 1], max_side=dataset.kitti_utils.area_extents[0, 1]) # Save gt to output file save_filtered_objs(all_gt_objs, idx, out_dir)
def test_path_drop_weights(self): # Tests the effect of path-drop on network's feature maps. # It sets up a minimal-training process to check the # feature before and after running the 'train_op' while # path-drop is in effect. train_val_test = 'train' # overwrite the training iterations self.train_config.max_iterations = 2 self.train_config.overwrite_checkpoints = True # Overwrite path drop probabilities model_config = config_builder.proto_to_obj(self.model_config) model_config.path_drop_probabilities = [0.0, 0.8] with tf.Graph().as_default(): # Set a graph-level seed tf.set_random_seed(1245) model = RpnModel(model_config, train_val_test=train_val_test, dataset=self.dataset) prediction_dict = model.build() losses_dict, total_loss = model.loss(prediction_dict) global_summaries = set([]) # Optimizer training_optimizer = optimizer_builder.build( self.train_config.optimizer, global_summaries) train_op = slim.learning.create_train_op( total_loss, training_optimizer) init_op = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init_op) for step in range(1, self.train_config.max_iterations): feed_dict = model.create_feed_dict() if step == 1: current_feature_maps = sess.run(model.img_feature_maps, feed_dict=feed_dict) exp_feature_maps = current_feature_maps train_op_loss = sess.run(train_op, feed_dict=feed_dict) print('Step {}, Total Loss {:0.3f} '. format(step, train_op_loss)) updated_feature_maps = sess.run(model.img_feature_maps, feed_dict=feed_dict) # The feature maps should have remained the same since # the image path was dropped np.testing.assert_array_almost_equal( updated_feature_maps, exp_feature_maps, decimal=4)
def get_dataset(dataset_config, data_split): # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset_config.has_labels = data_split in ['train', 'val'] # Remove augmentation during evaluation in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) return dataset
def test_disable_path_drop(self): # Test path drop is disabled when the probabilities # are set to 1.0. train_val_test = 'train' # Overwrite path drop probabilities model_config = config_builder.proto_to_obj(self.model_config) model_config.path_drop_probabilities = [1.0, 1.0] with tf.Graph().as_default(): model = RpnModel(model_config, train_val_test=train_val_test, dataset=self.dataset) model.build() # These variables are set during path drop only # in the case of no path-drop, they should be non-existence self.assertFalse(hasattr(model, 'img_path_drop_mask')) self.assertFalse(hasattr(model, 'bev_path_drop_mask'))
def set_up_model_test_mode(pipeline_config_path, data_split): """Returns the model and its config in test mode.""" model_config, _, _, dataset_config = \ config_builder.get_configs_from_pipeline_file( pipeline_config_path, is_training=False) dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) # Use the validation set dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' # Remove augmentation when in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) model_name = model_config.model_name if model_name == 'rpn_model': model = RpnModel(model_config, train_val_test='test', dataset=dataset) elif model_name == 'avod_model': model = AvodModel(model_config, train_val_test='test', dataset=dataset) elif model_name == 'avod_ssd_model': model = AvodSSDModel(model_config, train_val_test='test', dataset=dataset) else: raise ValueError('Invalid model_name') return model, model_config
def evaluate(model_config, eval_config, dataset_config): # Parse eval config eval_mode = eval_config.eval_mode if eval_mode not in ['val', 'test']: raise ValueError('Evaluation mode can only be set to `val` or `test`') evaluate_repeatedly = eval_config.evaluate_repeatedly # Parse dataset config data_split = dataset_config.data_split if data_split == 'train': dataset_config.data_split_dir = 'training' dataset_config.has_labels = True elif data_split.startswith('val'): dataset_config.data_split_dir = 'training' # Don't load labels for val split when running in test mode if eval_mode == 'val': dataset_config.has_labels = True elif eval_mode == 'test': dataset_config.has_labels = False elif data_split == 'test': dataset_config.data_split_dir = 'testing' dataset_config.has_labels = False else: raise ValueError('Invalid data split', data_split) # Convert to object to overwrite repeated fields dataset_config = config_builder.proto_to_obj(dataset_config) # Remove augmentation during evaluation dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Convert to object to overwrite repeated fields model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] with tf.Graph().as_default(): if model_name == 'avod_model': model = AvodModel(model_config, train_val_test=eval_mode, dataset=dataset) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=eval_mode, dataset=dataset) else: raise ValueError('Invalid model name {}'.format(model_name)) model_evaluator = Evaluator(model, dataset_config, eval_config) if evaluate_repeatedly: model_evaluator.repeated_checkpoint_run() else: model_evaluator.run_latest_checkpoints()
def main(): """ Converts a set of network predictions into text files required for KITTI evaluation. """ ############################## # Options ############################## checkpoint_name = 'fpn_people_dual_NHSP_train'#'fpn_people_dual_SHPL_train'#'pyramid_people_with_NHSP_example_train'#'pyramid_people_with_NHSP_example' data_split = 'val'#'test'#'val' global_steps = None # global_steps = [28000, 19000, 33000, 34000] score_threshold = 0.1 save_2d = False # Save 2D predictions save_3d = True # Save 2D and 3D predictions together save_alphas = True # Save alphas (observation angles) # Checkpoints below this are skipped min_step = 20000 ############################## # End of Options ############################## # Parse experiment config pipeline_config_file = \ avod.root_dir() + '/data/outputs/' + checkpoint_name + \ '/' + checkpoint_name + '.config' _, _, _, dataset_config = \ config_builder_util.get_configs_from_pipeline_file( pipeline_config_file, is_training=False) # Overwrite defaults dataset_config = config_builder_util.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.aug_list = [] if data_split == 'test': dataset_config.data_split_dir = 'testing' dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Get available prediction folders predictions_root_dir = avod.root_dir() + '/data/outputs/' + \ checkpoint_name + '/predictions' final_predictions_root_dir = predictions_root_dir + \ '/final_predictions_and_scores/' + dataset.data_split print('Converting detections from', final_predictions_root_dir) if not global_steps: global_steps = os.listdir(final_predictions_root_dir) global_steps.sort(key=int) print('Checkpoints found ', global_steps) for step_idx in range(len(global_steps)): global_step = global_steps[step_idx] # Skip first checkpoint if int(global_step) < min_step: continue final_predictions_dir = final_predictions_root_dir + \ '/' + str(global_step) # 2D and 3D prediction directories kitti_predictions_2d_dir = predictions_root_dir + \ '/kitti_predictions_2d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' kitti_predictions_3d_dir = predictions_root_dir + \ '/kitti_predictions_3d/' + \ dataset.data_split + '/' + \ str(score_threshold) + '/' + \ str(global_step) + '/data' if save_2d and not os.path.exists(kitti_predictions_2d_dir): os.makedirs(kitti_predictions_2d_dir) if save_3d and not os.path.exists(kitti_predictions_3d_dir): os.makedirs(kitti_predictions_3d_dir) # Do conversion num_samples = dataset.num_samples num_valid_samples = 0 print('\nGlobal step:', global_step) print('Converting detections from:', final_predictions_dir) if save_2d: print('2D Detections saved to:', kitti_predictions_2d_dir) if save_3d: print('3D Detections saved to:', kitti_predictions_3d_dir) for sample_idx in range(num_samples): # Print progress sys.stdout.write('\rConverting {} / {}'.format( sample_idx + 1, num_samples)) sys.stdout.flush() sample_name = dataset.sample_names[sample_idx] prediction_file = sample_name + '.txt' kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \ '/' + prediction_file kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \ '/' + prediction_file predictions_file_path = final_predictions_dir + \ '/' + prediction_file # If no predictions, skip to next file if not os.path.exists(predictions_file_path): if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue all_predictions = np.loadtxt(predictions_file_path) # # Swap l, w for predictions where w > l # swapped_indices = all_predictions[:, 4] > all_predictions[:, 3] # fixed_predictions = np.copy(all_predictions) # fixed_predictions[swapped_indices, 3] = all_predictions[ # swapped_indices, 4] # fixed_predictions[swapped_indices, 4] = all_predictions[ # swapped_indices, 3] score_filter = all_predictions[:, 7] >= score_threshold all_predictions = all_predictions[score_filter] # If no predictions, skip to next file if len(all_predictions) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue # Project to image space sample_name = prediction_file.split('.')[0] img_idx = int(sample_name) # Load image for truncation image = Image.open(dataset.get_rgb_image_path(sample_name)) stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx).p2 boxes = [] image_filter = [] for i in range(len(all_predictions)): box_3d = all_predictions[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, stereo_calib_p2, truncate=True, image_size=image.size) # Skip invalid boxes (outside image space) if img_box is None: image_filter.append(False) else: image_filter.append(True) boxes.append(img_box) boxes = np.asarray(boxes) all_predictions = all_predictions[image_filter] # If no predictions, skip to next file if len(boxes) == 0: if save_2d: np.savetxt(kitti_predictions_2d_file_path, []) if save_3d: np.savetxt(kitti_predictions_3d_file_path, []) continue num_valid_samples += 1 # To keep each value in its appropriate position, an array of zeros # (N, 16) is allocated but only values [4:16] are used kitti_predictions = np.zeros([len(boxes), 16]) # Get object types all_pred_classes = all_predictions[:, 8].astype(np.int32) obj_types = [dataset.classes[class_idx] for class_idx in all_pred_classes] # Truncation and Occlusion are always empty (see below) # Alpha if not save_alphas: kitti_predictions[:, 3] = -10 * \ np.ones((len(kitti_predictions)), dtype=np.int32) else: alphas = all_predictions[:, 6] - \ np.arctan2(all_predictions[:, 0], all_predictions[:, 2]) kitti_predictions[:, 3] = alphas # 2D predictions kitti_predictions[:, 4:8] = boxes[:, 0:4] # 3D predictions # (l, w, h) kitti_predictions[:, 8] = all_predictions[:, 5] kitti_predictions[:, 9] = all_predictions[:, 4] kitti_predictions[:, 10] = all_predictions[:, 3] # (x, y, z) kitti_predictions[:, 11:14] = all_predictions[:, 0:3] # (ry, score) kitti_predictions[:, 14:16] = all_predictions[:, 6:8] # Round detections to 3 decimal places kitti_predictions = np.round(kitti_predictions, 3) # Empty Truncation, Occlusion kitti_empty_1 = -1 * np.ones((len(kitti_predictions), 2), dtype=np.int32) # Empty 3D (x, y, z) kitti_empty_2 = -1 * np.ones((len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (h, w, l) kitti_empty_3 = -1000 * np.ones((len(kitti_predictions), 3), dtype=np.int32) # Empty 3D (ry) kitti_empty_4 = -10 * np.ones((len(kitti_predictions), 1), dtype=np.int32) # Stack 2D predictions text kitti_text_2d = np.column_stack([obj_types, kitti_empty_1, kitti_predictions[:, 3:8], kitti_empty_2, kitti_empty_3, kitti_empty_4, kitti_predictions[:, 15]]) # Stack 3D predictions text kitti_text_3d = np.column_stack([obj_types, kitti_empty_1, kitti_predictions[:, 3:16]]) # Save to text files if save_2d: np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d, newline='\r\n', fmt='%s') if save_3d: np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d, newline='\r\n', fmt='%s') print('\nNum valid:', num_valid_samples) print('Num samples:', num_samples)
def test(model_config, eval_config, dataset_config, data_split, ckpt_indices): # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_split = data_split dataset_config.data_split_dir = 'training' if data_split == 'test': dataset_config.data_split_dir = 'testing' eval_config.eval_mode = 'test' eval_config.evaluate_repeatedly = False dataset_config.has_labels = False # Enable this to see the actually memory being used eval_config.allow_gpu_mem_growth = True eval_config = config_builder.proto_to_obj(eval_config) # Grab the checkpoint indices to evaluate eval_config.ckpt_indices = ckpt_indices # Remove augmentation during evaluation in test mode dataset_config.aug_list = [] # Build the dataset object dataset = DatasetBuilder.build_kitti_dataset(dataset_config, use_defaults=False) # Setup the model model_name = model_config.model_name # Overwrite repeated field model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] with tf.Graph().as_default(): if model_name == 'avod_model': model = AvodModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) elif model_name == 'rpn_model': model = RpnModel(model_config, train_val_test=eval_config.eval_mode, dataset=dataset) else: raise ValueError('Invalid model name {}'.format(model_name)) #model_evaluator = Evaluator(model, dataset_config, eval_config) #model_evaluator.run_latest_checkpoints() # Create a variable tensor to hold the global step global_step_tensor = tf.Variable(0, trainable=False, name='global_step') allow_gpu_mem_growth = eval_config.allow_gpu_mem_growth if allow_gpu_mem_growth: # GPU memory config config = tf.ConfigProto() config.gpu_options.allow_growth = allow_gpu_mem_growth _sess = tf.Session(config=config) else: _sess = tf.Session() _prediction_dict = model.build() _saver = tf.train.Saver() trainer_utils.load_checkpoints(model_config.paths_config.checkpoint_dir, _saver) num_checkpoints = len(_saver.last_checkpoints) print("test:",num_checkpoints) checkpoint_to_restore = _saver.last_checkpoints[num_checkpoints-1] _saver.restore(_sess, checkpoint_to_restore) num_samples = model.dataset.num_samples num_valid_samples = 0 current_epoch = model.dataset.epochs_completed while current_epoch == model.dataset.epochs_completed: # Keep track of feed_dict speed start_time = time.time() feed_dict = model.create_feed_dict() feed_dict_time = time.time() - start_time # Get sample name from model sample_name = model.sample_info['sample_name'] num_valid_samples += 1 print("Step: {} / {}, Inference on sample {}".format( num_valid_samples, num_samples, sample_name)) print("test mode") inference_start_time = time.time() # Don't calculate loss or run summaries for test predictions = _sess.run(_prediction_dict, feed_dict=feed_dict) inference_time = time.time() - inference_start_time print("inference time:", inference_time) predictions_and_scores = get_avod_predicted_boxes_3d_and_scores(predictions) #print(predictions_and_scores) #im_path = os.path.join(dataset_dir, 'training/image_2/{:06d}.png'.format(img_idx)) #im = cv2.imread(im_path) #cv2.imshow('result',im) #cv2.waitKey(30) prediction_boxes_3d = predictions_and_scores[:, 0:7] prediction_scores = predictions_and_scores[:, 7] prediction_class_indices = predictions_and_scores[:, 8] gt_classes = ['Car'] fig_size = (10, 6.1) avod_score_threshold = 0.1 if len(prediction_boxes_3d) > 0: # Apply score mask avod_score_mask = prediction_scores >= avod_score_threshold prediction_boxes_3d = prediction_boxes_3d[avod_score_mask] prediction_scores = prediction_scores[avod_score_mask] prediction_class_indices = \ prediction_class_indices[avod_score_mask] if len(prediction_boxes_3d) > 0: dataset_dir = model.dataset.dataset_dir sample_name = (model.dataset.sample_names[model.dataset._index_in_epoch - 1]) img_idx = int(sample_name) print("frame_index",img_idx) image_path = model.dataset.get_rgb_image_path(sample_name) image = Image.open(image_path) image_size = image.size if model.dataset.has_labels: gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx) else: gt_objects = [] filtered_gt_objs = model.dataset.kitti_utils.filter_labels( gt_objects, classes=gt_classes) stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx) calib_p2 = stereo_calib.p2 # Project the 3D box predictions to image space image_filter = [] final_boxes_2d = [] for i in range(len(prediction_boxes_3d)): box_3d = prediction_boxes_3d[i, 0:7] img_box = box_3d_projector.project_to_image_space( box_3d, calib_p2, truncate=True, image_size=image_size, discard_before_truncation=False) if img_box is not None: image_filter.append(True) final_boxes_2d.append(img_box) else: image_filter.append(False) final_boxes_2d = np.asarray(final_boxes_2d) final_prediction_boxes_3d = prediction_boxes_3d[image_filter] final_scores = prediction_scores[image_filter] final_class_indices = prediction_class_indices[image_filter] num_of_predictions = final_boxes_2d.shape[0] # Convert to objs final_prediction_objs = \ [box_3d_encoder.box_3d_to_object_label( prediction, obj_type='Prediction') for prediction in final_prediction_boxes_3d] for (obj, score) in zip(final_prediction_objs, final_scores): obj.score = score pred_fig, pred_2d_axes, pred_3d_axes = \ vis_utils.visualization(dataset.rgb_image_dir, img_idx, display=False, fig_size=fig_size) draw_predictions(filtered_gt_objs, calib_p2, num_of_predictions, final_prediction_objs, final_class_indices, final_boxes_2d, pred_2d_axes, pred_3d_axes, True, True, gt_classes, False) #cv2.imshow('result',pred_fig) print(type(pred_fig)) pred_fig.canvas.draw() img = np.fromstring(pred_fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') img = img.reshape(pred_fig.canvas.get_width_height()[::-1] + (3,)) cv2.imshow('result',img) #draw bird view kitti_utils = model.dataset.kitti_utils print(img.shape[0:2]) point_cloud = kitti_utils.get_point_cloud( 'lidar', img_idx, (370, 1242)) ground_plane = kitti_utils.get_ground_plane(sample_name) bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane) density_map = np.array(bev_images.get("density_map")) _, box_points_norm = box_3d_projector.project_to_bev( final_prediction_boxes_3d, [[-40, 40], [0, 70]]) density_map = draw_boxes(density_map, box_points_norm) cv2.imshow('lidar',density_map) cv2.waitKey(-1)
def inference(model_config, eval_config, dataset_config, base_dir, ckpt_indices, additional_cls, start_perspective=0): logging.info("Additional class: {}".format(additional_cls)) logging.info("ckpt_indices: {}".format(ckpt_indices)) # Overwrite the defaults dataset_config = config_builder.proto_to_obj(dataset_config) dataset_config.data_dir = base_dir dataset_config.dataset_dir = base_dir dataset_config.data_split = 'train' dataset_config.data_split_dir = 'training' eval_config.eval_mode = 'test' eval_config.evaluate_repeatedly = False dataset_config.has_labels = False # Enable this to see the actually memory being used eval_config.allow_gpu_mem_growth = True eval_config = config_builder.proto_to_obj(eval_config) # Grab the checkpoint indices to evaluate eval_config.ckpt_indices = ckpt_indices # Remove augmentation during evaluation in test mode dataset_config.aug_list = [] # Setup the model # Overwrite repeated field model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] if start_perspective == 0: # Create validation split create_split.create_split( dataset_config.dataset_dir, dataset_config.dataset_dir + dataset_config.data_split_dir + '/', 'val') inferPerspective(model_config, eval_config, dataset_config, additional_cls) altPerspect_dir = base_dir + dataset_config.data_split_dir + '/alt_perspective/' p_idx = 0 perspective_dirs = const.valid_perspectives() perspective_dirs.sort(key=float) p_count = len(perspective_dirs) for entity_str in perspective_dirs: if not os.path.isdir(os.path.join(altPerspect_dir, entity_str)): continue p_idx += 1 # Option to skip some perspectives if int(entity_str) < start_perspective: continue dataset_config.data_split = entity_str dataset_config.data_split_dir = entity_str dataset_config.dataset_dir = altPerspect_dir inferPerspective(model_config, eval_config, dataset_config, additional_cls) logging.info( '\n\n********************Finished perspective: {} / {} ***********************\n\n' .format(p_idx, p_count))
def inference(rpn_model_path, detect_model_path, avod_config_path): model_config, _, eval_config, dataset_config = \ config_builder.get_configs_from_pipeline_file( avod_config_path, is_training=False) # Setup the model model_name = model_config.model_name # Overwrite repeated field model_config = config_builder.proto_to_obj(model_config) # Switch path drop off during evaluation model_config.path_drop_probabilities = [1.0, 1.0] dataset = get_dataset(dataset_config, 'val') # run avod proposal network rpn_endpoints, sess1, rpn_model = get_proposal_network(model_config, dataset, rpn_model_path) end_points, sess2 = get_detection_network(detect_model_path) all_prediction = [] all_id_list = None all_2d_boxes = [] for idx in range(3769): feed_dict1 = rpn_model.create_feed_dict() kitti_samples = dataset.load_samples([idx]) sample = kitti_samples[0] ''' if sample[constants.KEY_SAMPLE_NAME] < '001100': continue if sample[constants.KEY_SAMPLE_NAME] > '001200': break ''' start_time = time.time() rpn_predictions = sess1.run(rpn_endpoints, feed_dict=feed_dict1) top_anchors = rpn_predictions[RpnModel.PRED_TOP_ANCHORS] top_proposals = box_3d_encoder.anchors_to_box_3d(top_anchors) softmax_scores = rpn_predictions[RpnModel.PRED_TOP_OBJECTNESS_SOFTMAX] proposals_and_scores = np.column_stack((top_proposals, softmax_scores)) top_img_roi = rpn_predictions[RpnModel.PRED_TOP_IMG_ROI] top_bev_roi = rpn_predictions[RpnModel.PRED_TOP_BEV_ROI] roi_num = len(top_img_roi) top_img_roi = np.reshape(top_img_roi, (roi_num, -1)) top_bev_roi = np.reshape(top_bev_roi, (roi_num, -1)) roi_features = np.column_stack((top_img_roi, top_bev_roi)) ''' # save proposal if os.path.exists(os.path.join('/data/ssd/public/jlliu/Kitti/object/training/proposal', '%s.txt'%(sample[constants.KEY_SAMPLE_NAME]))): continue np.savetxt(os.path.join('./proposals_and_scores/', '%s.txt'%sample[constants.KEY_SAMPLE_NAME]), proposals_and_scores, fmt='%.3f') np.savetxt(os.path.join('./roi_features/', '%s_roi.txt'%sample[constants.KEY_SAMPLE_NAME]), roi_features, fmt='%.5f') print('save ' + sample[constants.KEY_SAMPLE_NAME]) ''' # run frustum_pointnets_v2 point_clouds, feature_vec, rot_angle_list, prop_cls_labels = get_pointnet_input(sample, proposals_and_scores, roi_features) try: prediction = detect_batch(sess2, end_points, point_clouds, feature_vec, rot_angle_list, prop_cls_labels) except: traceback.print_exc() continue elapsed_time = time.time() - start_time print(sample[constants.KEY_SAMPLE_NAME], elapsed_time) # concat all predictions for kitti eval id_list = np.ones((len(prediction),)) * int(sample[constants.KEY_SAMPLE_NAME]) if all_id_list is None: all_id_list = id_list else: all_id_list = np.concatenate((all_id_list, id_list), axis=0) for pred in prediction: obj = box_3d_encoder.box_3d_to_object_label(np.array(pred[0:7]), obj_type=type_whitelist[pred[8]]) corners = compute_box_3d(obj) projected = calib_utils.project_to_image(corners.T, sample[constants.KEY_STEREO_CALIB_P2]) x1 = np.amin(projected[0]) y1 = np.amin(projected[1]) x2 = np.amax(projected[0]) y2 = np.amax(projected[1]) all_2d_boxes.append([x1, y1, x2, y2]) all_prediction += prediction # save result pickle.dump({'proposals_and_scores': proposals_and_scores, 'roi_features': roi_features}, open("rpn_out/%s"%sample[constants.KEY_SAMPLE_NAME], "wb")) pickle.dump(prediction, open('final_out/%s' % sample[constants.KEY_SAMPLE_NAME], 'wb')) visualize(dataset, sample, prediction) # for kitti eval write_detection_results('./detection_results', all_prediction, all_id_list, all_2d_boxes)