def __init__(self, experiment_config_path, planes_dir, calib_dir):
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
experiment_config_path, is_training=False)
self.dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split_dir = 'testing'
# These two lines below are necessary for KittiUtils to function properly
self.cluster_split = self.dataset_config.cluster_split
self.config = dataset_config
self.data_split = self.config.data_split
self.name = self.config.name
self.dataset_dir = os.path.expanduser(self.config.dataset_dir)
self.has_labels = self.config.has_labels
self.cluster_split = self.config.cluster_split
self.classes = list(self.config.classes)
self.num_classes = len(self.classes)
self.num_clusters = np.asarray(self.config.num_clusters)
self.model_config = config_builder.proto_to_obj(model_config)
self.paths_config = self.model_config.paths_config
self.checkpoint_dir = self.paths_config.checkpoint_dir
# Build the dataset object
self.dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
self.dataset.train_val_test = "test"
# Score threshold for drawing bounding boxes
self.avod_score_threshold = 0.25
def inference(model_config, eval_config, dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'bev_only_rpn_model':
model = BevOnlyRpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'bev_only_avod_model':
model = BevOnlyAvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
model_evaluator.run_latest_checkpoints()
def set_up_model(pipeline_config, data_split):
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config, is_training=False)
dataset_config = config_builder.proto_to_obj(dataset_config)
train_val_test = data_split
# Always run in test mode
dataset_config.data_split = 'test'
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model
def filter_gt_labels(dataset_config, base_dir):
"""Plots detection errors for xyz, lwh, ry, and shows 3D IoU with
ground truth boxes
"""
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_dir = base_dir
dataset_config.dataset_dir = base_dir
dataset_config.data_split = 'train'
dataset_config.data_split_dir = 'training'
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
dataset.classes = ['Car', 'Bus', 'Pedestrian', 'Cyclist', 'Truck']
dataset.aug_sample_list = []
difficulty = 2
out_dir = base_dir + dataset_config.data_split_dir + '/label_filtered_2/'
if not os.path.exists(out_dir):
os.makedirs(out_dir)
print("Saving filtered labels to: ", out_dir)
# TODO read in augmented labels to filter by 3D point count
# Also changes synthetic to true in read_labels
# dataset.label_dir = base_dir + '/' + dataset_config.data_split_dir + '/label_aug_2/'
dataset.label_dir = base_dir + '/' + dataset_config.data_split_dir + '/label_2/'
files = os.listdir(dataset.label_dir)
num_files = len(files)
sample_idx = 0
for file in files:
sys.stdout.flush()
sys.stdout.write('\r{} / {}'.format(sample_idx + 1, num_files))
sample_idx = sample_idx + 1
filepath = dataset.label_dir + '/' + file
idx = int(os.path.splitext(file)[0])
# Get filtered ground truth
all_gt_objs = obj_utils.read_labels(dataset.label_dir,
idx,
synthetic=False)
if all_gt_objs != None:
all_gt_objs = dataset.kitti_utils.filter_labels(
all_gt_objs,
max_forward=dataset.kitti_utils.area_extents[2, 1],
max_side=dataset.kitti_utils.area_extents[0, 1])
# Save gt to output file
save_filtered_objs(all_gt_objs, idx, out_dir)
def test_path_drop_weights(self):
# Tests the effect of path-drop on network's feature maps.
# It sets up a minimal-training process to check the
# feature before and after running the 'train_op' while
# path-drop is in effect.
train_val_test = 'train'
# overwrite the training iterations
self.train_config.max_iterations = 2
self.train_config.overwrite_checkpoints = True
# Overwrite path drop probabilities
model_config = config_builder.proto_to_obj(self.model_config)
model_config.path_drop_probabilities = [0.0, 0.8]
with tf.Graph().as_default():
# Set a graph-level seed
tf.set_random_seed(1245)
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=self.dataset)
prediction_dict = model.build()
losses_dict, total_loss = model.loss(prediction_dict)
global_summaries = set([])
# Optimizer
training_optimizer = optimizer_builder.build(
self.train_config.optimizer,
global_summaries)
train_op = slim.learning.create_train_op(
total_loss,
training_optimizer)
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
for step in range(1, self.train_config.max_iterations):
feed_dict = model.create_feed_dict()
if step == 1:
current_feature_maps = sess.run(model.img_feature_maps,
feed_dict=feed_dict)
exp_feature_maps = current_feature_maps
train_op_loss = sess.run(train_op, feed_dict=feed_dict)
print('Step {}, Total Loss {:0.3f} '.
format(step, train_op_loss))
updated_feature_maps = sess.run(model.img_feature_maps,
feed_dict=feed_dict)
# The feature maps should have remained the same since
# the image path was dropped
np.testing.assert_array_almost_equal(
updated_feature_maps, exp_feature_maps, decimal=4)
def get_dataset(dataset_config, data_split):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = data_split in ['train', 'val']
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
return dataset
def test_disable_path_drop(self):
# Test path drop is disabled when the probabilities
# are set to 1.0.
train_val_test = 'train'
# Overwrite path drop probabilities
model_config = config_builder.proto_to_obj(self.model_config)
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=self.dataset)
model.build()
# These variables are set during path drop only
# in the case of no path-drop, they should be non-existence
self.assertFalse(hasattr(model, 'img_path_drop_mask'))
self.assertFalse(hasattr(model, 'bev_path_drop_mask'))
def set_up_model_test_mode(pipeline_config_path, data_split):
"""Returns the model and its config in test mode."""
model_config, _, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=False)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
# Use the validation set
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
# Remove augmentation when in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'avod_ssd_model':
model = AvodSSDModel(model_config,
train_val_test='test',
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model, model_config
def evaluate(model_config, eval_config, dataset_config):
# Parse eval config
eval_mode = eval_config.eval_mode
if eval_mode not in ['val', 'test']:
raise ValueError('Evaluation mode can only be set to `val` or `test`')
evaluate_repeatedly = eval_config.evaluate_repeatedly
# Parse dataset config
data_split = dataset_config.data_split
if data_split == 'train':
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = True
elif data_split.startswith('val'):
dataset_config.data_split_dir = 'training'
# Don't load labels for val split when running in test mode
if eval_mode == 'val':
dataset_config.has_labels = True
elif eval_mode == 'test':
dataset_config.has_labels = False
elif data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
else:
raise ValueError('Invalid data split', data_split)
# Convert to object to overwrite repeated fields
dataset_config = config_builder.proto_to_obj(dataset_config)
# Remove augmentation during evaluation
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Convert to object to overwrite repeated fields
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
if evaluate_repeatedly:
model_evaluator.repeated_checkpoint_run()
else:
model_evaluator.run_latest_checkpoints()
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'fpn_people_dual_NHSP_train'#'fpn_people_dual_SHPL_train'#'pyramid_people_with_NHSP_example_train'#'pyramid_people_with_NHSP_example'
data_split = 'val'#'test'#'val'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
save_alphas = True # Save alphas (observation angles)
# Checkpoints below this are skipped
min_step = 20000
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
avod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = avod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
final_predictions_root_dir = predictions_root_dir + \
'/final_predictions_and_scores/' + dataset.data_split
print('Converting detections from', final_predictions_root_dir)
if not global_steps:
global_steps = os.listdir(final_predictions_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = final_predictions_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_predictions_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_predictions_3d_dir = predictions_root_dir + \
'/kitti_predictions_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_predictions_3d_dir):
os.makedirs(kitti_predictions_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting detections from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('3D Detections saved to:', kitti_predictions_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_predictions_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# # Swap l, w for predictions where w > l
# swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
# fixed_predictions = np.copy(all_predictions)
# fixed_predictions[swapped_indices, 3] = all_predictions[
# swapped_indices, 4]
# fixed_predictions[swapped_indices, 4] = all_predictions[
# swapped_indices, 3]
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = all_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, stereo_calib_p2,
truncate=True, image_size=image.size)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Get object types
all_pred_classes = all_predictions[:, 8].astype(np.int32)
obj_types = [dataset.classes[class_idx]
for class_idx in all_pred_classes]
# Truncation and Occlusion are always empty (see below)
# Alpha
if not save_alphas:
kitti_predictions[:, 3] = -10 * \
np.ones((len(kitti_predictions)), dtype=np.int32)
else:
alphas = all_predictions[:, 6] - \
np.arctan2(all_predictions[:, 0], all_predictions[:, 2])
kitti_predictions[:, 3] = alphas
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones((len(kitti_predictions), 2),
dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones((len(kitti_predictions), 3),
dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones((len(kitti_predictions), 1),
dtype=np.int32)
# Stack 2D predictions text
kitti_text_2d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:8],
kitti_empty_2,
kitti_empty_3,
kitti_empty_4,
kitti_predictions[:, 15]])
# Stack 3D predictions text
kitti_text_3d = np.column_stack([obj_types,
kitti_empty_1,
kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, kitti_text_2d,
newline='\r\n', fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, kitti_text_3d,
newline='\r\n', fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def test(model_config, eval_config,
dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
#model_evaluator = Evaluator(model, dataset_config, eval_config)
#model_evaluator.run_latest_checkpoints()
# Create a variable tensor to hold the global step
global_step_tensor = tf.Variable(0, trainable=False, name='global_step')
allow_gpu_mem_growth = eval_config.allow_gpu_mem_growth
if allow_gpu_mem_growth:
# GPU memory config
config = tf.ConfigProto()
config.gpu_options.allow_growth = allow_gpu_mem_growth
_sess = tf.Session(config=config)
else:
_sess = tf.Session()
_prediction_dict = model.build()
_saver = tf.train.Saver()
trainer_utils.load_checkpoints(model_config.paths_config.checkpoint_dir,
_saver)
num_checkpoints = len(_saver.last_checkpoints)
print("test:",num_checkpoints)
checkpoint_to_restore = _saver.last_checkpoints[num_checkpoints-1]
_saver.restore(_sess, checkpoint_to_restore)
num_samples = model.dataset.num_samples
num_valid_samples = 0
current_epoch = model.dataset.epochs_completed
while current_epoch == model.dataset.epochs_completed:
# Keep track of feed_dict speed
start_time = time.time()
feed_dict = model.create_feed_dict()
feed_dict_time = time.time() - start_time
# Get sample name from model
sample_name = model.sample_info['sample_name']
num_valid_samples += 1
print("Step: {} / {}, Inference on sample {}".format(
num_valid_samples, num_samples,
sample_name))
print("test mode")
inference_start_time = time.time()
# Don't calculate loss or run summaries for test
predictions = _sess.run(_prediction_dict,
feed_dict=feed_dict)
inference_time = time.time() - inference_start_time
print("inference time:", inference_time)
predictions_and_scores = get_avod_predicted_boxes_3d_and_scores(predictions)
#print(predictions_and_scores)
#im_path = os.path.join(dataset_dir, 'training/image_2/{:06d}.png'.format(img_idx))
#im = cv2.imread(im_path)
#cv2.imshow('result',im)
#cv2.waitKey(30)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
gt_classes = ['Car']
fig_size = (10, 6.1)
avod_score_threshold = 0.1
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
if len(prediction_boxes_3d) > 0:
dataset_dir = model.dataset.dataset_dir
sample_name = (model.dataset.sample_names[model.dataset._index_in_epoch - 1])
img_idx = int(sample_name)
print("frame_index",img_idx)
image_path = model.dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
if model.dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
filtered_gt_objs = model.dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
stereo_calib = calib_utils.read_calibration(dataset.calib_dir,
img_idx)
calib_p2 = stereo_calib.p2
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, calib_p2,
truncate=True, image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_2d_axes,
pred_3d_axes,
True,
True,
gt_classes,
False)
#cv2.imshow('result',pred_fig)
print(type(pred_fig))
pred_fig.canvas.draw()
img = np.fromstring(pred_fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
img = img.reshape(pred_fig.canvas.get_width_height()[::-1] + (3,))
cv2.imshow('result',img)
#draw bird view
kitti_utils = model.dataset.kitti_utils
print(img.shape[0:2])
point_cloud = kitti_utils.get_point_cloud(
'lidar', img_idx, (370, 1242))
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
density_map = np.array(bev_images.get("density_map"))
_, box_points_norm = box_3d_projector.project_to_bev(
final_prediction_boxes_3d, [[-40, 40], [0, 70]])
density_map = draw_boxes(density_map, box_points_norm)
cv2.imshow('lidar',density_map)
cv2.waitKey(-1)
def inference(model_config,
eval_config,
dataset_config,
base_dir,
ckpt_indices,
additional_cls,
start_perspective=0):
logging.info("Additional class: {}".format(additional_cls))
logging.info("ckpt_indices: {}".format(ckpt_indices))
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_dir = base_dir
dataset_config.dataset_dir = base_dir
dataset_config.data_split = 'train'
dataset_config.data_split_dir = 'training'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Setup the model
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
if start_perspective == 0:
# Create validation split
create_split.create_split(
dataset_config.dataset_dir,
dataset_config.dataset_dir + dataset_config.data_split_dir + '/',
'val')
inferPerspective(model_config, eval_config, dataset_config,
additional_cls)
altPerspect_dir = base_dir + dataset_config.data_split_dir + '/alt_perspective/'
p_idx = 0
perspective_dirs = const.valid_perspectives()
perspective_dirs.sort(key=float)
p_count = len(perspective_dirs)
for entity_str in perspective_dirs:
if not os.path.isdir(os.path.join(altPerspect_dir, entity_str)):
continue
p_idx += 1
# Option to skip some perspectives
if int(entity_str) < start_perspective:
continue
dataset_config.data_split = entity_str
dataset_config.data_split_dir = entity_str
dataset_config.dataset_dir = altPerspect_dir
inferPerspective(model_config, eval_config, dataset_config,
additional_cls)
logging.info(
'\n\n********************Finished perspective: {} / {} ***********************\n\n'
.format(p_idx, p_count))
def inference(rpn_model_path, detect_model_path, avod_config_path):
model_config, _, eval_config, dataset_config = \
config_builder.get_configs_from_pipeline_file(
avod_config_path, is_training=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
dataset = get_dataset(dataset_config, 'val')
# run avod proposal network
rpn_endpoints, sess1, rpn_model = get_proposal_network(model_config, dataset, rpn_model_path)
end_points, sess2 = get_detection_network(detect_model_path)
all_prediction = []
all_id_list = None
all_2d_boxes = []
for idx in range(3769):
feed_dict1 = rpn_model.create_feed_dict()
kitti_samples = dataset.load_samples([idx])
sample = kitti_samples[0]
'''
if sample[constants.KEY_SAMPLE_NAME] < '001100':
continue
if sample[constants.KEY_SAMPLE_NAME] > '001200':
break
'''
start_time = time.time()
rpn_predictions = sess1.run(rpn_endpoints, feed_dict=feed_dict1)
top_anchors = rpn_predictions[RpnModel.PRED_TOP_ANCHORS]
top_proposals = box_3d_encoder.anchors_to_box_3d(top_anchors)
softmax_scores = rpn_predictions[RpnModel.PRED_TOP_OBJECTNESS_SOFTMAX]
proposals_and_scores = np.column_stack((top_proposals,
softmax_scores))
top_img_roi = rpn_predictions[RpnModel.PRED_TOP_IMG_ROI]
top_bev_roi = rpn_predictions[RpnModel.PRED_TOP_BEV_ROI]
roi_num = len(top_img_roi)
top_img_roi = np.reshape(top_img_roi, (roi_num, -1))
top_bev_roi = np.reshape(top_bev_roi, (roi_num, -1))
roi_features = np.column_stack((top_img_roi, top_bev_roi))
'''
# save proposal
if os.path.exists(os.path.join('/data/ssd/public/jlliu/Kitti/object/training/proposal', '%s.txt'%(sample[constants.KEY_SAMPLE_NAME]))):
continue
np.savetxt(os.path.join('./proposals_and_scores/', '%s.txt'%sample[constants.KEY_SAMPLE_NAME]), proposals_and_scores, fmt='%.3f')
np.savetxt(os.path.join('./roi_features/', '%s_roi.txt'%sample[constants.KEY_SAMPLE_NAME]), roi_features, fmt='%.5f')
print('save ' + sample[constants.KEY_SAMPLE_NAME])
'''
# run frustum_pointnets_v2
point_clouds, feature_vec, rot_angle_list, prop_cls_labels = get_pointnet_input(sample, proposals_and_scores, roi_features)
try:
prediction = detect_batch(sess2, end_points, point_clouds, feature_vec, rot_angle_list, prop_cls_labels)
except:
traceback.print_exc()
continue
elapsed_time = time.time() - start_time
print(sample[constants.KEY_SAMPLE_NAME], elapsed_time)
# concat all predictions for kitti eval
id_list = np.ones((len(prediction),)) * int(sample[constants.KEY_SAMPLE_NAME])
if all_id_list is None:
all_id_list = id_list
else:
all_id_list = np.concatenate((all_id_list, id_list), axis=0)
for pred in prediction:
obj = box_3d_encoder.box_3d_to_object_label(np.array(pred[0:7]), obj_type=type_whitelist[pred[8]])
corners = compute_box_3d(obj)
projected = calib_utils.project_to_image(corners.T, sample[constants.KEY_STEREO_CALIB_P2])
x1 = np.amin(projected[0])
y1 = np.amin(projected[1])
x2 = np.amax(projected[0])
y2 = np.amax(projected[1])
all_2d_boxes.append([x1, y1, x2, y2])
all_prediction += prediction
# save result
pickle.dump({'proposals_and_scores': proposals_and_scores, 'roi_features': roi_features}, open("rpn_out/%s"%sample[constants.KEY_SAMPLE_NAME], "wb"))
pickle.dump(prediction, open('final_out/%s' % sample[constants.KEY_SAMPLE_NAME], 'wb'))
visualize(dataset, sample, prediction)
# for kitti eval
write_detection_results('./detection_results', all_prediction, all_id_list, all_2d_boxes)
