def set_up_model_test_mode(pipeline_config_path, data_split):
"""Returns the model and its config in test mode."""
model_config, _, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=False)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
# Use the validation set
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
# Remove augmentation when in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config, train_val_test='test', dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model, model_config
def main():
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = "trainval"
unittest_dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
gen_label_clusters.main(unittest_dataset)
gen_mini_batches.main(unittest_dataset)
def setUpClass(cls):
# Initialize the Kitti dataset
test_dir = tests.test_path()
# Get the unittest-kitti dataset
dataset_builder = DatasetBuilder()
cls.dataset = dataset_builder.build_kitti_dataset(
dataset_builder.KITTI_UNITTEST)
cls.log_dir = test_dir + '/logs'
cls.bev_vgg_cls = vgg.BevVggClassification()
def get_fake_dataset(self, data_split, directory):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
# Overwrite config values
dataset_config.data_split = data_split
dataset_config.dataset_dir = directory
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
return dataset
def test_project_to_image_space_tensors(self):
anchors = np.asarray([[0, 0, 3, 2, 0, 6], [3, 0, 3, 2, 0, 2]],
dtype=np.float64)
img_idx = int('000217')
img_shape = [375, 1242]
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = 'train'
dataset_config.dataset_dir = tests.test_path() + \
"/datasets/Kitti/object"
dataset = DatasetBuilder().build_kitti_dataset(dataset_config)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
# Project the 3D points in numpy space
img_corners, img_corners_norm = anchor_projector.project_to_image_space(
anchors, stereo_calib_p2, img_shape)
# convert the required params to tensors
tf_stereo_calib_p2 = tf.convert_to_tensor(stereo_calib_p2,
dtype=tf.float32)
tf_anchors = tf.convert_to_tensor(anchors, dtype=tf.float32)
tf_img_shape = tf.convert_to_tensor(img_shape, dtype=tf.float32)
# Project the 3D points in tensor space
img_corners_tensor, img_corners_norm_tensor = \
anchor_projector.tf_project_to_image_space(tf_anchors,
tf_stereo_calib_p2,
tf_img_shape)
sess = tf.Session()
with sess.as_default():
img_corners_out = img_corners_tensor.eval()
img_corners_norm_out = img_corners_norm_tensor.eval()
np.testing.assert_allclose(img_corners,
img_corners_out,
atol=1e-04,
err_msg='Incorrect corner projection')
np.testing.assert_allclose(
img_corners_norm,
img_corners_norm_out,
atol=1e-04,
err_msg='Incorrect normalized corner projection')
def set_up_model():
test_pipeline_config_path = mlod.root_dir() + \
'/configs/mlod_exp_example.config'
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
dataset_config = config_builder.proto_to_obj(dataset_config)
train_val_test = 'test'
dataset_config.data_split = 'test'
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model
def main():
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
image_path = dataset.get_rgb_image_path('000000')
# cv2
cv2_image1 = cv2.imread(image_path)
cv2_image2 = cv2.imread(image_path)
cv2_image3 = cv2.imread(image_path)
cv2_image1_shape = cv2_image1.shape
cv2_image2_shape = cv2_image2.shape
cv2_image3_shape = cv2_image3.shape
print(cv2_image1_shape)
print(cv2_image2_shape)
print(cv2_image3_shape)
# PIL
pil_image1 = Image.open(image_path)
pil_image2 = Image.open(image_path)
pil_image3 = Image.open(image_path)
pil_image1_shape = pil_image1.size
pil_image2_shape = pil_image2.size
pil_image3_shape = pil_image3.size
print(pil_image1_shape)
print(pil_image2_shape)
print(pil_image3_shape)
pil_image1 = np.asarray(pil_image1)
pil_image2 = np.asarray(pil_image2)
pil_image3 = np.asarray(pil_image3)
def main():
test_pipeline_config_path = mlod.root_dir() + \
'/data/configs/official/cars/cars_000_vanilla.config'
model_config, train_config, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# train_val_test = 'val'
# dataset_config.data_split = 'val'
train_val_test = 'test'
dataset_config.data_split = 'trainval'
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = False
# dataset_config.cache_config.cache_images = True
# dataset_config.cache_config.cache_depth_maps = True
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
kitti_utils = dataset.kitti_utils
bev_source = 'lidar'
# sample_name = '000000'
# img_idx = np.random.randint(0, 1000)
# sample_name = '{:06d}'.format(img_idx)
num_samples = 200
all_load_times = []
all_bev_times = []
for sample_idx in range(num_samples):
sys.stdout.write('\rSample {} / {}'.format(sample_idx,
num_samples - 1))
img_idx = sample_idx
sample_name = '{:06d}'.format(sample_idx)
loading_start_time = time.time()
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx)
point_cloud = kitti_utils.get_point_cloud(bev_source, int(sample_name),
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
all_load_times.append(time.time() - loading_start_time)
bev_start_time = time.time()
bev_maps = kitti_utils.create_bev_maps(point_cloud, ground_plane)
bev_end_time = time.time()
all_bev_times.append(bev_end_time - bev_start_time)
print('')
print('Load mean:', np.mean(all_load_times))
print('Load median:', np.median(all_load_times))
print('BEV mean:', np.mean(all_bev_times))
print('BEV median:', np.median(all_bev_times))
def test_data_splits(self):
bad_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
# Test invalid splits
bad_config.data_split = "bad"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "train"
bad_config.data_split = "training"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "val"
bad_config.data_split = "validation"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "test"
bad_config.data_split = "testing"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Train split
train_dataset = self.get_fake_dataset('train', self.fake_kitti_dir)
self.assertEqual(train_dataset.num_samples, 7)
# Validation split
validation_dataset = self.get_fake_dataset('val', self.fake_kitti_dir)
self.assertEqual(validation_dataset.num_samples, 6)
# Train + validation split
trainval_dataset = self.get_fake_dataset('trainval',
self.fake_kitti_dir)
self.assertEqual(trainval_dataset.num_samples, 13)
# Test split
test_dataset = self.get_fake_dataset('test', self.fake_kitti_dir)
self.assertEqual(test_dataset.num_samples, 10)
def main():
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL)
# get proposals
proposal_output_dir = mlod.root_dir() + \
"/data/predictions/rpn_model/proposals_and_scores/" + \
dataset.data_split
global_steps = os.listdir(proposal_output_dir)
print('Checkpoints found ', global_steps)
all_recalls = []
for step in global_steps:
for sample_name in dataset.sample_list:
img_idx = int(sample_name)
# ------------------------------------
# Load proposals and scores from files
# ------------------------------------
proposals_scores_dir = proposal_output_dir + \
"{}/{}/{}.txt".format(dataset.data_split,
step,
sample_name)
if not os.path.exists(proposals_scores_dir):
print('File {} not found, skipping'.format(sample_name))
continue
proposals_scores = np.loadtxt(proposals_scores_dir)
proposals = proposals_scores[:, 0:-1]
proposal_iou_format = \
box_3d_encoder.box_3d_to_3d_iou_format(proposals)
# scores are in the last column
scores = proposals_scores[:, -1]
# -----------------------
# Get ground truth labels
# -----------------------
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
_, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects(
gt_objects, ['Car', 'car'])
score_thresholds = np.array([0.3])
iou_threshold = 0.0025
# calculate RPN recall and precision
precision, recall = evaluation.evaluate_3d([gt_3d_bbs],
[proposal_iou_format],
[scores],
score_thresholds,
iou_threshold)
print('Recall ', recall[0])
print('Precision ', precision[0])
all_recalls.append(recall)
def setUpClass(cls):
pipeline_config = pipeline_pb2.NetworkPipelineConfig()
dataset_config = pipeline_config.dataset_config
config_path = mlod.root_dir() + '/configs/unittest_model.config'
cls.model_config = config_build.get_model_config_from_file(config_path)
input_config = cls.model_config.input_config
input_config.bev_depth = 5 # unittest-kitti has [2, 1, 1] clusters
dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def inference(model_config, eval_config, dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
model_evaluator.run_latest_checkpoints()
def set_up_video_dataset(dataset_config, dataset_dir, data_split,
data_split_dir):
# Overwrite fields
dataset_config.name = 'kitti_video'
dataset_config.dataset_dir = dataset_dir
dataset_config.data_split = data_split
dataset_config.data_split_dir = data_split_dir
dataset_config.has_labels = False
# Overwrite repeated fields
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
return dataset
def setUp(self):
tf.test.TestCase.setUp(self)
test_pipeline_config_path = mlod.root_dir() + \
'/configs/unittest_stagewise.config'
self.model_config, self.train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# unittest-kitti has [2, 1, 1] clusters
input_config = self.model_config.input_config
input_config.bev_depth = 5
# Generate dataset
self.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST,
use_defaults=False,
new_cfg=dataset_config)
def train(rpn_model_config, mlod_model_config, rpn_train_config,
mlod_train_config, dataset_config):
train_val_test = 'train'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
paths_config = rpn_model_config.paths_config
rpn_checkpoint_dir = paths_config.checkpoint_dir
with tf.Graph().as_default():
model = RpnModel(rpn_model_config,
train_val_test=train_val_test,
dataset=dataset)
trainer.train(model, rpn_train_config)
# load the weights back in
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
trainer_utils.load_checkpoints(rpn_checkpoint_dir, saver)
checkpoint_to_restore = saver.last_checkpoints[-1]
trainer_utils.load_model_weights(model, sess,
checkpoint_to_restore)
# Merge RPN configs with MLOD - This will overwrite
# the appropriate configs set for MLOD while keeping
# the common configs the same.
rpn_model_config.MergeFrom(mlod_model_config)
rpn_train_config.MergeFrom(mlod_train_config)
mlod_model_merged = deepcopy(rpn_model_config)
mlod_train_merged = deepcopy(rpn_train_config)
with tf.Graph().as_default():
model = MlodModel(mlod_model_merged,
train_val_test=train_val_test,
dataset=dataset)
trainer.train(model,
mlod_train_merged,
stagewise_training=True,
init_checkpoint_dir=rpn_checkpoint_dir)
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
trainer.train(model, train_config)
def set_up_model_train_mode(pipeline_config_path, data_split):
"""Returns the model and its train_op."""
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=True)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=data_split,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=data_split,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
prediction_dict = model.build()
losses_dict, total_loss = model.loss(prediction_dict)
# These parameters are required to set up the optimizer
global_summaries = set([])
global_step_tensor = tf.Variable(0, trainable=False)
training_optimizer = optimizer_builder.build(train_config.optimizer,
global_summaries,
global_step_tensor)
# Set up the train op
train_op = slim.learning.create_train_op(total_loss, training_optimizer)
return model, train_op
def main():
"""
Calculates clusters for each class
Returns:
all_clusters: list of clusters for each class
all_std_devs: list of cluster standard deviations for each class
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
# Calculate the remaining clusters
# Load labels corresponding to the sample list for clustering
sample_list = dataset.load_sample_names(dataset.cluster_split)
all_dims = []
num_samples = len(sample_list)
for sample_idx in range(num_samples):
sys.stdout.write("\rClustering labels {} / {}".format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = sample_list[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_lwh = LabelClusterUtils._filter_labels_by_class(
obj_labels, dataset.classes)
if filtered_lwh[0]:
all_dims.extend(filtered_lwh[0])
all_dims = np.array(all_dims)
print("\nFinished reading labels, clustering data...\n")
# Print 3 decimal places
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
# Calculate average cluster
k_means = KMeans(n_clusters=1,
random_state=0).fit(all_dims)
cluster_centre = k_means.cluster_centers_[0]
# Calculate std. dev
std_dev = np.std(all_dims, axis=0)
# Calculate 2 and 3 standard deviations below the mean
two_sigma_length_lo = cluster_centre[0] - 2 * std_dev[0]
three_sigma_length_lo = cluster_centre[0] - 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
small_mask_2 = all_dims[:, 0] < two_sigma_length_lo
small_dims_2 = all_dims[small_mask_2]
small_mask_3 = all_dims[:, 0] < three_sigma_length_lo
small_dims_3 = all_dims[small_mask_3]
small_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(small_dims_2)
small_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(small_dims_3)
small_std_dev_2 = np.std(small_dims_2, axis=0)
small_std_dev_3 = np.std(small_dims_3, axis=0)
print('small_k_means_2:', small_k_means_2.cluster_centers_)
print('small_k_means_3:', small_k_means_3.cluster_centers_)
print('small_std_dev_2:', small_std_dev_2)
print('small_std_dev_3:', small_std_dev_3)
# Calculate 2 and 3 standard deviations above the mean
two_sigma_length_hi = cluster_centre[0] + 2 * std_dev[0]
three_sigma_length_hi = cluster_centre[0] + 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
large_mask_2 = all_dims[:, 0] > two_sigma_length_hi
large_dims_2 = all_dims[large_mask_2]
large_mask_3 = all_dims[:, 0] > three_sigma_length_hi
large_dims_3 = all_dims[large_mask_3]
large_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(large_dims_2)
large_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(large_dims_3)
large_std_dev_2 = np.std(large_dims_2, axis=0)
large_std_dev_3 = np.std(large_dims_3, axis=0)
print('large_k_means_2:', large_k_means_2.cluster_centers_)
print('large_k_means_3:', large_k_means_3.cluster_centers_)
print('large_std_dev_2:', large_std_dev_2)
print('large_std_dev_3:', large_std_dev_3)
def setUpClass(cls):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
cls.label_dir = cls.dataset.label_dir
def main():
"""Shows a flipped sample in 3D
"""
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL)
##############################
# Options
##############################
# sample_name = "000191"
sample_name = "000104"
img_idx = int(sample_name)
print("Showing anchors for sample {}".format(sample_name))
##############################
# Load Sample Data
##############################
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# Get point cloud
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = np.array(point_cloud).T
# Ground truth
gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
# Filter ground truth
gt_labels = dataset.kitti_utils.filter_labels(gt_labels)
##############################
# Flip stuff
##############################
image_flipped = np.fliplr(image)
# Flip ground plane coeff (x)
ground_plane_flipped = np.copy(ground_plane)
ground_plane_flipped[0] = -ground_plane_flipped[0]
# Flip 3D points
points_flipped = kitti_aug.flip_points(points)
# Get point cloud colours
point_colours_flipped = project_flipped_img_to_point_cloud(
points_flipped, image_flipped, dataset.calib_dir, img_idx)
# Flip ground truth boxes
gt_labels_flipped = [
kitti_aug.flip_label_in_3d_only(obj) for obj in gt_labels
]
##############################
# VTK Visualization
##############################
# Axes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Point cloud
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points_flipped,
point_colours=point_colours_flipped)
# # Ground Truth Boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(gt_labels_flipped,
VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations=True)
# Renderer
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Add Actors to Rendered
vtk_renderer.AddActor(axes)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_point_cloud.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""This demo shows RPN proposals and MLOD predictions in the
3D point cloud.
Keys:
F1: Toggle proposals
F2: Toggle predictions
F3: Toggle 3D voxel grid
F4: Toggle point cloud
F5: Toggle easy ground truth objects (Green)
F6: Toggle medium ground truth objects (Orange)
F7: Toggle hard ground truth objects (Red)
F8: Toggle all ground truth objects (default off)
F9: Toggle ground slice filter (default off)
F10: Toggle offset slice filter (default off)
"""
##############################
# Options
##############################
rpn_score_threshold = 0.1
mlod_score_threshold = 0.1
proposals_line_width = 1.0
predictions_line_width = 3.0
show_orientations = True
point_cloud_source = 'lidar'
# Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>)
config_dir = None
checkpoint_name = 'mlod_fpn_people_n_m'
global_step = 135000 # Latest checkpoint
#data_split = 'val_half'
#data_split = 'val'
data_split = 'test'
# Show 3D iou text
draw_ious_3d = False
sample_name = '000031'
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# val split
# sample_name = '000181'
# sample_name = '000751'
# sample_name = '000843'
# sample_name = '000944'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# val split
# sample_name = '000015'
# sample_name = '000048'
# sample_name = '000058'
# sample_name = '000076' # Medium, few ped, 1 cyc
# sample_name = '000108'
# sample_name = '000118'
# sample_name = '000145'
# sample_name = '000153'
# sample_name = '000186'
# sample_name = '000195'
# sample_name = '000199'
# sample_name = '000397'
# sample_name = '004425'
# sample_name = '004474' # Hard, many ped, 1 cyc
# sample_name = '004657' # Hard, Few cycl, few ped
# sample_name = '006071'
# sample_name = '006828' # Hard, Few cycl, few ped
# sample_name = '006908' # Hard, Few cycl, few ped
# sample_name = '007412'
# sample_name = '007318' # Hard, Few cycl, few ped
##############################
# End of Options
##############################
if data_split == 'test':
draw_ious_3d = False
if config_dir is None:
config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_names[sample_idx]
##############################
# Setup Paths
##############################
img_idx = int(sample_name)
# Text files directory
proposals_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_predictions_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
# Output images directory
img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/predictions/images_3d/{}/{}/{}'.format(dataset.data_split,
global_step,
rpn_score_threshold)
if not os.path.exists(img_out_dir):
os.makedirs(img_out_dir)
##############################
# Proposals
##############################
# Load proposals from files
proposals_and_scores = np.loadtxt(
proposals_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
proposals = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
rpn_score_mask = proposal_scores > rpn_score_threshold
proposals = proposals[rpn_score_mask]
proposal_scores = proposal_scores[rpn_score_mask]
print('Proposals:', len(proposal_scores), proposal_scores)
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposals]
##############################
# Predictions
##############################
# Load proposals from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32)
mlod_score_mask = prediction_scores >= mlod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask]
prediction_scores = prediction_scores[mlod_score_mask]
print('Predictions: ', len(prediction_scores), prediction_scores)
final_predictions = np.copy(prediction_boxes_3d)
# # Swap l, w for predictions where w > l
# swapped_indices = predictions[:, 4] > predictions[:, 3]
# final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4]
# final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3]
prediction_objs = []
dataset.classes = ['Pedestrian', 'Cyclist', 'Car']
for pred_idx in range(len(final_predictions)):
prediction_box_3d = final_predictions[pred_idx]
prediction_type = dataset.classes[prediction_types[pred_idx]]
prediction_obj = box_3d_encoder.box_3d_to_object_label(
prediction_box_3d, obj_type=prediction_type)
prediction_objs.append(prediction_obj)
##############################
# Ground Truth
##############################
dataset.has_labels = False
if dataset.has_labels:
# Get ground truth labels
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# 3D IoU
##############################
if draw_ious_3d:
# Convert to box_3d
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in prediction_objs
]
max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d,
pred_boxes_3d)
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source,
img_idx,
image_shape=image.shape)
point_cloud = np.asarray(point_cloud)
# Filter point cloud to extents
area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]])
bev_extents = area_extents[[0, 2]]
points = point_cloud.T
point_filter = obj_utils.get_point_filter(point_cloud, area_extents)
points = points[point_filter]
point_colours = vis_utils.project_img_to_point_cloud(
points, image, dataset.calib_dir, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
# Ground plane
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
vtk_axes = vtk.vtkAxesActor()
vtk_axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for proposal boxes
vtk_proposal_boxes = VtkBoxes()
vtk_proposal_boxes.set_line_width(proposals_line_width)
vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS)
# Create VtkBoxes for prediction boxes
vtk_prediction_boxes = VtkPyramidBoxes()
vtk_prediction_boxes.set_line_width(predictions_line_width)
vtk_prediction_boxes.set_objects(prediction_objs,
COLOUR_SCHEME_PREDICTIONS,
show_orientations)
# Create VtkBoxes for ground truth
vtk_hard_gt_boxes = VtkBoxes()
vtk_medium_gt_boxes = VtkBoxes()
vtk_easy_gt_boxes = VtkBoxes()
vtk_all_gt_boxes = VtkBoxes()
vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations)
# Create VtkTextLabels for 3D ious
vtk_text_labels = VtkTextLabels()
if draw_ious_3d and len(all_gt_boxes_3d) > 0:
gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3]
vtk_text_labels.set_text_labels(
gt_positions_3d,
['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d])
# Create VtkGroundPlane
vtk_ground_plane = VtkGroundPlane()
vtk_slice_bot_plane = VtkGroundPlane()
vtk_slice_top_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, bev_extents)
vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents)
vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_text_labels.vtk_actor)
# Add ground plane and slice planes
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor)
#vtk_renderer.AddActor(vtk_axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3)
vtk_proposal_boxes.vtk_actor.SetVisibility(0)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.SetVisibility(0)
vtk_slice_bot_plane.vtk_actor.SetVisibility(0)
vtk_slice_top_plane.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(3.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
custom_interactor = vis_utils.CameraInfoInteractorStyle([
vtk_proposal_boxes.vtk_actor,
vtk_prediction_boxes.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor,
vtk_slice_bot_plane.vtk_actor,
vtk_slice_top_plane.vtk_actor,
vtk_text_labels.vtk_actor,
])
vtk_render_window_interactor.SetInteractorStyle(custom_interactor)
# Render in VTK
vtk_render_window.Render()
# Take a screenshot
window_to_image_filter = vtk.vtkWindowToImageFilter()
window_to_image_filter.SetInput(vtk_render_window)
window_to_image_filter.Update()
png_writer = vtk.vtkPNGWriter()
file_name = img_out_dir + "/{}.png".format(sample_name)
png_writer.SetFileName(file_name)
png_writer.SetInputData(window_to_image_filter.GetOutput())
png_writer.Write()
print('Screenshot saved to ', file_name)
vtk_render_window_interactor.Start() # Blocking
dataset_config.data_split = 'trainval'
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = True
# Set CUDA device id
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
# Convert to object to overwrite repeated fields
dataset_config = config_builder.proto_to_obj(dataset_config)
# Remove augmentation during evaluation
dataset_config.aug_list = ['flipping']
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Convert to object to overwrite repeated fields
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
index = 76
samples = dataset.load_samples([index])
depth_map= samples[0].get(constants.KEY_DPT_INPUT)
print(depth_map.shape)
image_input = samples[0].get(constants.KEY_IMAGE_INPUT)
def setUpClass(cls):
cls.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST)
cls.mb_utils = cls.dataset.kitti_utils.mini_batch_utils
def main():
"""This demo visualizes box 8C format predicted by MLOD, before
getting converted to Box 3D.
Keys:
F1: Toggle predictions
F2: Toggle easy ground truth objects (Green)
F3: Toggle medium ground truth objects (Orange)
F4: Toggle hard ground truth objects (Red)
F5: Toggle all ground truth objects (default off)
F6: Toggle 3D voxel grid
F7: Toggle point cloud
"""
##############################
# Options
##############################
mlod_score_threshold = 0.1
show_orientations = True
checkpoint_name = 'mlod_exp_8c'
global_step = None
sample_name = None
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
##############################
# Setup Paths
##############################
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001'
sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx]
img_idx = int(sample_name)
# Text files directory
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_boxes_8c_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(predictions_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
##############################
# predictions
##############################
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
predictions_boxes_8c = predictions_and_scores[:, 0:24]
prediction_scores = predictions_and_scores[:, 24]
score_mask = prediction_scores >= mlod_score_threshold
predictions_boxes_8c = predictions_boxes_8c[score_mask]
all_vtk_box_corners = []
predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8])
for i in range(len(predictions_boxes_8c)):
box_8c = predictions_boxes_8c[i, :, :]
vtk_box_corners = VtkBox8c()
vtk_box_corners.set_objects(box_8c)
all_vtk_box_corners.append(vtk_box_corners)
##############################
# Ground Truth
##############################
if dataset.has_labels:
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset,
img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
img_idx = int(sample_name)
points, point_colours = demo_utils.get_filtered_pc_and_colours(
dataset, image, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for ground truth
vtk_easy_gt_boxes, vtk_medium_gt_boxes, \
vtk_hard_gt_boxes, vtk_all_gt_boxes = \
demo_utils.create_gt_vtk_boxes(easy_gt_objs,
medium_gt_objs,
hard_gt_objs,
all_gt_objs,
show_orientations)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_box_actors = vtk.vtkAssembly()
# Create VtkBoxes for prediction boxes
for i in range(len(all_vtk_box_corners)):
# Adding labels, slows down rendering
# vtk_renderer.AddActor(all_vtk_box_corners[i].
# vtk_text_labels.vtk_actor)
vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_box_actors)
vtk_renderer.AddActor(axes)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_box_actors, vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor
]))
vtk_render_window_interactor.Start()
def main():
"""
Visualization of the mini batch anchors for RpnModel training.
Keys:
F1: Toggle mini batch anchors
F2: Toggle positive/negative proposal anchors
F3: Toggle easy ground truth objects (Green)
F4: Toggle medium ground truth objects (Orange)
F5: Toggle hard ground truth objects (Red)
F6: Toggle all ground truth objects (default off)
F7: Toggle ground-plane
"""
anchor_colour_scheme = {
"Car": (255, 0, 0), # Red
"Pedestrian": (255, 150, 50), # Orange
"Cyclist": (150, 50, 100), # Purple
"DontCare": (255, 255, 255), # White
"Anchor": (150, 150, 150), # Gray
"Positive": (0, 255, 255), # Teal
"Negative": (255, 0, 255) # Bright Purple
}
##############################
# Options
##############################
show_orientations = True
# Classes name
config_name = 'car'
# config_name = 'ped'
# config_name = 'cyc'
# config_name = 'ppl'
# # # Random sample # # #
sample_name = None
# Small cars
# sample_name = '000008'
# sample_name = '000639'
# # # Cars # # #
# sample_name = "000001"
# sample_name = "000050"
# sample_name = "000112"
# sample_name = "000169"
# sample_name = "000191"
# # # People # # #
# sample_name = '000000'
# val_half
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# sample_name = "000000"
# sample_name = "000011"
# sample_name = "000015"
# sample_name = "000028"
# sample_name = "000035"
# sample_name = "000134"
# sample_name = "000167"
# sample_name = '000379'
# sample_name = '000381'
# sample_name = '000397'
# sample_name = '000398'
# sample_name = '000401'
# sample_name = '000407'
# sample_name = '000486'
# sample_name = '000509'
# # Cyclists # # #
# sample_name = '000122'
# sample_name = '000448'
# # # Multiple classes # # #
# sample_name = "000764"
##############################
# End of Options
##############################
# Dataset config
dataset_config_path = mlod.top_dir() + \
'/demos/configs/mb_rpn_{}.config'.format(config_name)
# Create Dataset
dataset = DatasetBuilder.load_dataset_from_config(
dataset_config_path)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx].name
anchor_strides = dataset.kitti_utils.anchor_strides
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# KittiUtils class
dataset_utils = dataset.kitti_utils
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = point_cloud.T
point_colours = vis_utils.project_img_to_point_cloud(points, image,
dataset.calib_dir,
img_idx)
clusters, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Read mini batch info
anchors_info = dataset_utils.get_anchors_info(
dataset.classes_name, anchor_strides, sample_name)
if not anchors_info:
# Exit early if anchors_info is empty
print("Anchors info is empty, please try a different sample")
return
# Generate anchors for all classes
all_anchor_boxes_3d = []
for class_idx in range(len(dataset.classes)):
anchor_boxes_3d = anchor_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=clusters[class_idx],
anchor_stride=anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d.extend(anchor_boxes_3d)
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
# Use anchors info
indices, ious, offsets, classes = anchors_info
# Get non empty anchors from the indices
anchor_boxes_3d = all_anchor_boxes_3d[indices]
# Sample an RPN mini batch from the non empty anchors
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
mb_mask_tf, _ = mini_batch_utils.sample_rpn_mini_batch(ious)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
mb_mask = sess.run(mb_mask_tf)
mb_anchor_boxes_3d = anchor_boxes_3d[mb_mask]
mb_anchor_ious = ious[mb_mask]
# ObjectLabel list that hold all boxes to visualize
obj_list = []
num_positives = 0
# Convert the mini_batch anchors to object list
mini_batch_size = mini_batch_utils.rpn_mini_batch_size
for i in range(mini_batch_size):
if mb_anchor_ious[i] > mini_batch_utils.rpn_pos_iou_range[0]:
obj_type = "Positive"
num_positives += 1
else:
obj_type = "Negative"
obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i],
obj_type)
obj_list.append(obj)
print('Num positives', num_positives)
# Convert all non-empty anchors to object list
non_empty_anchor_objs = \
[box_3d_encoder.box_3d_to_object_label(
anchor_box_3d, obj_type='Anchor')
for anchor_box_3d in anchor_boxes_3d]
##############################
# Ground Truth
##############################
if dataset.has_labels:
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = demo_utils.get_gts_based_on_difficulty(
dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
# Visualize 2D image
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for mini batch anchors
vtk_pos_anchor_boxes = VtkBoxes()
vtk_pos_anchor_boxes.set_objects(obj_list, anchor_colour_scheme)
# VtkBoxes for non empty anchors
vtk_non_empty_anchors = VtkBoxes()
vtk_non_empty_anchors.set_objects(non_empty_anchor_objs,
anchor_colour_scheme)
vtk_non_empty_anchors.set_line_width(0.1)
# Create VtkBoxes for ground truth
vtk_easy_gt_boxes, vtk_medium_gt_boxes, \
vtk_hard_gt_boxes, vtk_all_gt_boxes = \
demo_utils.create_gt_vtk_boxes(easy_gt_objs,
medium_gt_objs,
hard_gt_objs,
all_gt_objs,
show_orientations)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2)
vtk_ground_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, dataset.kitti_utils.bev_extents)
# vtk_voxel_grid = VtkVoxelGrid()
# vtk_voxel_grid.set_voxels(vx_grid)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
# vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_non_empty_anchors.vtk_actor)
vtk_renderer.AddActor(vtk_pos_anchor_boxes.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
mb_iou_thresholds = np.round(
[mini_batch_utils.rpn_neg_iou_range[1],
mini_batch_utils.rpn_pos_iou_range[0]], 3)
vtk_render_window.SetWindowName(
'Sample {} RPN Mini Batch {}/{}, '
'Num Positives {}'.format(
sample_name,
mb_iou_thresholds[0],
mb_iou_thresholds[1],
num_positives))
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_non_empty_anchors.vtk_actor,
vtk_pos_anchor_boxes.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'mlod_exp_example'
# data_split = 'val'
data_split = 'val_half'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
# Checkpoints below this are skipped
min_step = 20000
# Object Type
obj_type = 'obj'
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
proposals_root_dir = predictions_root_dir + \
'/proposals_and_scores/' + dataset.data_split
print('Converting proposals from', proposals_root_dir)
if not global_steps:
global_steps = os.listdir(proposals_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = proposals_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_proposals_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_proposals_3d_dir = predictions_root_dir + \
'/kitti_proposals_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_proposals_3d_dir):
os.makedirs(kitti_proposals_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting proposals from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('Proposals saved to:', kitti_proposals_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_proposals_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# Swap l, w for predictions where w > l
swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
fixed_predictions = np.copy(all_predictions)
fixed_predictions[swapped_indices,
3] = all_predictions[swapped_indices, 4]
fixed_predictions[swapped_indices,
4] = all_predictions[swapped_indices, 3]
fixed_predictions[swapped_indices, 6] = np.pi / 2
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = fixed_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(
dataset.calib_dir, img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
stereo_calib_p2,
truncate=True,
image_size=image.size,
discard_before_truncation=False)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Truncation and Occlusion are always empty (see below)
# Alpha (Not computed)
kitti_predictions[:, 3] = -10 * np.ones(
(len(kitti_predictions)), dtype=np.int32)
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones(
(len(kitti_predictions), 2), dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones(
(len(kitti_predictions), 3), dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones(
(len(kitti_predictions), 3), dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones(
(len(kitti_predictions), 1), dtype=np.int32)
# Create Type Array
obj_types = [obj_type for i in range(len(kitti_predictions))]
# Stack 2D predictions text
kitti_text_2d = np.column_stack([
obj_types, kitti_empty_1, kitti_predictions[:, 3:8],
kitti_empty_2, kitti_empty_3, kitti_empty_4,
kitti_predictions[:, 15]
])
# Stack 3D predictions text
kitti_text_3d = np.column_stack(
[obj_types, kitti_empty_1, kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path,
kitti_text_2d,
newline='\r\n',
fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path,
kitti_text_3d,
newline='\r\n',
fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def main():
"""
Visualization for comparison of anchor filtering with
2D vs 3D integral images
Keys:
F1: Toggle 3D integral image filtered anchors
F2: Toggle 2D integral image filtered anchors
F3: Toggle 2D integral image empty anchors
"""
anchor_2d_colour_scheme = {"Anchor": (0, 0, 255)} # Blue
anchor_3d_colour_scheme = {"Anchor": (0, 255, 0)} # Green
anchor_unfiltered_colour_scheme = {"Anchor": (255, 0, 255)} # Purple
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_TRAINVAL)
sample_name = "000001"
img_idx = int(sample_name)
print("Showing anchors for sample {}".format(sample_name))
# Options
# These clusters are from the trainval set and give more 2D anchors than 3D
clusters = np.array([[3.55, 1.835, 1.525], [4.173, 1.69, 1.49]])
anchor_stride = [3.0, 3.0]
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
area_extents = np.array([[-40, 40], [-5, 3], [0, 70]])
anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Generate anchors
start_time = time.time()
anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents,
anchor_3d_sizes=clusters,
anchor_stride=anchor_stride,
ground_plane=ground_plane)
end_time = time.time()
print("Anchors generated in {} s".format(end_time - start_time))
# Get point cloud
point_cloud = obj_utils.get_stereo_point_cloud(img_idx, dataset.calib_dir,
dataset.disp_dir)
ground_offset_dist = 0.2
offset_dist = 2.0
# Filter points within certain xyz range and offset from ground plane
# Filter points within 0.2m of the road plane
slice_filter = dataset.kitti_utils.create_slice_filter(point_cloud,
area_extents,
ground_plane,
ground_offset_dist,
offset_dist)
points = np.array(point_cloud).T
points = points[slice_filter]
anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
# Create 2D voxel grid
vx_grid_2d = voxel_grid_2d.VoxelGrid2D()
vx_grid_2d.voxelize_2d(points, 0.1, area_extents)
# Create 3D voxel grid
vx_grid_3d = voxel_grid.VoxelGrid()
vx_grid_3d.voxelize(points, 0.1, area_extents)
# Filter the boxes here!
start_time = time.time()
empty_filter_2d = anchor_filter.get_empty_anchor_filter_2d(
anchors=anchors,
voxel_grid_2d=vx_grid_2d,
density_threshold=1)
anchors_2d = anchor_boxes_3d[empty_filter_2d]
end_time = time.time()
print("2D Anchors filtered in {} s".format(end_time - start_time))
print("Number of 2D anchors remaining: %d" % (anchors_2d.shape[0]))
unfiltered_anchors_2d = anchor_boxes_3d[np.logical_not(empty_filter_2d)]
# 3D filtering
start_time = time.time()
empty_filter_3d = anchor_filter.get_empty_anchor_filter(
anchors=anchors,
voxel_grid_3d=vx_grid_3d,
density_threshold=1)
anchor_boxes_3d = anchor_boxes_3d[empty_filter_3d]
end_time = time.time()
print("3D Anchors filtered in {} s".format(end_time - start_time))
print("Number of 3D anchors remaining: %d" % (anchor_boxes_3d.shape[0]))
anchor_2d_objects = []
for anchor_idx in range(len(anchors_2d)):
anchor = anchors_2d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor')
# Append to a list for visualization in VTK later
anchor_2d_objects.append(obj_label)
anchor_3d_objects = []
for anchor_idx in range(len(anchor_boxes_3d)):
anchor = anchor_boxes_3d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor')
# Append to a list for visualization in VTK later
anchor_3d_objects.append(obj_label)
unfiltered_anchor_objects = []
for anchor_idx in range(len(unfiltered_anchors_2d)):
anchor = unfiltered_anchors_2d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor')
# Append to a list for visualization in VTK later
unfiltered_anchor_objects.append(obj_label)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for boxes
vtk_2d_boxes = VtkBoxes()
vtk_2d_boxes.set_objects(anchor_2d_objects, anchor_2d_colour_scheme)
vtk_3d_boxes = VtkBoxes()
vtk_3d_boxes.set_objects(anchor_3d_objects, anchor_3d_colour_scheme)
vtk_unfiltered_boxes = VtkBoxes()
vtk_unfiltered_boxes.set_objects(unfiltered_anchor_objects,
anchor_unfiltered_colour_scheme)
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(vx_grid_3d)
vtk_voxel_grid_2d = VtkVoxelGrid()
vtk_voxel_grid_2d.set_voxels(vx_grid_2d)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_2d_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_3d_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_unfiltered_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid_2d.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_2d_boxes.vtk_actor,
vtk_3d_boxes.vtk_actor,
vtk_unfiltered_boxes.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""
Displays the bird's eye view maps for a KITTI sample.
"""
##############################
# Options
##############################
# one of ['height_priors', 'slices']
bev_generator = 'slices'
height_priors_config = \
"""
height_priors {
ground_filter_offset: 0.2
offset_filter_distance: 2.0
std_dev_multiplier: 2.0
}
"""
slices_config = \
"""
slices {
height_lo: -0.2
height_hi: 2.3
num_slices: 5
}
"""
# Use None for a random image
img_idx = None
# img_idx = 142
# img_idx = 191
show_ground_truth = True # Whether to overlay ground_truth boxes
point_cloud_source = 'lidar'
##############################
# End of Options
##############################
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
# Overwrite bev_generator
if bev_generator == 'height_priors':
text_format.Merge(height_priors_config,
dataset_config.kitti_utils_config.bev_generator)
elif bev_generator == 'slices':
text_format.Merge(slices_config,
dataset_config.kitti_utils_config.bev_generator)
else:
raise ValueError('Invalid bev_generator')
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
if img_idx is None:
img_idx = int(random.random() * dataset.num_samples)
sample_name = "{:06}".format(img_idx)
print('=== Showing BEV maps for image: {}.png ==='.format(sample_name))
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
kitti_utils = dataset.kitti_utils
point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx,
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
height_maps = np.array(bev_images.get("height_maps"))
density_map = np.array(bev_images.get("density_map"))
box_points, box_points_norm = [None, None]
if show_ground_truth:
# Get projected boxes
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_objs = obj_labels
label_boxes = []
for label in filtered_objs:
box = box_3d_encoder.object_label_to_box_3d(label)
label_boxes.append(box)
label_boxes = np.array(label_boxes)
box_points, box_points_norm = box_3d_projector.project_to_bev(
label_boxes, [[-40, 40], [0, 70]])
rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16)
img_x_start = 60
img_y_start = 330
img_x = img_x_start
img_y = img_y_start
img_w = 400
img_h = 350
img_titlebar_h = 20
# Show images
vis_utils.cv2_show_image("Image",
image,
size_wh=rgb_img_size,
location_xy=(img_x, 0))
# Height maps
for map_idx in range(len(height_maps)):
height_map = height_maps[map_idx]
height_map = draw_boxes(height_map, box_points_norm)
vis_utils.cv2_show_image("Height Map {}".format(map_idx),
height_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
img_x += img_w
# Wrap around
if (img_x + img_w) > 1920:
img_x = img_x_start
img_y += img_h + img_titlebar_h
# Density map
density_map = draw_boxes(density_map, box_points_norm)
vis_utils.cv2_show_image("Density Map",
density_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
cv2.waitKey()
def main():
# Create Dataset
dataset_config_path = mlod.root_dir() + \
'/configs/mb_preprocessing/rpn_cars.config'
dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path)
# Random sample
sample_name = '000169'
anchor_strides = dataset.kitti_utils.anchor_strides
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# KittiUtils class
dataset_utils = dataset.kitti_utils
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
# Grab ground truth
ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx)
ground_truth_list = dataset_utils.filter_labels(ground_truth_list)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
##############################
# Flip sample info
##############################
start_time = time.time()
flipped_image = kitti_aug.flip_image(image)
flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud)
flipped_gt_list = [kitti_aug.flip_label_in_3d_only(obj)
for obj in ground_truth_list]
flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane)
flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2(
stereo_calib_p2, image_shape)
flipped_points = flipped_point_cloud.T
print('flip sample', time.time() - start_time)
##############################
# Generate anchors
##############################
clusters, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Read mini batch info
anchors_info = dataset_utils.get_anchors_info(
dataset.classes_name,
anchor_strides,
sample_name)
all_anchor_boxes_3d = []
all_ious = []
for class_idx in range(len(dataset.classes)):
anchor_boxes_3d = anchor_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=clusters[class_idx],
anchor_stride=anchor_strides[class_idx],
ground_plane=ground_plane)
if anchors_info:
indices, ious, offsets, classes = anchors_info
# Get non empty anchors from the indices
non_empty_anchor_boxes_3d = anchor_boxes_3d[indices]
all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d)
all_ious.extend(ious)
if not len(all_anchor_boxes_3d) > 0:
# Exit early if anchors_info is empty
print("No anchors, Please try a different sample")
return
# Convert to ndarrays
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
all_ious = np.asarray(all_ious)
##############################
# Flip anchors
##############################
start_time = time.time()
flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d,
flip_ry=False)
print('flip anchors', time.time() - start_time)
# Overwrite with flipped things
all_anchor_boxes_3d = flipped_anchor_boxes_3d
points = flipped_points
ground_truth_list = flipped_gt_list
ground_plane = flipped_ground_plane
def main():
"""Plots detection errors for xyz, lwh, ry, and shows 3D IoU with
ground truth boxes
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL_HALF,
use_defaults=True)
difficulty = 2
# Full path to kitti predictions
# (e.g. '.../data/outputs/mlod_exp_example/predictions/'
# 'kitti_predictions_3d/val/0.1/100000/data'
predictions_data_path = 'path_to_detections/data'
# Loop through split and save ious and errors
all_3d_ious = []
all_errors = []
for sample_idx in range(dataset.num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx + 1,
dataset.num_samples))
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
# Get filtered ground truth
all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx)
all_gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs,
difficulty=difficulty)
pred_objs = obj_utils.read_labels(predictions_data_path, img_idx)
##############################
# Check IoUs
##############################
if len(all_gt_objs) > 0 and \
pred_objs is not None and len(pred_objs) > 0:
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in pred_objs
]
# Convert to iou format
gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
all_gt_boxes_3d)
pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
pred_boxes_3d)
max_ious_3d = np.zeros(len(all_gt_objs))
max_iou_pred_indices = -np.ones(len(all_gt_objs))
for gt_obj_idx in range(len(all_gt_objs)):
gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx]
ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt,
pred_objs_iou_fmt)
max_iou_3d = np.amax(ious_3d)
max_ious_3d[gt_obj_idx] = max_iou_3d
if max_iou_3d > 0.0:
max_iou_pred_indices[gt_obj_idx] = np.argmax(ious_3d)
for gt_obj_idx in range(len(all_gt_objs)):
max_iou_pred_idx = int(max_iou_pred_indices[gt_obj_idx])
if max_iou_pred_idx >= 0:
error = all_gt_boxes_3d[gt_obj_idx] - \
pred_boxes_3d[max_iou_pred_idx]
all_errors.append(error)
all_3d_ious.extend(max_ious_3d)
print('Done')
all_errors = np.asarray(all_errors)
# Plot Data Histograms
f, ax_arr = plt.subplots(3, 3)
xyzlwh_bins = 51
ry_bins = 31
iou_bins = 51
# xyz
ax_arr[0, 0].hist(all_errors[:, 0],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[0, 1].hist(all_errors[:, 1],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[0, 2].hist(all_errors[:, 2],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
# lwh
ax_arr[1, 0].hist(all_errors[:, 3],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[1, 1].hist(all_errors[:, 4],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[1, 2].hist(all_errors[:, 5],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
# orientation
ax_arr[2, 0].hist(all_errors[:, 6], ry_bins, facecolor='green', alpha=0.75)
# iou
ax_arr[2, 2].hist(all_3d_ious, iou_bins, facecolor='green', alpha=0.75)
plt.show()
