def main():
"""
Visualization of the mini batch anchors for RpnModel training.
Keys:
F1: Toggle mini batch anchors
F2: Toggle positive/negative proposal anchors
F3: Toggle easy ground truth objects (Green)
F4: Toggle medium ground truth objects (Orange)
F5: Toggle hard ground truth objects (Red)
F6: Toggle all ground truth objects (default off)
F7: Toggle ground-plane
"""
anchor_colour_scheme = {
"Car": (255, 0, 0), # Red
"Pedestrian": (255, 150, 50), # Orange
"Cyclist": (150, 50, 100), # Purple
"DontCare": (255, 255, 255), # White
"Anchor": (150, 150, 150), # Gray
"Positive": (0, 255, 255), # Teal
"Negative": (255, 0, 255) # Bright Purple
}
##############################
# Options
##############################
show_orientations = True
# Classes name
config_name = 'car'
# config_name = 'ped'
# config_name = 'cyc'
# config_name = 'ppl'
# # # Random sample # # #
sample_name = None
# Small cars
# sample_name = '000008'
# sample_name = '000639'
# # # Cars # # #
# sample_name = "000001"
# sample_name = "000050"
# sample_name = "000112"
# sample_name = "000169"
# sample_name = "000191"
# # # People # # #
# sample_name = '000000'
# val_half
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# sample_name = "000000"
# sample_name = "000011"
# sample_name = "000015"
# sample_name = "000028"
# sample_name = "000035"
# sample_name = "000134"
# sample_name = "000167"
# sample_name = '000379'
# sample_name = '000381'
# sample_name = '000397'
# sample_name = '000398'
# sample_name = '000401'
# sample_name = '000407'
# sample_name = '000486'
# sample_name = '000509'
# # Cyclists # # #
# sample_name = '000122'
# sample_name = '000448'
# # # Multiple classes # # #
# sample_name = "000764"
##############################
# End of Options
##############################
# Dataset config
dataset_config_path = mlod.top_dir() + \
'/demos/configs/mb_rpn_{}.config'.format(config_name)
# Create Dataset
dataset = DatasetBuilder.load_dataset_from_config(
dataset_config_path)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx].name
anchor_strides = dataset.kitti_utils.anchor_strides
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# KittiUtils class
dataset_utils = dataset.kitti_utils
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = point_cloud.T
point_colours = vis_utils.project_img_to_point_cloud(points, image,
dataset.calib_dir,
img_idx)
clusters, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Read mini batch info
anchors_info = dataset_utils.get_anchors_info(
dataset.classes_name, anchor_strides, sample_name)
if not anchors_info:
# Exit early if anchors_info is empty
print("Anchors info is empty, please try a different sample")
return
# Generate anchors for all classes
all_anchor_boxes_3d = []
for class_idx in range(len(dataset.classes)):
anchor_boxes_3d = anchor_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=clusters[class_idx],
anchor_stride=anchor_strides[class_idx],
ground_plane=ground_plane)
all_anchor_boxes_3d.extend(anchor_boxes_3d)
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
# Use anchors info
indices, ious, offsets, classes = anchors_info
# Get non empty anchors from the indices
anchor_boxes_3d = all_anchor_boxes_3d[indices]
# Sample an RPN mini batch from the non empty anchors
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
mb_mask_tf, _ = mini_batch_utils.sample_rpn_mini_batch(ious)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
mb_mask = sess.run(mb_mask_tf)
mb_anchor_boxes_3d = anchor_boxes_3d[mb_mask]
mb_anchor_ious = ious[mb_mask]
# ObjectLabel list that hold all boxes to visualize
obj_list = []
num_positives = 0
# Convert the mini_batch anchors to object list
mini_batch_size = mini_batch_utils.rpn_mini_batch_size
for i in range(mini_batch_size):
if mb_anchor_ious[i] > mini_batch_utils.rpn_pos_iou_range[0]:
obj_type = "Positive"
num_positives += 1
else:
obj_type = "Negative"
obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i],
obj_type)
obj_list.append(obj)
print('Num positives', num_positives)
# Convert all non-empty anchors to object list
non_empty_anchor_objs = \
[box_3d_encoder.box_3d_to_object_label(
anchor_box_3d, obj_type='Anchor')
for anchor_box_3d in anchor_boxes_3d]
##############################
# Ground Truth
##############################
if dataset.has_labels:
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = demo_utils.get_gts_based_on_difficulty(
dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
# Visualize 2D image
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for mini batch anchors
vtk_pos_anchor_boxes = VtkBoxes()
vtk_pos_anchor_boxes.set_objects(obj_list, anchor_colour_scheme)
# VtkBoxes for non empty anchors
vtk_non_empty_anchors = VtkBoxes()
vtk_non_empty_anchors.set_objects(non_empty_anchor_objs,
anchor_colour_scheme)
vtk_non_empty_anchors.set_line_width(0.1)
# Create VtkBoxes for ground truth
vtk_easy_gt_boxes, vtk_medium_gt_boxes, \
vtk_hard_gt_boxes, vtk_all_gt_boxes = \
demo_utils.create_gt_vtk_boxes(easy_gt_objs,
medium_gt_objs,
hard_gt_objs,
all_gt_objs,
show_orientations)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2)
vtk_ground_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, dataset.kitti_utils.bev_extents)
# vtk_voxel_grid = VtkVoxelGrid()
# vtk_voxel_grid.set_voxels(vx_grid)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
# vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_non_empty_anchors.vtk_actor)
vtk_renderer.AddActor(vtk_pos_anchor_boxes.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
mb_iou_thresholds = np.round(
[mini_batch_utils.rpn_neg_iou_range[1],
mini_batch_utils.rpn_pos_iou_range[0]], 3)
vtk_render_window.SetWindowName(
'Sample {} RPN Mini Batch {}/{}, '
'Num Positives {}'.format(
sample_name,
mb_iou_thresholds[0],
mb_iou_thresholds[1],
num_positives))
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_non_empty_anchors.vtk_actor,
vtk_pos_anchor_boxes.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""This demo shows RPN proposals and MLOD predictions in the
3D point cloud.
Keys:
F1: Toggle proposals
F2: Toggle predictions
F3: Toggle 3D voxel grid
F4: Toggle point cloud
F5: Toggle easy ground truth objects (Green)
F6: Toggle medium ground truth objects (Orange)
F7: Toggle hard ground truth objects (Red)
F8: Toggle all ground truth objects (default off)
F9: Toggle ground slice filter (default off)
F10: Toggle offset slice filter (default off)
"""
##############################
# Options
##############################
rpn_score_threshold = 0.1
mlod_score_threshold = 0.1
proposals_line_width = 1.0
predictions_line_width = 3.0
show_orientations = True
point_cloud_source = 'lidar'
# Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>)
config_dir = None
checkpoint_name = 'mlod_fpn_people_n_m'
global_step = 135000 # Latest checkpoint
#data_split = 'val_half'
#data_split = 'val'
data_split = 'test'
# Show 3D iou text
draw_ious_3d = False
sample_name = '000031'
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# val split
# sample_name = '000181'
# sample_name = '000751'
# sample_name = '000843'
# sample_name = '000944'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# val split
# sample_name = '000015'
# sample_name = '000048'
# sample_name = '000058'
# sample_name = '000076' # Medium, few ped, 1 cyc
# sample_name = '000108'
# sample_name = '000118'
# sample_name = '000145'
# sample_name = '000153'
# sample_name = '000186'
# sample_name = '000195'
# sample_name = '000199'
# sample_name = '000397'
# sample_name = '004425'
# sample_name = '004474' # Hard, many ped, 1 cyc
# sample_name = '004657' # Hard, Few cycl, few ped
# sample_name = '006071'
# sample_name = '006828' # Hard, Few cycl, few ped
# sample_name = '006908' # Hard, Few cycl, few ped
# sample_name = '007412'
# sample_name = '007318' # Hard, Few cycl, few ped
##############################
# End of Options
##############################
if data_split == 'test':
draw_ious_3d = False
if config_dir is None:
config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_names[sample_idx]
##############################
# Setup Paths
##############################
img_idx = int(sample_name)
# Text files directory
proposals_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_predictions_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
# Output images directory
img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/predictions/images_3d/{}/{}/{}'.format(dataset.data_split,
global_step,
rpn_score_threshold)
if not os.path.exists(img_out_dir):
os.makedirs(img_out_dir)
##############################
# Proposals
##############################
# Load proposals from files
proposals_and_scores = np.loadtxt(
proposals_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
proposals = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
rpn_score_mask = proposal_scores > rpn_score_threshold
proposals = proposals[rpn_score_mask]
proposal_scores = proposal_scores[rpn_score_mask]
print('Proposals:', len(proposal_scores), proposal_scores)
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposals]
##############################
# Predictions
##############################
# Load proposals from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32)
mlod_score_mask = prediction_scores >= mlod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask]
prediction_scores = prediction_scores[mlod_score_mask]
print('Predictions: ', len(prediction_scores), prediction_scores)
final_predictions = np.copy(prediction_boxes_3d)
# # Swap l, w for predictions where w > l
# swapped_indices = predictions[:, 4] > predictions[:, 3]
# final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4]
# final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3]
prediction_objs = []
dataset.classes = ['Pedestrian', 'Cyclist', 'Car']
for pred_idx in range(len(final_predictions)):
prediction_box_3d = final_predictions[pred_idx]
prediction_type = dataset.classes[prediction_types[pred_idx]]
prediction_obj = box_3d_encoder.box_3d_to_object_label(
prediction_box_3d, obj_type=prediction_type)
prediction_objs.append(prediction_obj)
##############################
# Ground Truth
##############################
dataset.has_labels = False
if dataset.has_labels:
# Get ground truth labels
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# 3D IoU
##############################
if draw_ious_3d:
# Convert to box_3d
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in prediction_objs
]
max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d,
pred_boxes_3d)
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source,
img_idx,
image_shape=image.shape)
point_cloud = np.asarray(point_cloud)
# Filter point cloud to extents
area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]])
bev_extents = area_extents[[0, 2]]
points = point_cloud.T
point_filter = obj_utils.get_point_filter(point_cloud, area_extents)
points = points[point_filter]
point_colours = vis_utils.project_img_to_point_cloud(
points, image, dataset.calib_dir, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
# Ground plane
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
vtk_axes = vtk.vtkAxesActor()
vtk_axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for proposal boxes
vtk_proposal_boxes = VtkBoxes()
vtk_proposal_boxes.set_line_width(proposals_line_width)
vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS)
# Create VtkBoxes for prediction boxes
vtk_prediction_boxes = VtkPyramidBoxes()
vtk_prediction_boxes.set_line_width(predictions_line_width)
vtk_prediction_boxes.set_objects(prediction_objs,
COLOUR_SCHEME_PREDICTIONS,
show_orientations)
# Create VtkBoxes for ground truth
vtk_hard_gt_boxes = VtkBoxes()
vtk_medium_gt_boxes = VtkBoxes()
vtk_easy_gt_boxes = VtkBoxes()
vtk_all_gt_boxes = VtkBoxes()
vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations)
# Create VtkTextLabels for 3D ious
vtk_text_labels = VtkTextLabels()
if draw_ious_3d and len(all_gt_boxes_3d) > 0:
gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3]
vtk_text_labels.set_text_labels(
gt_positions_3d,
['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d])
# Create VtkGroundPlane
vtk_ground_plane = VtkGroundPlane()
vtk_slice_bot_plane = VtkGroundPlane()
vtk_slice_top_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, bev_extents)
vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents)
vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_text_labels.vtk_actor)
# Add ground plane and slice planes
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor)
#vtk_renderer.AddActor(vtk_axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3)
vtk_proposal_boxes.vtk_actor.SetVisibility(0)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.SetVisibility(0)
vtk_slice_bot_plane.vtk_actor.SetVisibility(0)
vtk_slice_top_plane.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(3.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
custom_interactor = vis_utils.CameraInfoInteractorStyle([
vtk_proposal_boxes.vtk_actor,
vtk_prediction_boxes.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor,
vtk_slice_bot_plane.vtk_actor,
vtk_slice_top_plane.vtk_actor,
vtk_text_labels.vtk_actor,
])
vtk_render_window_interactor.SetInteractorStyle(custom_interactor)
# Render in VTK
vtk_render_window.Render()
# Take a screenshot
window_to_image_filter = vtk.vtkWindowToImageFilter()
window_to_image_filter.SetInput(vtk_render_window)
window_to_image_filter.Update()
png_writer = vtk.vtkPNGWriter()
file_name = img_out_dir + "/{}.png".format(sample_name)
png_writer.SetFileName(file_name)
png_writer.SetInputData(window_to_image_filter.GetOutput())
png_writer.Write()
print('Screenshot saved to ', file_name)
vtk_render_window_interactor.Start() # Blocking
def main():
"""Shows a flipped sample in 3D
"""
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL)
##############################
# Options
##############################
# sample_name = "000191"
sample_name = "000104"
img_idx = int(sample_name)
print("Showing anchors for sample {}".format(sample_name))
##############################
# Load Sample Data
##############################
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# Get point cloud
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = np.array(point_cloud).T
# Ground truth
gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
# Filter ground truth
gt_labels = dataset.kitti_utils.filter_labels(gt_labels)
##############################
# Flip stuff
##############################
image_flipped = np.fliplr(image)
# Flip ground plane coeff (x)
ground_plane_flipped = np.copy(ground_plane)
ground_plane_flipped[0] = -ground_plane_flipped[0]
# Flip 3D points
points_flipped = kitti_aug.flip_points(points)
# Get point cloud colours
point_colours_flipped = project_flipped_img_to_point_cloud(
points_flipped, image_flipped, dataset.calib_dir, img_idx)
# Flip ground truth boxes
gt_labels_flipped = [
kitti_aug.flip_label_in_3d_only(obj) for obj in gt_labels
]
##############################
# VTK Visualization
##############################
# Axes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Point cloud
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points_flipped,
point_colours=point_colours_flipped)
# # Ground Truth Boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(gt_labels_flipped,
VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations=True)
# Renderer
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Add Actors to Rendered
vtk_renderer.AddActor(axes)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_point_cloud.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""This demo visualizes box 8C format predicted by MLOD, before
getting converted to Box 3D.
Keys:
F1: Toggle predictions
F2: Toggle easy ground truth objects (Green)
F3: Toggle medium ground truth objects (Orange)
F4: Toggle hard ground truth objects (Red)
F5: Toggle all ground truth objects (default off)
F6: Toggle 3D voxel grid
F7: Toggle point cloud
"""
##############################
# Options
##############################
mlod_score_threshold = 0.1
show_orientations = True
checkpoint_name = 'mlod_exp_8c'
global_step = None
sample_name = None
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
##############################
# Setup Paths
##############################
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001'
sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx]
img_idx = int(sample_name)
# Text files directory
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_boxes_8c_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(predictions_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
##############################
# predictions
##############################
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
predictions_boxes_8c = predictions_and_scores[:, 0:24]
prediction_scores = predictions_and_scores[:, 24]
score_mask = prediction_scores >= mlod_score_threshold
predictions_boxes_8c = predictions_boxes_8c[score_mask]
all_vtk_box_corners = []
predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8])
for i in range(len(predictions_boxes_8c)):
box_8c = predictions_boxes_8c[i, :, :]
vtk_box_corners = VtkBox8c()
vtk_box_corners.set_objects(box_8c)
all_vtk_box_corners.append(vtk_box_corners)
##############################
# Ground Truth
##############################
if dataset.has_labels:
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset,
img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
img_idx = int(sample_name)
points, point_colours = demo_utils.get_filtered_pc_and_colours(
dataset, image, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for ground truth
vtk_easy_gt_boxes, vtk_medium_gt_boxes, \
vtk_hard_gt_boxes, vtk_all_gt_boxes = \
demo_utils.create_gt_vtk_boxes(easy_gt_objs,
medium_gt_objs,
hard_gt_objs,
all_gt_objs,
show_orientations)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_box_actors = vtk.vtkAssembly()
# Create VtkBoxes for prediction boxes
for i in range(len(all_vtk_box_corners)):
# Adding labels, slows down rendering
# vtk_renderer.AddActor(all_vtk_box_corners[i].
# vtk_text_labels.vtk_actor)
vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_box_actors)
vtk_renderer.AddActor(axes)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_box_actors, vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor
]))
vtk_render_window_interactor.Start()
def main():
"""
This demo shows example mini batch info for full MlodModel training.
This includes ground truth, ortho rotated ground truth,
negative proposal anchors, positive proposal anchors, and a sampled
mini batch.
The 2D iou can be modified to show the effect of changing the iou
threshold for mini batch sampling.
In order to let this demo run without training an RPN, the proposals
shown are being read from a text file.
Keys:
F1: Toggle ground truth
F2: Toggle ortho rotated ground truth
F3: Toggle negative proposal anchors
F4: Toggle positive proposal anchors
F5: Toggle mini batch anchors
"""
##############################
# Options
##############################
# Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>)
config_dir = None
# checkpoint_name = None
checkpoint_name = 'mlod_exp_example'
data_split = 'val_half'
# global_step = None
global_step = 100000
# # # Cars # # #
# sample_name = "000050"
sample_name = "000104"
# sample_name = "000764"
# # # People # # #
# val_half
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
mini_batch_size = 512
neg_proposal_2d_iou_hi = 0.6
pos_proposal_2d_iou_lo = 0.65
bkg_proposals_line_width = 0.5
neg_proposals_line_width = 0.5
mid_proposals_line_width = 0.5
pos_proposals_line_width = 1.0
##############################
# End of Options
##############################
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
# Read proposals from file
if checkpoint_name is None:
# Use VAL Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL)
# Load demo proposals
proposals_and_scores_dir = mlod.top_dir() + \
'/demos/data/predictions/' + checkpoint_name + \
'/proposals_and_scores/' + dataset.data_split
else:
if config_dir is None:
config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
mini_batch_utils.mlod_neg_iou_range[1] = neg_proposal_2d_iou_hi
mini_batch_utils.mlod_pos_iou_range[0] = pos_proposal_2d_iou_lo
# Load proposals from outputs folder
proposals_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + \
'/predictions/proposals_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
proposals_and_scores = np.loadtxt(
proposals_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_anchors = box_3d_encoder.box_3d_to_anchor(proposal_boxes_3d)
# Get filtered ground truth
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_objs = dataset.kitti_utils.filter_labels(obj_labels)
# Convert ground truth to anchors
gt_boxes_3d = np.asarray([
box_3d_encoder.object_label_to_box_3d(obj_label)
for obj_label in filtered_objs
])
gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d,
ortho_rotate=True)
# Ortho rotate ground truth
gt_ortho_boxes_3d = box_3d_encoder.anchors_to_box_3d(gt_anchors)
gt_ortho_objs = [
box_3d_encoder.box_3d_to_object_label(box_3d, obj_type='OrthoGt')
for box_3d in gt_ortho_boxes_3d
]
# Project gt and anchors into BEV
gt_bev_anchors, _ = \
anchor_projector.project_to_bev(gt_anchors,
dataset.kitti_utils.bev_extents)
bev_anchors, _ = \
anchor_projector.project_to_bev(proposal_anchors,
dataset.kitti_utils.bev_extents)
# Reorder boxes into (y1, x1, y2, x2) order
gt_bev_anchors_tf_order = anchor_projector.reorder_projected_boxes(
gt_bev_anchors)
bev_anchors_tf_order = anchor_projector.reorder_projected_boxes(
bev_anchors)
# Convert to box_list format for iou calculation
gt_anchor_box_list = box_list.BoxList(
tf.cast(gt_bev_anchors_tf_order, tf.float32))
anchor_box_list = box_list.BoxList(
tf.cast(bev_anchors_tf_order, tf.float32))
# Get IoU for every anchor
tf_all_ious = box_list_ops.iou(gt_anchor_box_list, anchor_box_list)
valid_ious = True
# Make sure the calculated IoUs contain values. Since its a [N, M]
# tensor, if there are no gt's for instance, that entry will be zero.
if tf_all_ious.shape[0] == 0 or tf_all_ious.shape[1] == 0:
print('#################################################')
print('Warning: This sample does not contain valid IoUs')
print('#################################################')
valid_ious = False
if valid_ious:
tf_max_ious = tf.reduce_max(tf_all_ious, axis=0)
tf_max_iou_indices = tf.argmax(tf_all_ious, axis=0)
# Sample an RPN mini batch from the non empty anchors
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
# Overwrite mini batch size and sample a mini batch
mini_batch_utils.mlod_mini_batch_size = mini_batch_size
mb_mask_tf, _ = mini_batch_utils.sample_mlod_mini_batch(tf_max_ious)
# Create a session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
# Run the graph to calculate ious for every proposal and
# to get the mini batch mask
all_ious, max_ious, max_iou_indices = sess.run(
[tf_all_ious, tf_max_ious, tf_max_iou_indices])
mb_mask = sess.run(mb_mask_tf)
mb_anchors = proposal_anchors[mb_mask]
mb_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(mb_anchors)
mb_anchor_ious = max_ious[mb_mask]
else:
# We have no valid IoU's, so assume all IoUs are zeros
# and the mini-batch contains all the anchors since we cannot
# mask without IoUs.
max_ious = np.zeros(proposal_boxes_3d.shape[0])
mb_anchor_ious = max_ious
mb_anchors = proposal_anchors
mb_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(mb_anchors)
# Create list of positive/negative proposals based on iou
pos_proposal_objs = []
mid_proposal_objs = []
neg_proposal_objs = []
bkg_proposal_objs = []
for i in range(len(proposal_boxes_3d)):
box_3d = proposal_boxes_3d[i]
if max_ious[i] == 0.0:
# Background proposals
bkg_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='BackgroundProposal'))
elif max_ious[i] < neg_proposal_2d_iou_hi:
# Negative proposals
neg_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='NegativeProposal'))
elif max_ious[i] < pos_proposal_2d_iou_lo:
# Middle proposals (in between negative and positive)
mid_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='MiddleProposal'))
elif max_ious[i] <= 1.0:
# Positive proposals
pos_proposal_objs.append(
box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='PositiveProposal'))
else:
raise ValueError('Invalid IoU > 1.0')
print('{} bkg, {} neg, {} mid, {} pos proposals:'.format(
len(bkg_proposal_objs), len(neg_proposal_objs), len(mid_proposal_objs),
len(pos_proposal_objs)))
# Convert the mini_batch anchors to object list
mb_obj_list = []
for i in range(len(mb_anchor_ious)):
if valid_ious and (mb_anchor_ious[i] >
mini_batch_utils.mlod_pos_iou_range[0]):
obj_type = "Positive"
else:
obj_type = "Negative"
obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i],
obj_type)
mb_obj_list.append(obj)
# Point cloud
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
points, point_colours = demo_utils.get_filtered_pc_and_colours(
dataset, image, img_idx)
# Visualize from here
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
# VtkPointCloud
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# VtkBoxes for ground truth
vtk_gt_boxes = VtkBoxes()
vtk_gt_boxes.set_objects(filtered_objs, COLOUR_SCHEME)
# VtkBoxes for ortho ground truth
vtk_gt_ortho_boxes = VtkBoxes()
vtk_gt_ortho_boxes.set_objects(gt_ortho_objs, COLOUR_SCHEME)
# VtkBoxes for background proposals
vtk_bkg_proposal_boxes = VtkBoxes()
vtk_bkg_proposal_boxes.set_objects(bkg_proposal_objs, COLOUR_SCHEME)
vtk_bkg_proposal_boxes.set_line_width(bkg_proposals_line_width)
# VtkBoxes for negative proposals
vtk_neg_proposal_boxes = VtkBoxes()
vtk_neg_proposal_boxes.set_objects(neg_proposal_objs, COLOUR_SCHEME)
vtk_neg_proposal_boxes.set_line_width(neg_proposals_line_width)
# VtkBoxes for middle proposals
vtk_mid_proposal_boxes = VtkBoxes()
vtk_mid_proposal_boxes.set_objects(mid_proposal_objs, COLOUR_SCHEME)
vtk_mid_proposal_boxes.set_line_width(mid_proposals_line_width)
# VtkBoxes for positive proposals
vtk_pos_proposal_boxes = VtkBoxes()
vtk_pos_proposal_boxes.set_objects(pos_proposal_objs, COLOUR_SCHEME)
vtk_pos_proposal_boxes.set_line_width(pos_proposals_line_width)
# Create VtkBoxes for mini batch anchors
vtk_mb_boxes = VtkBoxes()
vtk_mb_boxes.set_objects(mb_obj_list, COLOUR_SCHEME)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Add actors
vtk_renderer.AddActor(axes)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_gt_ortho_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_bkg_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_neg_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_mid_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_pos_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_mb_boxes.vtk_actor)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("MLOD Mini Batch")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_gt_boxes.vtk_actor,
vtk_gt_ortho_boxes.vtk_actor,
vtk_bkg_proposal_boxes.vtk_actor,
vtk_neg_proposal_boxes.vtk_actor,
vtk_mid_proposal_boxes.vtk_actor,
vtk_pos_proposal_boxes.vtk_actor,
vtk_mb_boxes.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""
Visualization of anchor filtering using 3D integral images
"""
anchor_colour_scheme = {
"Car": (0, 255, 0), # Green
"Pedestrian": (255, 150, 50), # Orange
"Cyclist": (150, 50, 100), # Purple
"DontCare": (255, 0, 0), # Red
"Anchor": (0, 0, 255), # Blue
}
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL)
# Options
clusters, _ = dataset.get_cluster_info()
sample_name = "000000"
img_idx = int(sample_name)
anchor_stride = [0.5, 0.5]
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator(
anchor_3d_sizes=clusters, anchor_stride=anchor_stride)
area_extents = np.array([[-40, 40], [-5, 3], [0, 70]])
# Generate anchors in box_3d format
start_time = time.time()
anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents,
ground_plane=ground_plane)
end_time = time.time()
print("Anchors generated in {} s".format(end_time - start_time))
point_cloud = obj_utils.get_lidar_point_cloud(img_idx, dataset.calib_dir,
dataset.velo_dir)
offset_dist = 2.0
# Filter points within certain xyz range and offset from ground plane
offset_filter = obj_utils.get_point_filter(point_cloud, area_extents,
ground_plane, offset_dist)
# Filter points within 0.2m of the road plane
road_filter = obj_utils.get_point_filter(point_cloud, area_extents,
ground_plane, 0.1)
slice_filter = np.logical_xor(offset_filter, road_filter)
point_cloud = point_cloud.T[slice_filter]
# Generate Voxel Grid
vx_grid_3d = voxel_grid.VoxelGrid()
vx_grid_3d.voxelize(point_cloud, 0.1, area_extents)
# Anchors in anchor format
all_anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
# Filter the boxes here!
start_time = time.time()
empty_filter = \
anchor_filter.get_empty_anchor_filter(anchors=all_anchors,
voxel_grid_3d=vx_grid_3d,
density_threshold=1)
anchor_boxes_3d = anchor_boxes_3d[empty_filter]
end_time = time.time()
print("Anchors filtered in {} s".format(end_time - start_time))
# Visualize GT boxes
# Grab ground truth
ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx)
# ----------
# Test Sample extraction
# Visualize from here
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
image_path = dataset.get_rgb_image_path(sample_name)
image_shape = np.array(Image.open(image_path)).shape
rgb_boxes, rgb_normalized_boxes = \
anchor_projector.project_to_image_space(all_anchors, dataset,
image_shape, img_idx)
# Overlay boxes on images
anchor_objects = []
for anchor_idx in range(len(anchor_boxes_3d)):
anchor_box_3d = anchor_boxes_3d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(
anchor_box_3d, 'Anchor')
# Append to a list for visualization in VTK later
anchor_objects.append(obj_label)
for idx in range(len(ground_truth_list)):
ground_truth_obj = ground_truth_list[idx]
# Append to a list for visualization in VTK later
anchor_objects.append(ground_truth_obj)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(anchor_objects, anchor_colour_scheme)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(point_cloud)
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(vx_grid_3d)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
# vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vtk.vtkInteractorStyleTrackballCamera())
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start() # Blocking
def main():
# Setting Paths
cam = 2
# dataset_dir = '/media/bradenhurl/hd/gta/object/'
data_set = 'training'
dataset_dir = os.path.expanduser('~') + '/wavedata-dev/demos/gta'
#dataset_dir = os.path.expanduser('~') + '/Kitti/object/'
dataset_dir = os.path.expanduser(
'~') + '/GTAData/TruPercept/object_tru_percept8/'
#Set to true to see predictions (results) from all perspectives
use_results = True
altPerspective = False
perspID = 48133
perspStr = '%07d' % perspID
altPerspect_dir = os.path.join(dataset_dir, data_set + '/alt_perspective/')
if altPerspective:
data_set = data_set + '/alt_perspective/' + perspStr
fromWiseWindows = False
useEVE = False
if fromWiseWindows:
data_set = 'object'
if useEVE:
dataset_dir = '/media/bradenhurl/hd/data/eve/'
else:
dataset_dir = '/media/bradenhurl/hd/data/'
image_dir = os.path.join(dataset_dir, data_set) + '/image_2'
velo_dir = os.path.join(dataset_dir, data_set) + '/velodyne'
calib_dir = os.path.join(dataset_dir, data_set) + '/calib'
if use_results:
label_dir = os.path.join(dataset_dir, data_set) + '/predictions'
else:
label_dir = os.path.join(dataset_dir, data_set) + '/label_2'
base_dir = os.path.join(dataset_dir, data_set)
comparePCs = False
if comparePCs:
velo_dir2 = os.path.join(dataset_dir, data_set) + '/velodyne'
tracking = False
if tracking:
seq_idx = 1
data_set = '%04d' % seq_idx
dataset_dir = '/media/bradenhurl/hd/GTAData/August-01/tracking'
image_dir = os.path.join(dataset_dir, 'images', data_set)
label_dir = os.path.join(dataset_dir, 'labels', data_set)
velo_dir = os.path.join(dataset_dir, 'velodyne', data_set)
calib_dir = os.path.join(dataset_dir, 'training', 'calib', '0000')
#Used for visualizing inferences
#label_dir = '/media/bradenhurl/hd/avod/avod/data/outputs/pyramid_people_gta_40k'
#label_dir = label_dir + '/predictions/kitti_predictions_3d/test/0.02/154000/data/'
closeView = False
pitch = 170
pointSize = 3
zoom = 1
if closeView:
pitch = 180.5
pointSize = 3
zoom = 35
image_list = os.listdir(image_dir)
fulcrum_of_points = True
use_intensity = False
img_idx = 2
print('=== Loading image: {:06d}.png ==='.format(img_idx))
print(image_dir)
image = cv2.imread(image_dir + '/{:06d}.png'.format(img_idx))
image_shape = (image.shape[1], image.shape[0])
if use_intensity:
point_cloud, intensity = obj_utils.get_lidar_point_cloud(
img_idx, calib_dir, velo_dir, ret_i=use_intensity)
else:
point_cloud = obj_utils.get_lidar_point_cloud(img_idx,
calib_dir,
velo_dir,
im_size=image_shape)
if comparePCs:
point_cloud2 = obj_utils.get_lidar_point_cloud(img_idx,
calib_dir,
velo_dir2,
im_size=image_shape)
point_cloud = np.hstack((point_cloud, point_cloud2))
# Reshape points into N x [x, y, z]
all_points = np.array(point_cloud).transpose().reshape((-1, 3))
# Define Fixed Sizes for the voxel grid
x_min = -85
x_max = 85
y_min = -5
y_max = 5
z_min = 3
z_max = 85
x_min = min(point_cloud[0])
x_max = max(point_cloud[0])
y_min = min(point_cloud[1])
y_max = max(point_cloud[1])
#z_min = min(point_cloud[2])
z_max = max(point_cloud[2])
# Filter points within certain xyz range
area_filter = (point_cloud[0] > x_min) & (point_cloud[0] < x_max) & \
(point_cloud[1] > y_min) & (point_cloud[1] < y_max) & \
(point_cloud[2] > z_min) & (point_cloud[2] < z_max)
all_points = all_points[area_filter]
#point_colours = np.zeros(point_cloud.shape[1],0)
#print(point_colours.shape)
if fulcrum_of_points:
# Get point colours
point_colours = vis_utils.project_img_to_point_cloud(
all_points, image, calib_dir, img_idx)
print("Point colours shape: ", point_colours.shape)
print("Sample 0 of colour: ", point_colours[0])
elif use_intensity:
adjusted = intensity == 65535
intensity = intensity > 0
intensity = np.expand_dims(intensity, -1)
point_colours = np.hstack(
(intensity * 255, intensity * 255 - adjusted * 255,
intensity * 255 - adjusted * 255))
print("Intensity shape:", point_colours.shape)
print("Intensity sample: ", point_colours[0])
# Create Voxel Grid
voxel_grid = VoxelGrid()
voxel_grid_extents = [[x_min, x_max], [y_min, y_max], [z_min, z_max]]
print(voxel_grid_extents)
start_time = time.time()
voxel_grid.voxelize(all_points, 0.2, voxel_grid_extents)
end_time = time.time()
print("Voxelized in {} s".format(end_time - start_time))
# Get bounding boxes
gt_detections = obj_utils.read_labels(label_dir,
img_idx,
results=use_results)
if gt_detections is None:
gt_detections = []
#perspective_utils.to_world(gt_detections, base_dir, img_idx)
#perspective_utils.to_perspective(gt_detections, base_dir, img_idx)
for entity_str in os.listdir(altPerspect_dir):
if os.path.isdir(os.path.join(altPerspect_dir, entity_str)):
perspect_detections = perspective_utils.get_detections(
base_dir,
altPerspect_dir,
img_idx,
perspID,
entity_str,
results=use_results)
if perspect_detections != None:
if use_results:
stripped_detections = trust_utils.strip_objs(
perspect_detections)
gt_detections = gt_detections + stripped_detections
else:
gt_detections = gt_detections + perspect_detections
# Create VtkPointCloud for visualization
vtk_point_cloud = VtkPointCloud()
if fulcrum_of_points or use_intensity:
vtk_point_cloud.set_points(all_points, point_colours)
else:
vtk_point_cloud.set_points(all_points)
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(pointSize)
# Create VtkVoxelGrid for visualization
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
COLOUR_SCHEME_PAPER = {
"Car": (0, 0, 255), # Blue
"Pedestrian": (255, 0, 0), # Red
"Bus": (0, 0, 255), #Blue
"Cyclist": (150, 50, 100), # Purple
"Van": (255, 150, 150), # Peach
"Person_sitting": (150, 200, 255), # Sky Blue
"Truck": (0, 0, 255), # Light Grey
"Tram": (150, 150, 150), # Grey
"Misc": (100, 100, 100), # Dark Grey
"DontCare": (255, 255, 255), # White
}
# Create VtkBoxes for boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(gt_detections,
COLOUR_SCHEME_PAPER) #vtk_boxes.COLOUR_SCHEME_KITTI)
# Create Axes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
#vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(pitch)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(zoom)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Point Cloud and Voxel Grid, Image {}".format(img_idx))
vtk_render_window.SetSize(1920, 1080)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_point_cloud.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_boxes.vtk_actor,
]))
# Show image
image = cv2.imread(image_dir + "/%06d.png" % img_idx)
cv2.imshow("Press any key to continue", image)
cv2.waitKey()
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start() # Blocking
def main():
"""Flip RPN Mini Batch
Visualization of the mini batch anchors for RpnModel training.
Keys:
F1: Toggle mini batch anchors
F2: Flipped
"""
anchor_colour_scheme = {
"Car": (255, 0, 0), # Red
"Pedestrian": (255, 150, 50), # Orange
"Cyclist": (150, 50, 100), # Purple
"DontCare": (255, 255, 255), # White
"Anchor": (150, 150, 150), # Gray
"Regressed Anchor": (255, 255, 0), # Yellow
"Positive": (0, 255, 255), # Teal
"Negative": (255, 0, 255) # Purple
}
dataset_config_path = mlod.root_dir() + \
'/configs/mb_rpn_demo_cars.config'
# dataset_config_path = mlod.root_dir() + \
# '/configs/mb_rpn_demo_people.config'
##############################
# Options
##############################
# # # Random sample # # #
sample_name = None
# # # Cars # # #
# sample_name = "000001"
# sample_name = "000050"
# sample_name = "000104"
# sample_name = "000112"
# sample_name = "000169"
# sample_name = "000191"
sample_name = "003801"
# # # Pedestrians # # #
# sample_name = "000000"
# sample_name = "000011"
# sample_name = "000015"
# sample_name = "000028"
# sample_name = "000035"
# sample_name = "000134"
# sample_name = "000167"
# sample_name = '000379'
# sample_name = '000381'
# sample_name = '000397'
# sample_name = '000398'
# sample_name = '000401'
# sample_name = '000407'
# sample_name = '000486'
# sample_name = '000509'
# # Cyclists # # #
# sample_name = '000122'
# sample_name = '000448'
# # # Multiple classes # # #
# sample_name = "000764"
##############################
# End of Options
##############################
# Create Dataset
dataset = DatasetBuilder.load_dataset_from_config(dataset_config_path)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx]
anchor_strides = dataset.kitti_utils.anchor_strides
img_idx = int(sample_name)
print("Showing mini batch for sample {}".format(sample_name))
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# KittiUtils class
dataset_utils = dataset.kitti_utils
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = point_cloud.T
# Grab ground truth
ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx)
ground_truth_list = dataset_utils.filter_labels(ground_truth_list)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
##############################
# Flip sample info
##############################
start_time = time.time()
flipped_image = kitti_aug.flip_image(image)
flipped_point_cloud = kitti_aug.flip_point_cloud(point_cloud)
flipped_gt_list = [
kitti_aug.flip_label_in_3d_only(obj) for obj in ground_truth_list
]
flipped_ground_plane = kitti_aug.flip_ground_plane(ground_plane)
flipped_calib_p2 = kitti_aug.flip_stereo_calib_p2(stereo_calib_p2,
image_shape)
print('flip sample', time.time() - start_time)
flipped_points = flipped_point_cloud.T
point_colours = vis_utils.project_img_to_point_cloud(
points, image, dataset.calib_dir, img_idx)
##############################
# Generate anchors
##############################
clusters, _ = dataset.get_cluster_info()
anchor_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Read mini batch info
anchors_info = dataset_utils.get_anchors_info(sample_name)
all_anchor_boxes_3d = []
all_ious = []
all_offsets = []
for class_idx in range(len(dataset.classes)):
anchor_boxes_3d = anchor_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=clusters[class_idx],
anchor_stride=anchor_strides[class_idx],
ground_plane=ground_plane)
if len(anchors_info[class_idx]) > 0:
indices, ious, offsets, classes = anchors_info[class_idx]
# Get non empty anchors from the indices
non_empty_anchor_boxes_3d = anchor_boxes_3d[indices]
all_anchor_boxes_3d.extend(non_empty_anchor_boxes_3d)
all_ious.extend(ious)
all_offsets.extend(offsets)
if not len(all_anchor_boxes_3d) > 0:
# Exit early if anchors_info is empty
print("No anchors, Please try a different sample")
return
# Convert to ndarrays
all_anchor_boxes_3d = np.asarray(all_anchor_boxes_3d)
all_ious = np.asarray(all_ious)
all_offsets = np.asarray(all_offsets)
##############################
# Flip anchors
##############################
start_time = time.time()
# Flip anchors and offsets
flipped_anchor_boxes_3d = kitti_aug.flip_boxes_3d(all_anchor_boxes_3d,
flip_ry=False)
all_offsets[:, 0] = -all_offsets[:, 0]
print('flip anchors and offsets', time.time() - start_time)
# Overwrite with flipped things
all_anchor_boxes_3d = flipped_anchor_boxes_3d
points = flipped_points
ground_truth_list = flipped_gt_list
ground_plane = flipped_ground_plane
##############################
# Mini batch sampling
##############################
# Sample an RPN mini batch from the non empty anchors
mini_batch_utils = dataset.kitti_utils.mini_batch_utils
mb_mask_tf, _ = mini_batch_utils.sample_rpn_mini_batch(all_ious)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
mb_mask = sess.run(mb_mask_tf)
mb_anchor_boxes_3d = all_anchor_boxes_3d[mb_mask]
mb_anchor_ious = all_ious[mb_mask]
mb_anchor_offsets = all_offsets[mb_mask]
# ObjectLabel list that hold all boxes to visualize
obj_list = []
# Convert the mini_batch anchors to object list
for i in range(len(mb_anchor_boxes_3d)):
if mb_anchor_ious[i] > mini_batch_utils.rpn_pos_iou_range[0]:
obj_type = "Positive"
else:
obj_type = "Negative"
obj = box_3d_encoder.box_3d_to_object_label(mb_anchor_boxes_3d[i],
obj_type)
obj_list.append(obj)
# Convert all non-empty anchors to object list
non_empty_anchor_objs = \
[box_3d_encoder.box_3d_to_object_label(
anchor_box_3d, obj_type='Anchor')
for anchor_box_3d in all_anchor_boxes_3d]
##############################
# Regress Positive Anchors
##############################
# Convert anchor_boxes_3d to anchors and apply offsets
mb_pos_mask = mb_anchor_ious > mini_batch_utils.rpn_pos_iou_range[0]
mb_pos_anchor_boxes_3d = mb_anchor_boxes_3d[mb_pos_mask]
mb_pos_anchor_offsets = mb_anchor_offsets[mb_pos_mask]
mb_pos_anchors = box_3d_encoder.box_3d_to_anchor(mb_pos_anchor_boxes_3d)
regressed_pos_anchors = anchor_encoder.offset_to_anchor(
mb_pos_anchors, mb_pos_anchor_offsets)
# Convert regressed anchors to ObjectLabels for visualization
regressed_anchor_boxes_3d = box_3d_encoder.anchors_to_box_3d(
regressed_pos_anchors, fix_lw=True)
regressed_anchor_objs = \
[box_3d_encoder.box_3d_to_object_label(
box_3d, obj_type='Regressed Anchor')
for box_3d in regressed_anchor_boxes_3d]
##############################
# Visualization
##############################
cv2.imshow('{} flipped'.format(sample_name), flipped_image)
cv2.waitKey()
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for mini batch anchors
vtk_pos_anchor_boxes = VtkBoxes()
vtk_pos_anchor_boxes.set_objects(obj_list, anchor_colour_scheme)
# VtkBoxes for non empty anchors
vtk_non_empty_anchors = VtkBoxes()
vtk_non_empty_anchors.set_objects(non_empty_anchor_objs,
anchor_colour_scheme)
vtk_non_empty_anchors.set_line_width(0.1)
# VtkBoxes for regressed anchors
vtk_regressed_anchors = VtkBoxes()
vtk_regressed_anchors.set_objects(regressed_anchor_objs,
anchor_colour_scheme)
vtk_regressed_anchors.set_line_width(5.0)
# Create VtkBoxes for ground truth
vtk_gt_boxes = VtkBoxes()
vtk_gt_boxes.set_objects(ground_truth_list,
anchor_colour_scheme,
show_orientations=True)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
vtk_ground_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, dataset.kitti_utils.bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_non_empty_anchors.vtk_actor)
vtk_renderer.AddActor(vtk_pos_anchor_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_regressed_anchors.vtk_actor)
vtk_renderer.AddActor(vtk_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("RPN Mini Batch")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_non_empty_anchors.vtk_actor,
vtk_pos_anchor_boxes.vtk_actor,
vtk_regressed_anchors.vtk_actor,
vtk_ground_plane.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""
Simple demo script for debugging integral images with visualization
"""
anchor_colour_scheme = {"Anchor": (0, 0, 255)} # Blue
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
label_cluster_utils = LabelClusterUtils(dataset)
clusters, _ = label_cluster_utils.get_clusters()
area_extents = np.array([[0, 2], [-1, 0.], [0, 2]])
boxes_3d = np.array([
[2, 0, 1, 1, 1, 1, 0],
[1, 0, 2, 1, 1, 1, 0],
])
xyz = np.array([[0.5, -0.01, 1.1],
[1.5, -0.01, 1.1],
[0.5, -0.01, 1.6],
[1.5, -0.01, 1.6],
[0.5, -0.49, 1.1],
[1.5, -0.49, 1.1],
[0.5, -0.51, 1.6],
[1.5, -0.51, 1.6]
])
vx_grid_3d = voxel_grid.VoxelGrid()
vx_grid_3d.voxelize(xyz, 0.1, area_extents)
anchors = box_3d_encoder.box_3d_to_anchor(boxes_3d)
# Filter the boxes here!
start_time = time.time()
empty_filter = anchor_filter.get_empty_anchor_filter(anchors=anchors,
voxel_grid_3d=vx_grid_3d,
density_threshold=1)
boxes_3d = boxes_3d[empty_filter]
end_time = time.time()
print("Anchors filtered in {} s".format(end_time - start_time))
box_objects = []
for box_idx in range(len(boxes_3d)):
box = boxes_3d[box_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(box, 'Anchor')
# Append to a list for visualization in VTK later
box_objects.append(obj_label)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(box_objects, anchor_colour_scheme)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(xyz)
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(vx_grid_3d)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
# vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vtk.vtkInteractorStyleTrackballCamera())
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def vis_pc(pc, obj_list, frustum_points=None):
# Define Fixed Sizes for the voxel grid
x_min = -85
x_max = 85
y_min = -5
y_max = 5
z_min = 3
z_max = 85
# Comment these out to filter points by area
x_min = min(pc[0])
x_max = max(pc[0])
y_min = min(pc[1])
y_max = max(pc[1])
z_min = min(pc[2])
z_max = max(pc[2])
# Reshape points into N x [x, y, z]
all_points = np.array(pc).transpose().reshape((-1, 3))
# Filter points within certain xyz range
area_filter = (pc[0] > x_min) & (pc[0] < x_max) & \
(pc[1] > y_min) & (pc[1] < y_max) & \
(pc[2] > z_min) & (pc[2] < z_max)
all_points = all_points[area_filter]
# Create Voxel Grid
voxel_grid = VoxelGrid()
voxel_grid_extents = [[x_min, x_max], [y_min, y_max], [z_min, z_max]]
print(voxel_grid_extents)
start_time = time.time()
voxel_grid.voxelize(all_points, 0.2, voxel_grid_extents)
end_time = time.time()
print("Voxelized in {} s".format(end_time - start_time))
# Some settings for the initial camera view and point size
closeView = False
pitch = 170
pointSize = 4
zoom = 1
if closeView:
pitch = 180.5
pointSize = 3
zoom = 35
# Create VtkPointCloud for visualization
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(all_points)
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(pointSize)
# Create VtkVoxelGrid for visualization
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
# Create VtkBoxes for boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(obj_list, COLOUR_SCHEME, False)
# Create Axes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(2, 2, 2)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Add lines for frustum
if frustum_points is not None:
frustum_actor = get_frustum_actor(frustum_points)
vtk_renderer.AddActor(frustum_actor)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(pitch)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(zoom)
# Zoom/navigate to the desired camera view then exit
# Three lines will be output. Paste these here
# Above forward view
current_cam.SetPosition(7.512679241328601, -312.20497623371926, -130.38469206536766)
current_cam.SetViewUp(-0.01952407393317445, -0.44874501090739727, 0.893446543293314)
current_cam.SetFocalPoint(11.624950999358777, 14.835920755080867, 33.965665867613836)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Point Cloud and Voxel Grid")
vtk_render_window.SetSize(1920, 1080)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_point_cloud.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_boxes.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start() # Blocking
def visualize_objects_in_pointcloud(objects, COLOUR_SCHEME, dataset_dir,
img_idx, fulcrum_of_points, use_intensity,
receive_from_perspective, compare_pcs=False,
show_3d_point_count=False, show_orientation=cfg.VISUALIZE_ORIENTATION,
final_results=False, show_score=False,
compare_with_gt=False, show_image=True,
_text_positions=None, _text_labels=None):
image_dir = os.path.join(dataset_dir, 'image_2')
velo_dir = os.path.join(dataset_dir, 'velodyne')
calib_dir = os.path.join(dataset_dir, 'calib')
if compare_pcs:
fulcrum_of_points = False
print('=== Loading image: {:06d}.png ==='.format(img_idx))
print(image_dir)
image = cv2.imread(image_dir + '/{:06d}.png'.format(img_idx))
image_shape = (image.shape[1], image.shape[0])
if use_intensity:
point_cloud,intensity = obj_utils.get_lidar_point_cloud(img_idx, calib_dir, velo_dir,
ret_i=use_intensity)
else:
point_cloud = obj_utils.get_lidar_point_cloud(img_idx, calib_dir, velo_dir,
im_size=image_shape)
if compare_pcs:
receive_persp_dir = os.path.join(altPerspect_dir, '{:07d}'.format(receive_from_perspective))
velo_dir2 = os.path.join(receive_persp_dir, 'velodyne')
print(velo_dir2)
if not os.path.isdir(velo_dir2):
print("Error: cannot find velo_dir2: ", velo_dir2)
exit()
point_cloud2 = obj_utils.get_lidar_point_cloud(img_idx, calib_dir, velo_dir2,
im_size=image_shape)
#Set to true to display point clouds in world coordinates (for debugging)
display_in_world=False
if display_in_world:
point_cloud = perspective_utils.pc_to_world(point_cloud.T, receive_persp_dir, img_idx)
point_cloud2 = perspective_utils.pc_to_world(point_cloud2.T, dataset_dir, img_idx)
point_cloud = np.hstack((point_cloud.T, point_cloud2.T))
else:
point_cloud2 = perspective_utils.pc_persp_transform(point_cloud2.T, receive_persp_dir, dataset_dir, img_idx)
point_cloud = np.hstack((point_cloud, point_cloud2.T))
# Reshape points into N x [x, y, z]
all_points = np.array(point_cloud).transpose().reshape((-1, 3))
# Define Fixed Sizes for the voxel grid
x_min = -85
x_max = 85
y_min = -5
y_max = 5
z_min = 3
z_max = 85
# Comment these out to filter points by area
x_min = min(point_cloud[0])
x_max = max(point_cloud[0])
y_min = min(point_cloud[1])
y_max = max(point_cloud[1])
z_min = min(point_cloud[2])
z_max = max(point_cloud[2])
# Filter points within certain xyz range
area_filter = (point_cloud[0] > x_min) & (point_cloud[0] < x_max) & \
(point_cloud[1] > y_min) & (point_cloud[1] < y_max) & \
(point_cloud[2] > z_min) & (point_cloud[2] < z_max)
all_points = all_points[area_filter]
point_colours = None
if fulcrum_of_points:
# Get point colours
point_colours = vis_utils.project_img_to_point_cloud(all_points, image,
calib_dir, img_idx)
elif use_intensity:
adjusted = intensity == 65535
intensity = intensity > 0
intensity = np.expand_dims(intensity,-1)
point_colours = np.hstack((intensity*255,intensity*255-adjusted*255,intensity*255-adjusted*255))
# Create Voxel Grid
voxel_grid = VoxelGrid()
voxel_grid_extents = [[x_min, x_max], [y_min, y_max], [z_min, z_max]]
print(voxel_grid_extents)
start_time = time.time()
voxel_grid.voxelize(all_points, 0.2, voxel_grid_extents)
end_time = time.time()
print("Voxelized in {} s".format(end_time - start_time))
# Some settings for the initial camera view and point size
closeView = False
pitch = 170
pointSize = 2
zoom = 1
if closeView:
pitch = 180.5
pointSize = 3
zoom = 35
# Create VtkPointCloud for visualization
vtk_point_cloud = VtkPointCloud()
if point_colours is not None:
vtk_point_cloud.set_points(all_points, point_colours)
else:
vtk_point_cloud.set_points(all_points)
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(pointSize)
# Create VtkVoxelGrid for visualization
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
# Create VtkBoxes for boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(objects, COLOUR_SCHEME, show_orientation)#vtk_boxes.COLOUR_SCHEME_KITTI)
# Create Axes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
vtk_renderer.AddActor(axes)
if _text_positions is not None:
vtk_text_labels = VtkTextLabels()
vtk_text_labels.set_text_labels(_text_positions, _text_labels)
vtk_renderer.AddActor(vtk_text_labels.vtk_actor)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(pitch)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(zoom)
# Zoom/navigate to the desired camera view then exit
# Three lines will be output. Paste these here
# Above forward view
current_cam.SetPosition(7.512679241328601, -312.20497623371926, -130.38469206536766)
current_cam.SetViewUp(-0.01952407393317445, -0.44874501090739727, 0.893446543293314)
current_cam.SetFocalPoint(11.624950999358777, 14.835920755080867, 33.965665867613836)
# Top down view of synchronization
current_cam.SetPosition(28.384757950371405, -125.46190537888288, 63.60263366961189)
current_cam.SetViewUp(-0.02456679343399302, 0.0030507437719906913, 0.9996935358512673)
current_cam.SetFocalPoint(27.042134804730317, 15.654378427929846, 63.13899801247614)
current_cam.SetPosition(30.3869590224831, -50.28910856489952, 60.097631136698965)
current_cam.SetViewUp(-0.0237472244952177, -0.06015048799392083, 0.997906803325274)
current_cam.SetFocalPoint(27.06695416156647, 15.347824332314035, 63.97499987548391)
# current_cam.SetPosition(14.391008769593322, -120.06549828061613, -1.567028749253062)
# current_cam.SetViewUp(-0.02238762832327178, -0.1049057307562059, 0.9942301452644481)
# current_cam.SetFocalPoint(10.601112314728102, 20.237110061924664, 13.151596441968126)
# # Top down view of whole detection area
# current_cam.SetPosition(11.168659642537031, -151.97163016078756, 17.590894639193227)
# current_cam.SetViewUp(-0.02238762832327178, -0.1049057307562059, 0.9942301452644481)
# current_cam.SetFocalPoint(6.5828849321501055, 17.79452593368671, 35.400431120570865)
# Top down view of scenario
current_cam.SetPosition(2.075612197299923, -76.19063612245675, 5.948366424752178)
current_cam.SetViewUp(-0.02238762832327178, -0.1049057307562059, 0.9942301452644481)
current_cam.SetFocalPoint(-0.5129380758134061, 19.637933198314016, 16.00138547483155)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Point Cloud and Voxel Grid, Image {}".format(img_idx))
vtk_render_window.SetSize(1920, 1080)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_point_cloud.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_boxes.vtk_actor,
]))
# Show image
if show_image:
image = cv2.imread(image_dir + "/%06d.png" % img_idx)
cv2.imshow("Press any key to continue", image)
cv2.waitKey()
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start() # Blocking
# vtk_render_window_interactor.Initialize() # Non-Blocking
# Obtain camera positional information for repeatable views
print("current_cam.SetPosition{}".format(current_cam.GetPosition()))
print("current_cam.SetViewUp{}".format(current_cam.GetViewUp()))
print("current_cam.SetFocalPoint{}".format(current_cam.GetFocalPoint()))