def set_up_model_test_mode(pipeline_config_path, data_split):
"""Returns the model and its config in test mode."""
model_config, _, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=False)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
# Use the validation set
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
# Remove augmentation when in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config, train_val_test='test', dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config, train_val_test='test', dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model, model_config
def set_up_model():
test_pipeline_config_path = mlod.root_dir() + \
'/configs/mlod_exp_example.config'
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
dataset_config = config_builder.proto_to_obj(dataset_config)
train_val_test = 'test'
dataset_config.data_split = 'test'
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
return model
def main():
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
image_path = dataset.get_rgb_image_path('000000')
# cv2
cv2_image1 = cv2.imread(image_path)
cv2_image2 = cv2.imread(image_path)
cv2_image3 = cv2.imread(image_path)
cv2_image1_shape = cv2_image1.shape
cv2_image2_shape = cv2_image2.shape
cv2_image3_shape = cv2_image3.shape
print(cv2_image1_shape)
print(cv2_image2_shape)
print(cv2_image3_shape)
# PIL
pil_image1 = Image.open(image_path)
pil_image2 = Image.open(image_path)
pil_image3 = Image.open(image_path)
pil_image1_shape = pil_image1.size
pil_image2_shape = pil_image2.size
pil_image3_shape = pil_image3.size
print(pil_image1_shape)
print(pil_image2_shape)
print(pil_image3_shape)
pil_image1 = np.asarray(pil_image1)
pil_image2 = np.asarray(pil_image2)
pil_image3 = np.asarray(pil_image3)
def main():
test_pipeline_config_path = mlod.root_dir() + \
'/data/configs/official/cars/cars_000_vanilla.config'
model_config, train_config, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# train_val_test = 'val'
# dataset_config.data_split = 'val'
train_val_test = 'test'
dataset_config.data_split = 'trainval'
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = False
# dataset_config.cache_config.cache_images = True
# dataset_config.cache_config.cache_depth_maps = True
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
kitti_utils = dataset.kitti_utils
bev_source = 'lidar'
# sample_name = '000000'
# img_idx = np.random.randint(0, 1000)
# sample_name = '{:06d}'.format(img_idx)
num_samples = 200
all_load_times = []
all_bev_times = []
for sample_idx in range(num_samples):
sys.stdout.write('\rSample {} / {}'.format(sample_idx,
num_samples - 1))
img_idx = sample_idx
sample_name = '{:06d}'.format(sample_idx)
loading_start_time = time.time()
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
calib_p2 = calib_utils.read_calibration(dataset.calib_dir, img_idx)
point_cloud = kitti_utils.get_point_cloud(bev_source, int(sample_name),
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
all_load_times.append(time.time() - loading_start_time)
bev_start_time = time.time()
bev_maps = kitti_utils.create_bev_maps(point_cloud, ground_plane)
bev_end_time = time.time()
all_bev_times.append(bev_end_time - bev_start_time)
print('')
print('Load mean:', np.mean(all_load_times))
print('Load median:', np.median(all_load_times))
print('BEV mean:', np.mean(all_bev_times))
print('BEV median:', np.median(all_bev_times))
def main():
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = "trainval"
unittest_dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
gen_label_clusters.main(unittest_dataset)
gen_mini_batches.main(unittest_dataset)
def main():
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL)
# get proposals
proposal_output_dir = mlod.root_dir() + \
"/data/predictions/rpn_model/proposals_and_scores/" + \
dataset.data_split
global_steps = os.listdir(proposal_output_dir)
print('Checkpoints found ', global_steps)
all_recalls = []
for step in global_steps:
for sample_name in dataset.sample_list:
img_idx = int(sample_name)
# ------------------------------------
# Load proposals and scores from files
# ------------------------------------
proposals_scores_dir = proposal_output_dir + \
"{}/{}/{}.txt".format(dataset.data_split,
step,
sample_name)
if not os.path.exists(proposals_scores_dir):
print('File {} not found, skipping'.format(sample_name))
continue
proposals_scores = np.loadtxt(proposals_scores_dir)
proposals = proposals_scores[:, 0:-1]
proposal_iou_format = \
box_3d_encoder.box_3d_to_3d_iou_format(proposals)
# scores are in the last column
scores = proposals_scores[:, -1]
# -----------------------
# Get ground truth labels
# -----------------------
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
_, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects(
gt_objects, ['Car', 'car'])
score_thresholds = np.array([0.3])
iou_threshold = 0.0025
# calculate RPN recall and precision
precision, recall = evaluation.evaluate_3d([gt_3d_bbs],
[proposal_iou_format],
[scores],
score_thresholds,
iou_threshold)
print('Recall ', recall[0])
print('Precision ', precision[0])
all_recalls.append(recall)
def get_fake_dataset(self, data_split, directory):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
# Overwrite config values
dataset_config.data_split = data_split
dataset_config.dataset_dir = directory
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
return dataset
def setUpClass(cls):
# Initialize the Kitti dataset
test_dir = tests.test_path()
# Get the unittest-kitti dataset
dataset_builder = DatasetBuilder()
cls.dataset = dataset_builder.build_kitti_dataset(
dataset_builder.KITTI_UNITTEST)
cls.log_dir = test_dir + '/logs'
cls.bev_vgg_cls = vgg.BevVggClassification()
def setUpClass(cls):
pipeline_config = pipeline_pb2.NetworkPipelineConfig()
dataset_config = pipeline_config.dataset_config
config_path = mlod.root_dir() + '/configs/unittest_model.config'
cls.model_config = config_build.get_model_config_from_file(config_path)
input_config = cls.model_config.input_config
input_config.bev_depth = 5 # unittest-kitti has [2, 1, 1] clusters
dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def inference(model_config, eval_config, dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
model_evaluator.run_latest_checkpoints()
def set_up_video_dataset(dataset_config, dataset_dir, data_split,
data_split_dir):
# Overwrite fields
dataset_config.name = 'kitti_video'
dataset_config.dataset_dir = dataset_dir
dataset_config.data_split = data_split
dataset_config.data_split_dir = data_split_dir
dataset_config.has_labels = False
# Overwrite repeated fields
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.aug_list = []
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
return dataset
def setUp(self):
tf.test.TestCase.setUp(self)
test_pipeline_config_path = mlod.root_dir() + \
'/configs/unittest_stagewise.config'
self.model_config, self.train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# unittest-kitti has [2, 1, 1] clusters
input_config = self.model_config.input_config
input_config.bev_depth = 5
# Generate dataset
self.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST,
use_defaults=False,
new_cfg=dataset_config)
def train(rpn_model_config, mlod_model_config, rpn_train_config,
mlod_train_config, dataset_config):
train_val_test = 'train'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
paths_config = rpn_model_config.paths_config
rpn_checkpoint_dir = paths_config.checkpoint_dir
with tf.Graph().as_default():
model = RpnModel(rpn_model_config,
train_val_test=train_val_test,
dataset=dataset)
trainer.train(model, rpn_train_config)
# load the weights back in
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init_op)
trainer_utils.load_checkpoints(rpn_checkpoint_dir, saver)
checkpoint_to_restore = saver.last_checkpoints[-1]
trainer_utils.load_model_weights(model, sess,
checkpoint_to_restore)
# Merge RPN configs with MLOD - This will overwrite
# the appropriate configs set for MLOD while keeping
# the common configs the same.
rpn_model_config.MergeFrom(mlod_model_config)
rpn_train_config.MergeFrom(mlod_train_config)
mlod_model_merged = deepcopy(rpn_model_config)
mlod_train_merged = deepcopy(rpn_train_config)
with tf.Graph().as_default():
model = MlodModel(mlod_model_merged,
train_val_test=train_val_test,
dataset=dataset)
trainer.train(model,
mlod_train_merged,
stagewise_training=True,
init_checkpoint_dir=rpn_checkpoint_dir)
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
trainer.train(model, train_config)
def set_up_model_train_mode(pipeline_config_path, data_split):
"""Returns the model and its train_op."""
model_config, train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
pipeline_config_path, is_training=True)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
model_name = model_config.model_name
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=data_split,
dataset=dataset)
elif model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=data_split,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
prediction_dict = model.build()
losses_dict, total_loss = model.loss(prediction_dict)
# These parameters are required to set up the optimizer
global_summaries = set([])
global_step_tensor = tf.Variable(0, trainable=False)
training_optimizer = optimizer_builder.build(train_config.optimizer,
global_summaries,
global_step_tensor)
# Set up the train op
train_op = slim.learning.create_train_op(total_loss, training_optimizer)
return model, train_op
def setUpClass(cls):
cls.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST)
cls.mb_utils = cls.dataset.kitti_utils.mini_batch_utils
def main():
"""This demo shows RPN proposals and MLOD predictions in the
3D point cloud.
Keys:
F1: Toggle proposals
F2: Toggle predictions
F3: Toggle 3D voxel grid
F4: Toggle point cloud
F5: Toggle easy ground truth objects (Green)
F6: Toggle medium ground truth objects (Orange)
F7: Toggle hard ground truth objects (Red)
F8: Toggle all ground truth objects (default off)
F9: Toggle ground slice filter (default off)
F10: Toggle offset slice filter (default off)
"""
##############################
# Options
##############################
rpn_score_threshold = 0.1
mlod_score_threshold = 0.1
proposals_line_width = 1.0
predictions_line_width = 3.0
show_orientations = True
point_cloud_source = 'lidar'
# Config file folder, default (<mlod_root>/data/outputs/<checkpoint_name>)
config_dir = None
checkpoint_name = 'mlod_fpn_people_n_m'
global_step = 135000 # Latest checkpoint
#data_split = 'val_half'
#data_split = 'val'
data_split = 'test'
# Show 3D iou text
draw_ious_3d = False
sample_name = '000031'
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# val split
# sample_name = '000181'
# sample_name = '000751'
# sample_name = '000843'
# sample_name = '000944'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001' # Hard, 1 far cyc
# sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# val split
# sample_name = '000015'
# sample_name = '000048'
# sample_name = '000058'
# sample_name = '000076' # Medium, few ped, 1 cyc
# sample_name = '000108'
# sample_name = '000118'
# sample_name = '000145'
# sample_name = '000153'
# sample_name = '000186'
# sample_name = '000195'
# sample_name = '000199'
# sample_name = '000397'
# sample_name = '004425'
# sample_name = '004474' # Hard, many ped, 1 cyc
# sample_name = '004657' # Hard, Few cycl, few ped
# sample_name = '006071'
# sample_name = '006828' # Hard, Few cycl, few ped
# sample_name = '006908' # Hard, Few cycl, few ped
# sample_name = '007412'
# sample_name = '007318' # Hard, Few cycl, few ped
##############################
# End of Options
##############################
if data_split == 'test':
draw_ious_3d = False
if config_dir is None:
config_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name
# Parse experiment config
pipeline_config_file = \
config_dir + '/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
dataset_config.data_split = data_split
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_names[sample_idx]
##############################
# Setup Paths
##############################
img_idx = int(sample_name)
# Text files directory
proposals_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_predictions_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
# Output images directory
img_out_dir = mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/predictions/images_3d/{}/{}/{}'.format(dataset.data_split,
global_step,
rpn_score_threshold)
if not os.path.exists(img_out_dir):
os.makedirs(img_out_dir)
##############################
# Proposals
##############################
# Load proposals from files
proposals_and_scores = np.loadtxt(
proposals_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
proposals = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
rpn_score_mask = proposal_scores > rpn_score_threshold
proposals = proposals[rpn_score_mask]
proposal_scores = proposal_scores[rpn_score_mask]
print('Proposals:', len(proposal_scores), proposal_scores)
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposals]
##############################
# Predictions
##############################
# Load proposals from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name)).reshape(-1, 9)
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_types = np.asarray(predictions_and_scores[:, 8], dtype=np.int32)
mlod_score_mask = prediction_scores >= mlod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[mlod_score_mask]
prediction_scores = prediction_scores[mlod_score_mask]
print('Predictions: ', len(prediction_scores), prediction_scores)
final_predictions = np.copy(prediction_boxes_3d)
# # Swap l, w for predictions where w > l
# swapped_indices = predictions[:, 4] > predictions[:, 3]
# final_predictions[swapped_indices, 3] = predictions[swapped_indices, 4]
# final_predictions[swapped_indices, 4] = predictions[swapped_indices, 3]
prediction_objs = []
dataset.classes = ['Pedestrian', 'Cyclist', 'Car']
for pred_idx in range(len(final_predictions)):
prediction_box_3d = final_predictions[pred_idx]
prediction_type = dataset.classes[prediction_types[pred_idx]]
prediction_obj = box_3d_encoder.box_3d_to_object_label(
prediction_box_3d, obj_type=prediction_type)
prediction_objs.append(prediction_obj)
##############################
# Ground Truth
##############################
dataset.has_labels = False
if dataset.has_labels:
# Get ground truth labels
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset, img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# 3D IoU
##############################
if draw_ious_3d:
# Convert to box_3d
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in prediction_objs
]
max_ious_3d = demo_utils.get_max_ious_3d(all_gt_boxes_3d,
pred_boxes_3d)
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
point_cloud = dataset.kitti_utils.get_point_cloud(point_cloud_source,
img_idx,
image_shape=image.shape)
point_cloud = np.asarray(point_cloud)
# Filter point cloud to extents
area_extents = np.asarray([[-40, 40], [-5, 3], [0, 70]])
bev_extents = area_extents[[0, 2]]
points = point_cloud.T
point_filter = obj_utils.get_point_filter(point_cloud, area_extents)
points = points[point_filter]
point_colours = vis_utils.project_img_to_point_cloud(
points, image, dataset.calib_dir, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
# Ground plane
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
vtk_axes = vtk.vtkAxesActor()
vtk_axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for proposal boxes
vtk_proposal_boxes = VtkBoxes()
vtk_proposal_boxes.set_line_width(proposals_line_width)
vtk_proposal_boxes.set_objects(proposal_objs, COLOUR_SCHEME_PREDICTIONS)
# Create VtkBoxes for prediction boxes
vtk_prediction_boxes = VtkPyramidBoxes()
vtk_prediction_boxes.set_line_width(predictions_line_width)
vtk_prediction_boxes.set_objects(prediction_objs,
COLOUR_SCHEME_PREDICTIONS,
show_orientations)
# Create VtkBoxes for ground truth
vtk_hard_gt_boxes = VtkBoxes()
vtk_medium_gt_boxes = VtkBoxes()
vtk_easy_gt_boxes = VtkBoxes()
vtk_all_gt_boxes = VtkBoxes()
vtk_hard_gt_boxes.set_objects(hard_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_medium_gt_boxes.set_objects(medium_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_easy_gt_boxes.set_objects(easy_gt_objs, COLOUR_SCHEME_PREDICTIONS,
show_orientations)
vtk_all_gt_boxes.set_objects(all_gt_objs, VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations)
# Create VtkTextLabels for 3D ious
vtk_text_labels = VtkTextLabels()
if draw_ious_3d and len(all_gt_boxes_3d) > 0:
gt_positions_3d = np.asarray(all_gt_boxes_3d)[:, 0:3]
vtk_text_labels.set_text_labels(
gt_positions_3d,
['{:0.3f}'.format(iou_3d) for iou_3d in max_ious_3d])
# Create VtkGroundPlane
vtk_ground_plane = VtkGroundPlane()
vtk_slice_bot_plane = VtkGroundPlane()
vtk_slice_top_plane = VtkGroundPlane()
vtk_ground_plane.set_plane(ground_plane, bev_extents)
vtk_slice_bot_plane.set_plane(ground_plane + [0, 0, 0, -0.2], bev_extents)
vtk_slice_top_plane.set_plane(ground_plane + [0, 0, 0, -2.0], bev_extents)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_proposal_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_prediction_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_text_labels.vtk_actor)
# Add ground plane and slice planes
vtk_renderer.AddActor(vtk_ground_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_bot_plane.vtk_actor)
vtk_renderer.AddActor(vtk_slice_top_plane.vtk_actor)
#vtk_renderer.AddActor(vtk_axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(3)
vtk_proposal_boxes.vtk_actor.SetVisibility(0)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.SetVisibility(0)
vtk_slice_bot_plane.vtk_actor.SetVisibility(0)
vtk_slice_top_plane.vtk_actor.SetVisibility(0)
vtk_ground_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_bot_plane.vtk_actor.GetProperty().SetOpacity(0.9)
vtk_slice_top_plane.vtk_actor.GetProperty().SetOpacity(0.9)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(3.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
# Add custom interactor to toggle actor visibilities
custom_interactor = vis_utils.CameraInfoInteractorStyle([
vtk_proposal_boxes.vtk_actor,
vtk_prediction_boxes.vtk_actor,
vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor,
vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor,
vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor,
vtk_ground_plane.vtk_actor,
vtk_slice_bot_plane.vtk_actor,
vtk_slice_top_plane.vtk_actor,
vtk_text_labels.vtk_actor,
])
vtk_render_window_interactor.SetInteractorStyle(custom_interactor)
# Render in VTK
vtk_render_window.Render()
# Take a screenshot
window_to_image_filter = vtk.vtkWindowToImageFilter()
window_to_image_filter.SetInput(vtk_render_window)
window_to_image_filter.Update()
png_writer = vtk.vtkPNGWriter()
file_name = img_out_dir + "/{}.png".format(sample_name)
png_writer.SetFileName(file_name)
png_writer.SetInputData(window_to_image_filter.GetOutput())
png_writer.Write()
print('Screenshot saved to ', file_name)
vtk_render_window_interactor.Start() # Blocking
def main():
"""
Visualization of anchor filtering using 3D integral images
"""
anchor_colour_scheme = {
"Car": (0, 255, 0), # Green
"Pedestrian": (255, 150, 50), # Orange
"Cyclist": (150, 50, 100), # Purple
"DontCare": (255, 0, 0), # Red
"Anchor": (0, 0, 255), # Blue
}
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL)
# Options
clusters, _ = dataset.get_cluster_info()
sample_name = "000000"
img_idx = int(sample_name)
anchor_stride = [0.5, 0.5]
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator(
anchor_3d_sizes=clusters, anchor_stride=anchor_stride)
area_extents = np.array([[-40, 40], [-5, 3], [0, 70]])
# Generate anchors in box_3d format
start_time = time.time()
anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents,
ground_plane=ground_plane)
end_time = time.time()
print("Anchors generated in {} s".format(end_time - start_time))
point_cloud = obj_utils.get_lidar_point_cloud(img_idx, dataset.calib_dir,
dataset.velo_dir)
offset_dist = 2.0
# Filter points within certain xyz range and offset from ground plane
offset_filter = obj_utils.get_point_filter(point_cloud, area_extents,
ground_plane, offset_dist)
# Filter points within 0.2m of the road plane
road_filter = obj_utils.get_point_filter(point_cloud, area_extents,
ground_plane, 0.1)
slice_filter = np.logical_xor(offset_filter, road_filter)
point_cloud = point_cloud.T[slice_filter]
# Generate Voxel Grid
vx_grid_3d = voxel_grid.VoxelGrid()
vx_grid_3d.voxelize(point_cloud, 0.1, area_extents)
# Anchors in anchor format
all_anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
# Filter the boxes here!
start_time = time.time()
empty_filter = \
anchor_filter.get_empty_anchor_filter(anchors=all_anchors,
voxel_grid_3d=vx_grid_3d,
density_threshold=1)
anchor_boxes_3d = anchor_boxes_3d[empty_filter]
end_time = time.time()
print("Anchors filtered in {} s".format(end_time - start_time))
# Visualize GT boxes
# Grab ground truth
ground_truth_list = obj_utils.read_labels(dataset.label_dir, img_idx)
# ----------
# Test Sample extraction
# Visualize from here
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
image_path = dataset.get_rgb_image_path(sample_name)
image_shape = np.array(Image.open(image_path)).shape
rgb_boxes, rgb_normalized_boxes = \
anchor_projector.project_to_image_space(all_anchors, dataset,
image_shape, img_idx)
# Overlay boxes on images
anchor_objects = []
for anchor_idx in range(len(anchor_boxes_3d)):
anchor_box_3d = anchor_boxes_3d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(
anchor_box_3d, 'Anchor')
# Append to a list for visualization in VTK later
anchor_objects.append(obj_label)
for idx in range(len(ground_truth_list)):
ground_truth_obj = ground_truth_list[idx]
# Append to a list for visualization in VTK later
anchor_objects.append(ground_truth_obj)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(anchor_objects, anchor_colour_scheme)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(point_cloud)
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(vx_grid_3d)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
# vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vtk.vtkInteractorStyleTrackballCamera())
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start() # Blocking
def train_and_eval(model_config, train_config, eval_config, dataset_config):
# Dataset Configuration
dataset_config_train = DatasetBuilder.copy_config(dataset_config)
dataset_config_eval = DatasetBuilder.copy_config(dataset_config)
dataset_train = DatasetBuilder.build_kitti_dataset(dataset_config_train,
use_defaults=False)
dataset_eval = DatasetBuilder.build_kitti_dataset(dataset_config_eval,
use_defaults=False)
model_name = model_config.model_name
train_val_test = 'train'
eval_mode = eval_config.eval_mode
if eval_mode == 'train':
raise ValueError('Evaluation mode can only be set to `val` or `test`.')
# keep a copy as this will be overwritten inside
# the training loop below
max_train_iter = train_config.max_iterations
checkpoint_interval = train_config.checkpoint_interval
eval_interval = eval_config.eval_interval
if eval_interval < checkpoint_interval or \
(eval_interval % checkpoint_interval) != 0:
raise ValueError(
'Checkpoint interval (given {}) must be greater than and'
'divisible by the evaluation interval (given {}).'.format(
eval_interval, checkpoint_interval))
# Use the evaluation losses file to continue from the latest
# checkpoint
already_evaluated_ckpts = evaluator.get_evaluated_ckpts(
model_config, model_name)
if len(already_evaluated_ckpts) != 0:
current_train_iter = already_evaluated_ckpts[-1]
else:
current_train_iter = eval_interval
# while training is not finished
while current_train_iter <= max_train_iter:
# Train
with tf.Graph().as_default():
if model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=train_val_test,
dataset=dataset_train)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset_train)
else:
raise ValueError('Invalid model name {}'.format(model_name))
# overwrite the training epochs
train_config.max_iterations = current_train_iter
print('\n*************** Training ****************\n')
trainer.train(model, train_config)
current_train_iter += eval_interval
# Evaluate
with tf.Graph().as_default():
if model_name == 'mlod_model':
model = MlodModel(model_config,
train_val_test=eval_mode,
dataset=dataset_eval)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_mode,
dataset=dataset_eval)
else:
raise ValueError('Invalid model name {}'.format(model_name))
print('\n*************** Evaluating *****************\n')
evaluator.run_latest_checkpoints(model, dataset_config_eval)
print('\n************ Finished training and evaluating *************\n')
def main():
""" Converts a set of network predictions into text files required for
KITTI evaluation.
"""
##############################
# Options
##############################
checkpoint_name = 'mlod_exp_example'
# data_split = 'val'
data_split = 'val_half'
global_steps = None
# global_steps = [28000, 19000, 33000, 34000]
score_threshold = 0.1
save_2d = False # Save 2D predictions
save_3d = True # Save 2D and 3D predictions together
# Checkpoints below this are skipped
min_step = 20000
# Object Type
obj_type = 'obj'
##############################
# End of Options
##############################
# Parse experiment config
pipeline_config_file = \
mlod.root_dir() + '/data/outputs/' + checkpoint_name + \
'/' + checkpoint_name + '.config'
_, _, _, dataset_config = \
config_builder_util.get_configs_from_pipeline_file(
pipeline_config_file, is_training=False)
# Overwrite defaults
dataset_config = config_builder_util.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.aug_list = []
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Get available prediction folders
predictions_root_dir = mlod.root_dir() + '/data/outputs/' + \
checkpoint_name + '/predictions'
proposals_root_dir = predictions_root_dir + \
'/proposals_and_scores/' + dataset.data_split
print('Converting proposals from', proposals_root_dir)
if not global_steps:
global_steps = os.listdir(proposals_root_dir)
global_steps.sort(key=int)
print('Checkpoints found ', global_steps)
for step_idx in range(len(global_steps)):
global_step = global_steps[step_idx]
# Skip first checkpoint
if int(global_step) < min_step:
continue
final_predictions_dir = proposals_root_dir + \
'/' + str(global_step)
# 2D and 3D prediction directories
kitti_predictions_2d_dir = predictions_root_dir + \
'/kitti_proposals_2d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
kitti_proposals_3d_dir = predictions_root_dir + \
'/kitti_proposals_3d/' + \
dataset.data_split + '/' + \
str(score_threshold) + '/' + \
str(global_step) + '/data'
if save_2d and not os.path.exists(kitti_predictions_2d_dir):
os.makedirs(kitti_predictions_2d_dir)
if save_3d and not os.path.exists(kitti_proposals_3d_dir):
os.makedirs(kitti_proposals_3d_dir)
# Do conversion
num_samples = dataset.num_samples
num_valid_samples = 0
print('\nGlobal step:', global_step)
print('Converting proposals from:', final_predictions_dir)
if save_2d:
print('2D Detections saved to:', kitti_predictions_2d_dir)
if save_3d:
print('Proposals saved to:', kitti_proposals_3d_dir)
for sample_idx in range(num_samples):
# Print progress
sys.stdout.write('\rConverting {} / {}'.format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
prediction_file = sample_name + '.txt'
kitti_predictions_2d_file_path = kitti_predictions_2d_dir + \
'/' + prediction_file
kitti_predictions_3d_file_path = kitti_proposals_3d_dir + \
'/' + prediction_file
predictions_file_path = final_predictions_dir + \
'/' + prediction_file
# If no predictions, skip to next file
if not os.path.exists(predictions_file_path):
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
all_predictions = np.loadtxt(predictions_file_path)
# Swap l, w for predictions where w > l
swapped_indices = all_predictions[:, 4] > all_predictions[:, 3]
fixed_predictions = np.copy(all_predictions)
fixed_predictions[swapped_indices,
3] = all_predictions[swapped_indices, 4]
fixed_predictions[swapped_indices,
4] = all_predictions[swapped_indices, 3]
fixed_predictions[swapped_indices, 6] = np.pi / 2
score_filter = all_predictions[:, 7] >= score_threshold
all_predictions = fixed_predictions[score_filter]
# If no predictions, skip to next file
if len(all_predictions) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
# Project to image space
sample_name = prediction_file.split('.')[0]
img_idx = int(sample_name)
# Load image for truncation
image = Image.open(dataset.get_rgb_image_path(sample_name))
stereo_calib_p2 = calib_utils.read_calibration(
dataset.calib_dir, img_idx).p2
boxes = []
image_filter = []
for i in range(len(all_predictions)):
box_3d = all_predictions[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
stereo_calib_p2,
truncate=True,
image_size=image.size,
discard_before_truncation=False)
# Skip invalid boxes (outside image space)
if img_box is None:
image_filter.append(False)
else:
image_filter.append(True)
boxes.append(img_box)
boxes = np.asarray(boxes)
all_predictions = all_predictions[image_filter]
# If no predictions, skip to next file
if len(boxes) == 0:
if save_2d:
np.savetxt(kitti_predictions_2d_file_path, [])
if save_3d:
np.savetxt(kitti_predictions_3d_file_path, [])
continue
num_valid_samples += 1
# To keep each value in its appropriate position, an array of zeros
# (N, 16) is allocated but only values [4:16] are used
kitti_predictions = np.zeros([len(boxes), 16])
# Truncation and Occlusion are always empty (see below)
# Alpha (Not computed)
kitti_predictions[:, 3] = -10 * np.ones(
(len(kitti_predictions)), dtype=np.int32)
# 2D predictions
kitti_predictions[:, 4:8] = boxes[:, 0:4]
# 3D predictions
# (l, w, h)
kitti_predictions[:, 8] = all_predictions[:, 5]
kitti_predictions[:, 9] = all_predictions[:, 4]
kitti_predictions[:, 10] = all_predictions[:, 3]
# (x, y, z)
kitti_predictions[:, 11:14] = all_predictions[:, 0:3]
# (ry, score)
kitti_predictions[:, 14:16] = all_predictions[:, 6:8]
# Round detections to 3 decimal places
kitti_predictions = np.round(kitti_predictions, 3)
# Empty Truncation, Occlusion
kitti_empty_1 = -1 * np.ones(
(len(kitti_predictions), 2), dtype=np.int32)
# Empty 3D (x, y, z)
kitti_empty_2 = -1 * np.ones(
(len(kitti_predictions), 3), dtype=np.int32)
# Empty 3D (h, w, l)
kitti_empty_3 = -1000 * np.ones(
(len(kitti_predictions), 3), dtype=np.int32)
# Empty 3D (ry)
kitti_empty_4 = -10 * np.ones(
(len(kitti_predictions), 1), dtype=np.int32)
# Create Type Array
obj_types = [obj_type for i in range(len(kitti_predictions))]
# Stack 2D predictions text
kitti_text_2d = np.column_stack([
obj_types, kitti_empty_1, kitti_predictions[:, 3:8],
kitti_empty_2, kitti_empty_3, kitti_empty_4,
kitti_predictions[:, 15]
])
# Stack 3D predictions text
kitti_text_3d = np.column_stack(
[obj_types, kitti_empty_1, kitti_predictions[:, 3:16]])
# Save to text files
if save_2d:
np.savetxt(kitti_predictions_2d_file_path,
kitti_text_2d,
newline='\r\n',
fmt='%s')
if save_3d:
np.savetxt(kitti_predictions_3d_file_path,
kitti_text_3d,
newline='\r\n',
fmt='%s')
print('\nNum valid:', num_valid_samples)
print('Num samples:', num_samples)
def main():
"""This demo visualizes box 8C format predicted by MLOD, before
getting converted to Box 3D.
Keys:
F1: Toggle predictions
F2: Toggle easy ground truth objects (Green)
F3: Toggle medium ground truth objects (Orange)
F4: Toggle hard ground truth objects (Red)
F5: Toggle all ground truth objects (default off)
F6: Toggle 3D voxel grid
F7: Toggle point cloud
"""
##############################
# Options
##############################
mlod_score_threshold = 0.1
show_orientations = True
checkpoint_name = 'mlod_exp_8c'
global_step = None
sample_name = None
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL_HALF)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
##############################
# Setup Paths
##############################
# # # Cars # # #
# sample_name = '000050'
# sample_name = '000104'
# sample_name = '000169'
# sample_name = '000191'
# sample_name = '000360'
# sample_name = '001783'
# sample_name = '001820'
# sample_name = '006338'
# # # People # # #
# val_half split
# sample_name = '000001'
sample_name = '000005' # Easy, 1 ped
# sample_name = '000122' # Easy, 1 cyc
# sample_name = '000134' # Hard, lots of people
# sample_name = '000167' # Medium, 1 ped, 2 cycs
# sample_name = '000187' # Medium, 1 ped on left
# sample_name = '000381' # Easy, 1 ped
# sample_name = '000398' # Easy, 1 ped
# sample_name = '000401' # Hard, obscured peds
# sample_name = '000407' # Easy, 1 ped
# sample_name = '000448' # Hard, several far people
# sample_name = '000486' # Hard 2 obscured peds
# sample_name = '000509' # Easy, 1 ped
# sample_name = '000718' # Hard, lots of people
# sample_name = '002216' # Easy, 1 cyc
# Random sample
if sample_name is None:
sample_idx = np.random.randint(0, dataset.num_samples)
sample_name = dataset.sample_list[sample_idx]
img_idx = int(sample_name)
# Text files directory
predictions_and_scores_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions' + \
'/final_boxes_8c_and_scores/' + dataset.data_split
# Get checkpoint step
steps = os.listdir(predictions_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
##############################
# predictions
##############################
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
predictions_boxes_8c = predictions_and_scores[:, 0:24]
prediction_scores = predictions_and_scores[:, 24]
score_mask = prediction_scores >= mlod_score_threshold
predictions_boxes_8c = predictions_boxes_8c[score_mask]
all_vtk_box_corners = []
predictions_boxes_8c = np.reshape(predictions_boxes_8c, [-1, 3, 8])
for i in range(len(predictions_boxes_8c)):
box_8c = predictions_boxes_8c[i, :, :]
vtk_box_corners = VtkBox8c()
vtk_box_corners.set_objects(box_8c)
all_vtk_box_corners.append(vtk_box_corners)
##############################
# Ground Truth
##############################
if dataset.has_labels:
easy_gt_objs, medium_gt_objs, \
hard_gt_objs, all_gt_objs = \
demo_utils.get_gts_based_on_difficulty(dataset,
img_idx)
else:
easy_gt_objs = medium_gt_objs = hard_gt_objs = all_gt_objs = []
##############################
# Point Cloud
##############################
image_path = dataset.get_rgb_image_path(sample_name)
image = cv2.imread(image_path)
img_idx = int(sample_name)
points, point_colours = demo_utils.get_filtered_pc_and_colours(
dataset, image, img_idx)
# Voxelize the point cloud for visualization
voxel_grid = VoxelGrid()
voxel_grid.voxelize(points, voxel_size=0.1, create_leaf_layout=False)
##############################
# Visualization
##############################
# Create VtkVoxelGrid
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(voxel_grid)
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points, point_colours)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for ground truth
vtk_easy_gt_boxes, vtk_medium_gt_boxes, \
vtk_hard_gt_boxes, vtk_all_gt_boxes = \
demo_utils.create_gt_vtk_boxes(easy_gt_objs,
medium_gt_objs,
hard_gt_objs,
all_gt_objs,
show_orientations)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_box_actors = vtk.vtkAssembly()
# Create VtkBoxes for prediction boxes
for i in range(len(all_vtk_box_corners)):
# Adding labels, slows down rendering
# vtk_renderer.AddActor(all_vtk_box_corners[i].
# vtk_text_labels.vtk_actor)
vtk_box_actors.AddPart(all_vtk_box_corners[i].vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_hard_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_medium_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_easy_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_all_gt_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_box_actors)
vtk_renderer.AddActor(axes)
# Set initial properties for some actors
vtk_point_cloud.vtk_actor.GetProperty().SetPointSize(2)
vtk_voxel_grid.vtk_actor.SetVisibility(0)
vtk_all_gt_boxes.vtk_actor.SetVisibility(0)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(160.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName(
"Predictions: Step {}, Sample {}, Min Score {}".format(
global_step,
sample_name,
mlod_score_threshold,
))
vtk_render_window.SetSize(900, 600)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_box_actors, vtk_easy_gt_boxes.vtk_actor,
vtk_medium_gt_boxes.vtk_actor, vtk_hard_gt_boxes.vtk_actor,
vtk_all_gt_boxes.vtk_actor, vtk_voxel_grid.vtk_actor,
vtk_point_cloud.vtk_actor
]))
vtk_render_window_interactor.Start()
def main():
"""
Displays the bird's eye view maps for a KITTI sample.
"""
##############################
# Options
##############################
# one of ['height_priors', 'slices']
bev_generator = 'slices'
height_priors_config = \
"""
height_priors {
ground_filter_offset: 0.2
offset_filter_distance: 2.0
std_dev_multiplier: 2.0
}
"""
slices_config = \
"""
slices {
height_lo: -0.2
height_hi: 2.3
num_slices: 5
}
"""
# Use None for a random image
img_idx = None
# img_idx = 142
# img_idx = 191
show_ground_truth = True # Whether to overlay ground_truth boxes
point_cloud_source = 'lidar'
##############################
# End of Options
##############################
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
# Overwrite bev_generator
if bev_generator == 'height_priors':
text_format.Merge(height_priors_config,
dataset_config.kitti_utils_config.bev_generator)
elif bev_generator == 'slices':
text_format.Merge(slices_config,
dataset_config.kitti_utils_config.bev_generator)
else:
raise ValueError('Invalid bev_generator')
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
if img_idx is None:
img_idx = int(random.random() * dataset.num_samples)
sample_name = "{:06}".format(img_idx)
print('=== Showing BEV maps for image: {}.png ==='.format(sample_name))
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
kitti_utils = dataset.kitti_utils
point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx,
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
height_maps = np.array(bev_images.get("height_maps"))
density_map = np.array(bev_images.get("density_map"))
box_points, box_points_norm = [None, None]
if show_ground_truth:
# Get projected boxes
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_objs = obj_labels
label_boxes = []
for label in filtered_objs:
box = box_3d_encoder.object_label_to_box_3d(label)
label_boxes.append(box)
label_boxes = np.array(label_boxes)
box_points, box_points_norm = box_3d_projector.project_to_bev(
label_boxes, [[-40, 40], [0, 70]])
rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16)
img_x_start = 60
img_y_start = 330
img_x = img_x_start
img_y = img_y_start
img_w = 400
img_h = 350
img_titlebar_h = 20
# Show images
vis_utils.cv2_show_image("Image",
image,
size_wh=rgb_img_size,
location_xy=(img_x, 0))
# Height maps
for map_idx in range(len(height_maps)):
height_map = height_maps[map_idx]
height_map = draw_boxes(height_map, box_points_norm)
vis_utils.cv2_show_image("Height Map {}".format(map_idx),
height_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
img_x += img_w
# Wrap around
if (img_x + img_w) > 1920:
img_x = img_x_start
img_y += img_h + img_titlebar_h
# Density map
density_map = draw_boxes(density_map, box_points_norm)
vis_utils.cv2_show_image("Density Map",
density_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
cv2.waitKey()
def generate_fake_dataset():
return DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_UNITTEST)
def main():
"""Plots detection errors for xyz, lwh, ry, and shows 3D IoU with
ground truth boxes
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL_HALF,
use_defaults=True)
difficulty = 2
# Full path to kitti predictions
# (e.g. '.../data/outputs/mlod_exp_example/predictions/'
# 'kitti_predictions_3d/val/0.1/100000/data'
predictions_data_path = 'path_to_detections/data'
# Loop through split and save ious and errors
all_3d_ious = []
all_errors = []
for sample_idx in range(dataset.num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx + 1,
dataset.num_samples))
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
# Get filtered ground truth
all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx)
all_gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs,
difficulty=difficulty)
pred_objs = obj_utils.read_labels(predictions_data_path, img_idx)
##############################
# Check IoUs
##############################
if len(all_gt_objs) > 0 and \
pred_objs is not None and len(pred_objs) > 0:
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in pred_objs
]
# Convert to iou format
gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
all_gt_boxes_3d)
pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
pred_boxes_3d)
max_ious_3d = np.zeros(len(all_gt_objs))
max_iou_pred_indices = -np.ones(len(all_gt_objs))
for gt_obj_idx in range(len(all_gt_objs)):
gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx]
ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt,
pred_objs_iou_fmt)
max_iou_3d = np.amax(ious_3d)
max_ious_3d[gt_obj_idx] = max_iou_3d
if max_iou_3d > 0.0:
max_iou_pred_indices[gt_obj_idx] = np.argmax(ious_3d)
for gt_obj_idx in range(len(all_gt_objs)):
max_iou_pred_idx = int(max_iou_pred_indices[gt_obj_idx])
if max_iou_pred_idx >= 0:
error = all_gt_boxes_3d[gt_obj_idx] - \
pred_boxes_3d[max_iou_pred_idx]
all_errors.append(error)
all_3d_ious.extend(max_ious_3d)
print('Done')
all_errors = np.asarray(all_errors)
# Plot Data Histograms
f, ax_arr = plt.subplots(3, 3)
xyzlwh_bins = 51
ry_bins = 31
iou_bins = 51
# xyz
ax_arr[0, 0].hist(all_errors[:, 0],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[0, 1].hist(all_errors[:, 1],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[0, 2].hist(all_errors[:, 2],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
# lwh
ax_arr[1, 0].hist(all_errors[:, 3],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[1, 1].hist(all_errors[:, 4],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[1, 2].hist(all_errors[:, 5],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
# orientation
ax_arr[2, 0].hist(all_errors[:, 6], ry_bins, facecolor='green', alpha=0.75)
# iou
ax_arr[2, 2].hist(all_3d_ious, iou_bins, facecolor='green', alpha=0.75)
plt.show()
def main():
"""This demo shows RPN proposals and MLOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val'
fig_size = (10, 6.1)
rpn_score_threshold = 0.1
mlod_score_threshold = 0.1
# Flag for projecting the 3D boxes to image space
# in tensor format (for testing purposes)
test_img_tensor_projection = False
gt_classes = ['Pedestrian', 'Cyclist']
# gt_classes = ['Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = 44000
checkpoint_name = 'mlod_fpn_people'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False
draw_orientations_on_pred = False
# Draw 2D bounding boxes
draw_projected_2d_boxes = False
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = False
iou_3d = False
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
# Setup Paths
predictions_dir = mlod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, mlod_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step,
mlod_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
for sample_idx in range(dataset.num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'. format(
sample_idx + 1,
dataset.num_samples,
avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step,
sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
mlod_score_mask = prediction_scores >= 0.1
mlod_show_mask = mlod_score_mask
prediction_boxes_3d = prediction_boxes_3d[mlod_show_mask]
prediction_scores = prediction_scores[mlod_show_mask]
prediction_class_indices = \
prediction_class_indices[mlod_show_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
dataset.has_labels = False
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir,
img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
if test_img_tensor_projection:
proposal_boxes = demo_utils.tf_project_to_image_space(
proposals_as_anchors, calib_p2, image_size, img_idx)
else:
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs,
calib_p2,
num_of_proposals,
proposal_objs,
proposal_boxes,
prop_2d_axes,
prop_3d_axes,
draw_orientations_on_prop)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d, calib_p2,
truncate=True, image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
prop_2d_axes,
prop_3d_axes,
draw_score,
draw_iou,
gt_classes,
draw_orientations_on_pred,
iou_3d)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_2d_axes,
pred_3d_axes,
draw_score,
draw_iou,
gt_classes,
draw_orientations_on_pred,
iou_3d)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs,
calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices,
final_boxes_2d,
pred_3d_axes,
draw_score,
draw_iou,
gt_classes,
draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def main():
"""
Visualization for comparison of anchor filtering with
2D vs 3D integral images
Keys:
F1: Toggle 3D integral image filtered anchors
F2: Toggle 2D integral image filtered anchors
F3: Toggle 2D integral image empty anchors
"""
anchor_2d_colour_scheme = {"Anchor": (0, 0, 255)} # Blue
anchor_3d_colour_scheme = {"Anchor": (0, 255, 0)} # Green
anchor_unfiltered_colour_scheme = {"Anchor": (255, 0, 255)} # Purple
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_TRAINVAL)
sample_name = "000001"
img_idx = int(sample_name)
print("Showing anchors for sample {}".format(sample_name))
# Options
# These clusters are from the trainval set and give more 2D anchors than 3D
clusters = np.array([[3.55, 1.835, 1.525], [4.173, 1.69, 1.49]])
anchor_stride = [3.0, 3.0]
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
area_extents = np.array([[-40, 40], [-5, 3], [0, 70]])
anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
# Generate anchors
start_time = time.time()
anchor_boxes_3d = anchor_3d_generator.generate(area_3d=area_extents,
anchor_3d_sizes=clusters,
anchor_stride=anchor_stride,
ground_plane=ground_plane)
end_time = time.time()
print("Anchors generated in {} s".format(end_time - start_time))
# Get point cloud
point_cloud = obj_utils.get_stereo_point_cloud(img_idx, dataset.calib_dir,
dataset.disp_dir)
ground_offset_dist = 0.2
offset_dist = 2.0
# Filter points within certain xyz range and offset from ground plane
# Filter points within 0.2m of the road plane
slice_filter = dataset.kitti_utils.create_slice_filter(point_cloud,
area_extents,
ground_plane,
ground_offset_dist,
offset_dist)
points = np.array(point_cloud).T
points = points[slice_filter]
anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
# Create 2D voxel grid
vx_grid_2d = voxel_grid_2d.VoxelGrid2D()
vx_grid_2d.voxelize_2d(points, 0.1, area_extents)
# Create 3D voxel grid
vx_grid_3d = voxel_grid.VoxelGrid()
vx_grid_3d.voxelize(points, 0.1, area_extents)
# Filter the boxes here!
start_time = time.time()
empty_filter_2d = anchor_filter.get_empty_anchor_filter_2d(
anchors=anchors,
voxel_grid_2d=vx_grid_2d,
density_threshold=1)
anchors_2d = anchor_boxes_3d[empty_filter_2d]
end_time = time.time()
print("2D Anchors filtered in {} s".format(end_time - start_time))
print("Number of 2D anchors remaining: %d" % (anchors_2d.shape[0]))
unfiltered_anchors_2d = anchor_boxes_3d[np.logical_not(empty_filter_2d)]
# 3D filtering
start_time = time.time()
empty_filter_3d = anchor_filter.get_empty_anchor_filter(
anchors=anchors,
voxel_grid_3d=vx_grid_3d,
density_threshold=1)
anchor_boxes_3d = anchor_boxes_3d[empty_filter_3d]
end_time = time.time()
print("3D Anchors filtered in {} s".format(end_time - start_time))
print("Number of 3D anchors remaining: %d" % (anchor_boxes_3d.shape[0]))
anchor_2d_objects = []
for anchor_idx in range(len(anchors_2d)):
anchor = anchors_2d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor')
# Append to a list for visualization in VTK later
anchor_2d_objects.append(obj_label)
anchor_3d_objects = []
for anchor_idx in range(len(anchor_boxes_3d)):
anchor = anchor_boxes_3d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor')
# Append to a list for visualization in VTK later
anchor_3d_objects.append(obj_label)
unfiltered_anchor_objects = []
for anchor_idx in range(len(unfiltered_anchors_2d)):
anchor = unfiltered_anchors_2d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor, 'Anchor')
# Append to a list for visualization in VTK later
unfiltered_anchor_objects.append(obj_label)
# Create VtkAxes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Create VtkBoxes for boxes
vtk_2d_boxes = VtkBoxes()
vtk_2d_boxes.set_objects(anchor_2d_objects, anchor_2d_colour_scheme)
vtk_3d_boxes = VtkBoxes()
vtk_3d_boxes.set_objects(anchor_3d_objects, anchor_3d_colour_scheme)
vtk_unfiltered_boxes = VtkBoxes()
vtk_unfiltered_boxes.set_objects(unfiltered_anchor_objects,
anchor_unfiltered_colour_scheme)
vtk_voxel_grid = VtkVoxelGrid()
vtk_voxel_grid.set_voxels(vx_grid_3d)
vtk_voxel_grid_2d = VtkVoxelGrid()
vtk_voxel_grid_2d.set_voxels(vx_grid_2d)
# Create Voxel Grid Renderer in bottom half
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.AddActor(vtk_2d_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_3d_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_unfiltered_boxes.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid.vtk_actor)
vtk_renderer.AddActor(vtk_voxel_grid_2d.vtk_actor)
vtk_renderer.AddActor(axes)
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_2d_boxes.vtk_actor,
vtk_3d_boxes.vtk_actor,
vtk_unfiltered_boxes.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def setUpClass(cls):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
cls.label_dir = cls.dataset.label_dir
def main():
"""Shows a flipped sample in 3D
"""
# Create Dataset
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL)
##############################
# Options
##############################
# sample_name = "000191"
sample_name = "000104"
img_idx = int(sample_name)
print("Showing anchors for sample {}".format(sample_name))
##############################
# Load Sample Data
##############################
ground_plane = obj_utils.get_road_plane(img_idx, dataset.planes_dir)
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = [image.shape[1], image.shape[0]]
# Get point cloud
point_cloud = obj_utils.get_depth_map_point_cloud(img_idx,
dataset.calib_dir,
dataset.depth_dir,
image_shape)
points = np.array(point_cloud).T
# Ground truth
gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
# Filter ground truth
gt_labels = dataset.kitti_utils.filter_labels(gt_labels)
##############################
# Flip stuff
##############################
image_flipped = np.fliplr(image)
# Flip ground plane coeff (x)
ground_plane_flipped = np.copy(ground_plane)
ground_plane_flipped[0] = -ground_plane_flipped[0]
# Flip 3D points
points_flipped = kitti_aug.flip_points(points)
# Get point cloud colours
point_colours_flipped = project_flipped_img_to_point_cloud(
points_flipped, image_flipped, dataset.calib_dir, img_idx)
# Flip ground truth boxes
gt_labels_flipped = [
kitti_aug.flip_label_in_3d_only(obj) for obj in gt_labels
]
##############################
# VTK Visualization
##############################
# Axes
axes = vtk.vtkAxesActor()
axes.SetTotalLength(5, 5, 5)
# Point cloud
vtk_point_cloud = VtkPointCloud()
vtk_point_cloud.set_points(points_flipped,
point_colours=point_colours_flipped)
# # Ground Truth Boxes
vtk_boxes = VtkBoxes()
vtk_boxes.set_objects(gt_labels_flipped,
VtkBoxes.COLOUR_SCHEME_KITTI,
show_orientations=True)
# Renderer
vtk_renderer = vtk.vtkRenderer()
vtk_renderer.SetBackground(0.2, 0.3, 0.4)
# Add Actors to Rendered
vtk_renderer.AddActor(axes)
vtk_renderer.AddActor(vtk_point_cloud.vtk_actor)
vtk_renderer.AddActor(vtk_boxes.vtk_actor)
# Setup Camera
current_cam = vtk_renderer.GetActiveCamera()
current_cam.Pitch(170.0)
current_cam.Roll(180.0)
# Zooms out to fit all points on screen
vtk_renderer.ResetCamera()
# Zoom in slightly
current_cam.Zoom(2.5)
# Reset the clipping range to show all points
vtk_renderer.ResetCameraClippingRange()
# Setup Render Window
vtk_render_window = vtk.vtkRenderWindow()
vtk_render_window.SetWindowName("Anchors")
vtk_render_window.SetSize(900, 500)
vtk_render_window.AddRenderer(vtk_renderer)
# Setup custom interactor style, which handles mouse and key events
vtk_render_window_interactor = vtk.vtkRenderWindowInteractor()
vtk_render_window_interactor.SetRenderWindow(vtk_render_window)
vtk_render_window_interactor.SetInteractorStyle(
vis_utils.ToggleActorsInteractorStyle([
vtk_point_cloud.vtk_actor,
]))
# Render in VTK
vtk_render_window.Render()
vtk_render_window_interactor.Start()
def main():
"""
Calculates clusters for each class
Returns:
all_clusters: list of clusters for each class
all_std_devs: list of cluster standard deviations for each class
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
# Calculate the remaining clusters
# Load labels corresponding to the sample list for clustering
sample_list = dataset.load_sample_names(dataset.cluster_split)
all_dims = []
num_samples = len(sample_list)
for sample_idx in range(num_samples):
sys.stdout.write("\rClustering labels {} / {}".format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = sample_list[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_lwh = LabelClusterUtils._filter_labels_by_class(
obj_labels, dataset.classes)
if filtered_lwh[0]:
all_dims.extend(filtered_lwh[0])
all_dims = np.array(all_dims)
print("\nFinished reading labels, clustering data...\n")
# Print 3 decimal places
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
# Calculate average cluster
k_means = KMeans(n_clusters=1,
random_state=0).fit(all_dims)
cluster_centre = k_means.cluster_centers_[0]
# Calculate std. dev
std_dev = np.std(all_dims, axis=0)
# Calculate 2 and 3 standard deviations below the mean
two_sigma_length_lo = cluster_centre[0] - 2 * std_dev[0]
three_sigma_length_lo = cluster_centre[0] - 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
small_mask_2 = all_dims[:, 0] < two_sigma_length_lo
small_dims_2 = all_dims[small_mask_2]
small_mask_3 = all_dims[:, 0] < three_sigma_length_lo
small_dims_3 = all_dims[small_mask_3]
small_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(small_dims_2)
small_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(small_dims_3)
small_std_dev_2 = np.std(small_dims_2, axis=0)
small_std_dev_3 = np.std(small_dims_3, axis=0)
print('small_k_means_2:', small_k_means_2.cluster_centers_)
print('small_k_means_3:', small_k_means_3.cluster_centers_)
print('small_std_dev_2:', small_std_dev_2)
print('small_std_dev_3:', small_std_dev_3)
# Calculate 2 and 3 standard deviations above the mean
two_sigma_length_hi = cluster_centre[0] + 2 * std_dev[0]
three_sigma_length_hi = cluster_centre[0] + 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
large_mask_2 = all_dims[:, 0] > two_sigma_length_hi
large_dims_2 = all_dims[large_mask_2]
large_mask_3 = all_dims[:, 0] > three_sigma_length_hi
large_dims_3 = all_dims[large_mask_3]
large_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(large_dims_2)
large_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(large_dims_3)
large_std_dev_2 = np.std(large_dims_2, axis=0)
large_std_dev_3 = np.std(large_dims_3, axis=0)
print('large_k_means_2:', large_k_means_2.cluster_centers_)
print('large_k_means_3:', large_k_means_3.cluster_centers_)
print('large_std_dev_2:', large_std_dev_2)
print('large_std_dev_3:', large_std_dev_3)
dataset_config.data_split = 'trainval'
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = True
# Set CUDA device id
os.environ['CUDA_VISIBLE_DEVICES'] = args.device
# Convert to object to overwrite repeated fields
dataset_config = config_builder.proto_to_obj(dataset_config)
# Remove augmentation during evaluation
dataset_config.aug_list = ['flipping']
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Convert to object to overwrite repeated fields
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
index = 76
samples = dataset.load_samples([index])
depth_map= samples[0].get(constants.KEY_DPT_INPUT)
print(depth_map.shape)
image_input = samples[0].get(constants.KEY_IMAGE_INPUT)
