def main():
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = "trainval"
unittest_dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
gen_label_clusters.main(unittest_dataset)
gen_mini_batches.main(unittest_dataset)
def setUpClass(cls):
# Initialize the Kitti dataset
test_dir = tests.test_path()
# Get the unittest-kitti dataset
dataset_builder = DatasetBuilder()
cls.dataset = dataset_builder.build_kitti_dataset(
dataset_builder.KITTI_UNITTEST)
cls.log_dir = test_dir + '/logs'
cls.bev_vgg_cls = vgg.BevVggClassification()
def get_fake_dataset(self, data_split, directory):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.PANOPTIC_UNITTEST)
# Overwrite config values
dataset_config.data_split = data_split
dataset_config.dataset_dir = directory
dataset = DatasetBuilder.build_panoptic_dataset(dataset_config)
return dataset
def test_project_to_image_space_tensors(self):
anchors = np.asarray([[0, 0, 3, 2, 0, 6], [3, 0, 3, 2, 0, 2]],
dtype=np.float64)
img_idx = int('000217')
img_shape = [375, 1242]
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
dataset_config.data_split = 'train'
dataset_config.dataset_dir = tests.test_path() + \
"/datasets/Kitti/object"
dataset = DatasetBuilder().build_kitti_dataset(dataset_config)
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
# Project the 3D points in numpy space
img_corners, img_corners_norm = anchor_projector.project_to_image_space(
anchors, stereo_calib_p2, img_shape)
# convert the required params to tensors
tf_stereo_calib_p2 = tf.convert_to_tensor(stereo_calib_p2,
dtype=tf.float32)
tf_anchors = tf.convert_to_tensor(anchors, dtype=tf.float32)
tf_img_shape = tf.convert_to_tensor(img_shape, dtype=tf.float32)
# Project the 3D points in tensor space
img_corners_tensor, img_corners_norm_tensor = \
anchor_projector.tf_project_to_image_space(tf_anchors,
tf_stereo_calib_p2,
tf_img_shape)
sess = tf.Session()
with sess.as_default():
img_corners_out = img_corners_tensor.eval()
img_corners_norm_out = img_corners_norm_tensor.eval()
np.testing.assert_allclose(img_corners,
img_corners_out,
atol=1e-04,
err_msg='Incorrect corner projection')
np.testing.assert_allclose(
img_corners_norm,
img_corners_norm_out,
atol=1e-04,
err_msg='Incorrect normalized corner projection')
def test_data_splits(self):
bad_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_UNITTEST)
# Test invalid splits
bad_config.data_split = "bad"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "train"
bad_config.data_split = "training"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "val"
bad_config.data_split = "validation"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Should be "test"
bad_config.data_split = "testing"
self.assertRaises(ValueError, KittiDataset, bad_config)
# Train split
train_dataset = self.get_fake_dataset('train', self.fake_kitti_dir)
self.assertEqual(train_dataset.num_samples, 7)
# Validation split
validation_dataset = self.get_fake_dataset('val', self.fake_kitti_dir)
self.assertEqual(validation_dataset.num_samples, 6)
# Train + validation split
trainval_dataset = self.get_fake_dataset('trainval',
self.fake_kitti_dir)
self.assertEqual(trainval_dataset.num_samples, 13)
# Test split
test_dataset = self.get_fake_dataset('test', self.fake_kitti_dir)
self.assertEqual(test_dataset.num_samples, 10)
def setUpClass(cls):
pipeline_config = pipeline_pb2.NetworkPipelineConfig()
dataset_config = pipeline_config.dataset_config
config_path = pplp.root_dir() + '/configs/unittest_model.config'
cls.model_config = config_build.get_model_config_from_file(config_path)
dataset_config.MergeFrom(DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
def inference(model_config, eval_config,
dataset_config, data_split,
ckpt_indices):
# Overwrite the defaults
dataset_config = config_builder.proto_to_obj(dataset_config)
dataset_config.data_split = data_split
dataset_config.data_split_dir = 'training'
if data_split == 'test':
dataset_config.data_split_dir = 'testing'
eval_config.eval_mode = 'test'
eval_config.evaluate_repeatedly = False
dataset_config.has_labels = False
# Enable this to see the actually memory being used
eval_config.allow_gpu_mem_growth = True
eval_config = config_builder.proto_to_obj(eval_config)
# Grab the checkpoint indices to evaluate
eval_config.ckpt_indices = ckpt_indices
# Remove augmentation during evaluation in test mode
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Overwrite repeated field
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_config.eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
model_evaluator.run_latest_checkpoints()
def setUp(self):
tf.test.TestCase.setUp(self)
test_pipeline_config_path = pplp.root_dir() + \
'/configs/unittest_pipeline.config'
self.model_config, self.train_config, _, dataset_config = \
config_builder.get_configs_from_pipeline_file(
test_pipeline_config_path, is_training=True)
# Generate dataset
self.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST,
use_defaults=False,
new_cfg=dataset_config)
def main(dataset=None):
if not dataset:
dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_TRAIN)
label_cluster_utils = dataset.kitti_utils.label_cluster_utils
print("Generating clusters in {}/{}".format(label_cluster_utils.data_dir,
dataset.data_split))
clusters, std_devs = dataset.get_cluster_info()
print("Clusters generated")
print("classes: {}".format(dataset.classes))
print("num_clusters: {}".format(dataset.num_clusters))
print("all_clusters:\n {}".format(clusters))
print("all_std_devs:\n {}".format(std_devs))
def train(model_config, train_config, dataset_config):
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
train_val_test = 'train'
model_name = model_config.model_name
with tf.Graph().as_default():
if model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
elif model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=train_val_test,
dataset=dataset)
else:
raise ValueError('Invalid model_name')
trainer.train(model, train_config)
def main():
"""
Displays the bird's eye view maps for a KITTI sample.
"""
##############################
# Options
##############################
bev_generator = 'slices'
slices_config = \
"""
slices {
height_lo: 0.9 # -0.2
height_hi: 1.0 # 2.3
num_slices: 1 # 5
}
"""
# Use None for a random image
# img_idx = None
img_idx = 571
# img_idx = 191
show_ground_truth = True # Whether to overlay ground_truth boxes
point_cloud_source = 'lidar'
##############################
# End of Options
##############################
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
# Overwrite bev_generator
if bev_generator == 'slices':
text_format.Merge(slices_config,
dataset_config.kitti_utils_config.bev_generator)
else:
raise ValueError('Invalid bev_generator')
print('dataset_config = ', dataset_config)
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
if img_idx is None:
img_idx = int(random.random() * dataset.num_samples)
sample_name = "{:06}".format(img_idx)
print('=== Showing BEV maps for image: {}.png ==='.format(sample_name))
# Load image
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_shape = image.shape[0:2]
kitti_utils = dataset.kitti_utils
point_cloud = kitti_utils.get_point_cloud(point_cloud_source, img_idx,
image_shape)
ground_plane = kitti_utils.get_ground_plane(sample_name)
bev_images = kitti_utils.create_bev_maps(point_cloud, ground_plane)
height_maps = np.array(bev_images.get("height_maps"))
density_map = np.array(bev_images.get("density_map"))
box_points, box_points_norm = [None, None]
if show_ground_truth:
# Get projected boxes
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_objs = dataset.kitti_utils.filter_labels(
obj_labels, ['Car', 'Pedestrian'])
# filtered_objs = dataset.kitti_utils.filter_labels(
# obj_labels, ['Car', 'Van', 'Pedestrian', 'Cyclist'])
print('filtered_objs = ', filtered_objs)
label_boxes = []
for label in filtered_objs:
box = box_3d_encoder.object_label_to_box_3d(label)
label_boxes.append(box)
label_boxes = np.array(label_boxes)
box_points, box_points_norm = box_3d_projector.project_to_bev(
label_boxes, [[-40, 40], [0, 70]])
rgb_img_size = (np.array((1242, 375)) * 0.75).astype(np.int16)
img_x_start = 60
img_y_start = 330
img_x = img_x_start
img_y = img_y_start
img_w = 400
img_h = 350
img_titlebar_h = 20
# Show images
vis_utils.cv2_show_image("Image",
image,
size_wh=rgb_img_size,
location_xy=(img_x, 0))
# Height maps
for map_idx in range(len(height_maps)):
height_map = height_maps[map_idx]
height_map = draw_boxes(height_map, box_points_norm)
vis_utils.cv2_show_image("Height Map {}".format(map_idx),
height_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
img_x += img_w
# Wrap around
if (img_x + img_w) > 1920:
img_x = img_x_start
img_y += img_h + img_titlebar_h
# Density map
density_map = draw_boxes(density_map, box_points_norm)
vis_utils.cv2_show_image("Density Map",
density_map,
size_wh=(img_w, img_h),
location_xy=(img_x, img_y))
cv2.waitKey()
def evaluate(model_config, eval_config, dataset_config):
# Parse eval config
eval_mode = eval_config.eval_mode
if eval_mode not in ['val', 'test']:
raise ValueError('Evaluation mode can only be set to `val` or `test`')
evaluate_repeatedly = eval_config.evaluate_repeatedly
# Parse dataset config
data_split = dataset_config.data_split
if data_split == 'train':
dataset_config.data_split_dir = 'training'
dataset_config.has_labels = True
elif data_split.startswith('val'):
dataset_config.data_split_dir = 'training'
# Don't load labels for val split when running in test mode
if eval_mode == 'val':
dataset_config.has_labels = True
elif eval_mode == 'test':
dataset_config.has_labels = False
elif data_split == 'test':
dataset_config.data_split_dir = 'testing'
dataset_config.has_labels = False
else:
raise ValueError('Invalid data split', data_split)
# Convert to object to overwrite repeated fields
dataset_config = config_builder.proto_to_obj(dataset_config)
# Remove augmentation during evaluation
dataset_config.aug_list = []
# Build the dataset object
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup the model
model_name = model_config.model_name
# Convert to object to overwrite repeated fields
model_config = config_builder.proto_to_obj(model_config)
# Switch path drop off during evaluation
model_config.path_drop_probabilities = [1.0, 1.0]
with tf.Graph().as_default():
if model_name == 'avod_model':
model = AvodModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
elif model_name == 'rpn_model':
model = RpnModel(model_config,
train_val_test=eval_mode,
dataset=dataset)
else:
raise ValueError('Invalid model name {}'.format(model_name))
model_evaluator = Evaluator(model, dataset_config, eval_config)
if evaluate_repeatedly:
model_evaluator.repeated_checkpoint_run()
else:
model_evaluator.run_latest_checkpoints()
def setUpClass(cls):
cls.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST)
cls.mb_utils = cls.dataset.kitti_utils.mini_batch_utils
def main():
"""Generates anchors info which is used for mini batch sampling.
Processing on 'Cars' can be split into multiple processes, see the Options
section for configuration.
Args:
dataset: KittiDataset (optional)
If dataset is provided, only generate info for that dataset.
If no dataset provided, generates info for all 3 classes.
"""
kitti_dataset_config_path = pplp.root_dir() + \
'/configs/mb_preprocessing/rpn_pedestrians.config'
##############################
# Options
##############################
# Serial vs parallel processing
in_parallel = False
process_ped = True
# Number of child processes to fork, samples will
# be divided evenly amongst the processes (in_parallel must be True)
num_ped_children = 8
##############################
# Dataset setup
##############################
if process_ped:
ped_dataset = DatasetBuilder.load_dataset_from_config(
kitti_dataset_config_path)
##############################
# Serial Processing
##############################
if not in_parallel:
if process_ped:
do_preprocessing(ped_dataset, None)
print('All Done (Serial)')
##############################
# Parallel Processing
##############################
else:
# List of all child pids to wait on
all_child_pids = []
# Pedestrians
if process_ped:
ped_indices_split = split_indices(ped_dataset, num_ped_children)
split_work(all_child_pids, ped_dataset, ped_indices_split,
num_ped_children)
# Wait to child processes to finish
print('num children:', len(all_child_pids))
for i, child_pid in enumerate(all_child_pids):
os.waitpid(child_pid, 0)
print('All Done (Parallel)')
def main():
"""
Calculates and prints the mean values for the RGB channels in a dataset
"""
dataset_builder = DatasetBuilder()
dataset = dataset_builder.build_kitti_dataset(
dataset_builder.KITTI_TRAIN
# dataset_builder.KITTI_TRAIN_MINI
)
# Options
debug_print = True
get_bev_mean = False
# Dataset values
dataset_utils = dataset.kitti_utils
num_samples = dataset.num_samples
clusters, _ = dataset.get_cluster_info()
num_bev_maps = len(clusters) + 1 # Height Maps + Density Map
pixels_sum = np.zeros(3) # RGB
bev_sum = np.zeros(num_bev_maps)
for sample_idx in range(num_samples):
sample_name = dataset.sample_names[sample_idx]
image_path = dataset.get_rgb_image_path(sample_name)
image = np.asarray(Image.open(image_path))
pixels_r = np.mean(image[:, :, 0])
pixels_g = np.mean(image[:, :, 1])
pixels_b = np.mean(image[:, :, 2])
pixel_means = np.stack((pixels_r, pixels_g, pixels_b))
pixels_sum += pixel_means
if get_bev_mean:
bev_images = dataset_utils.create_bev_maps(sample_name,
source='lidar')
height_maps = np.asarray(bev_images['height_maps'])
density_map = np.asarray(bev_images['density_map'])
height_means = [np.mean(height_map) for height_map in height_maps]
density_mean = np.mean(density_map)
bev_means = np.stack((*height_means, density_mean))
bev_sum += bev_means
if debug_print:
debug_string = '{} / {}, Sample {}, pixel_means {}'.format(
sample_idx + 1, num_samples, sample_name, pixel_means)
if get_bev_mean:
debug_string += ' density_means {}'.format(bev_means)
print(debug_string)
print("Dataset: {}, split: {}".format(dataset.name, dataset.data_split))
print("Image mean: {}".format(pixels_sum / num_samples))
if get_bev_mean:
print("BEV mean: {}".format(bev_sum / num_samples))
def main():
"""
Calculates clusters for each class
Returns:
all_clusters: list of clusters for each class
all_std_devs: list of cluster standard deviations for each class
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAIN)
# Calculate the remaining clusters
# Load labels corresponding to the sample list for clustering
sample_list = dataset.load_sample_names(dataset.cluster_split)
all_dims = []
num_samples = len(sample_list)
for sample_idx in range(num_samples):
sys.stdout.write("\rClustering labels {} / {}".format(
sample_idx + 1, num_samples))
sys.stdout.flush()
sample_name = sample_list[sample_idx]
img_idx = int(sample_name)
obj_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
filtered_lwh = LabelClusterUtils._filter_labels_by_class(
obj_labels, dataset.classes)
if filtered_lwh[0]:
all_dims.extend(filtered_lwh[0])
all_dims = np.array(all_dims)
print("\nFinished reading labels, clustering data...\n")
# Print 3 decimal places
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
# Calculate average cluster
k_means = KMeans(n_clusters=1, random_state=0).fit(all_dims)
cluster_centre = k_means.cluster_centers_[0]
# Calculate std. dev
std_dev = np.std(all_dims, axis=0)
# Calculate 2 and 3 standard deviations below the mean
two_sigma_length_lo = cluster_centre[0] - 2 * std_dev[0]
three_sigma_length_lo = cluster_centre[0] - 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
small_mask_2 = all_dims[:, 0] < two_sigma_length_lo
small_dims_2 = all_dims[small_mask_2]
small_mask_3 = all_dims[:, 0] < three_sigma_length_lo
small_dims_3 = all_dims[small_mask_3]
small_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(small_dims_2)
small_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(small_dims_3)
small_std_dev_2 = np.std(small_dims_2, axis=0)
small_std_dev_3 = np.std(small_dims_3, axis=0)
print('small_k_means_2:', small_k_means_2.cluster_centers_)
print('small_k_means_3:', small_k_means_3.cluster_centers_)
print('small_std_dev_2:', small_std_dev_2)
print('small_std_dev_3:', small_std_dev_3)
# Calculate 2 and 3 standard deviations above the mean
two_sigma_length_hi = cluster_centre[0] + 2 * std_dev[0]
three_sigma_length_hi = cluster_centre[0] + 3 * std_dev[0]
# Remove all labels with length above two std dev
# from the mean and re-cluster
large_mask_2 = all_dims[:, 0] > two_sigma_length_hi
large_dims_2 = all_dims[large_mask_2]
large_mask_3 = all_dims[:, 0] > three_sigma_length_hi
large_dims_3 = all_dims[large_mask_3]
large_k_means_2 = KMeans(n_clusters=1, random_state=0).fit(large_dims_2)
large_k_means_3 = KMeans(n_clusters=1, random_state=0).fit(large_dims_3)
large_std_dev_2 = np.std(large_dims_2, axis=0)
large_std_dev_3 = np.std(large_dims_3, axis=0)
print('large_k_means_2:', large_k_means_2.cluster_centers_)
print('large_k_means_3:', large_k_means_3.cluster_centers_)
print('large_std_dev_2:', large_std_dev_2)
print('large_std_dev_3:', large_std_dev_3)
def main():
"""This demo shows RPN proposals and AVOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config = DatasetBuilder.merge_defaults(dataset_config)
dataset_config.data_split = 'val'
fig_size = (10, 6.1)
rpn_score_threshold = 0.1
avod_score_threshold = 0.1
# gt_classes = ['Car']
gt_classes = ['Pedestrian']
# gt_classes = ['Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = 115000 # None
checkpoint_name = 'pplp_pedestrian_kitti'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = True # To draw both proposal and predcition bounding boxes
draw_predictions_separate = False
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = True
draw_orientations_on_pred = True
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Draw pointclouds
draw_point_cloud = True
point_cloud_source = 'lidar'
slices_config = \
"""
slices {
height_lo: -1.0 # -0.2
height_hi: 2 # 2.3
num_slices: 1 # 5
}
"""
print('slices_config = ', slices_config)
text_format.Merge(slices_config,
dataset_config.kitti_utils_config.bev_generator)
# Save images for samples with no detections
save_empty_images = False
draw_score = True
draw_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config,
use_defaults=False)
# Setup Paths
predictions_dir = pplp.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, avod_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, avod_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
# for sample_idx in range(dataset.num_samples):
for sample_idx in [6]:
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals,
proposal_objs, proposal_boxes, prop_2d_axes,
prop_3d_axes, draw_orientations_on_prop)
if draw_point_cloud:
# Filter the useful pointclouds from all points
kitti_utils = dataset.kitti_utils
x, y, z, i = calib_utils.read_lidar(dataset.velo_dir, img_idx)
point_cloud = np.vstack((x, y, z))
# pts = calib_utils.lidar_to_cam_frame(pts, frame_calib)
# point_cloud = kitti_utils.get_point_cloud(
# point_cloud_source, img_idx, image_size)
point_cloud = kitti_utils.get_point_cloud(
point_cloud_source, img_idx, image_size)
print('point_cloud = ', point_cloud)
ground_plane = kitti_utils.get_ground_plane(sample_name)
all_points = np.transpose(point_cloud)
# print('dataset_config.kitti_utils_config.bev_generator.\
# slices.height_lo = ', dataset_config.kitti_utils_config.bev_generator.slices.height_lo)
height_lo = dataset_config.kitti_utils_config.bev_generator.slices.height_lo
height_hi = dataset_config.kitti_utils_config.bev_generator.slices.height_hi
# config = dataset.config.kitti_utils_config
# print('config = ', config)
area_extents = [[0., 70.], [-40, 40], [-1.1, -1]]
# ?,x,?
area_extents = np.reshape(area_extents, (3, 2))
print('area_extents = ', area_extents)
print('ground_plane = ', ground_plane)
slice_filter = get_point_filter(point_cloud,
area_extents,
ground_plane=None,
offset_dist=2.0)
# slice_filter = kitti_utils.create_slice_filter(
# point_cloud,
# area_extents,
# ground_plane,
# height_lo,
# height_hi)
# Apply slice filter
slice_points = all_points[slice_filter]
print('slice_points = ', slice_points)
# print('slice_points =', slice_points)
# Project the useful pointclouds on 2D image
point_2d = obj_utils.get_lidar_points_on_2D_image(
img_idx, dataset.calib_dir, dataset.velo_dir, image_size,
True, slice_points)
draw_points(prop_2d_axes, point_2d, 'red')
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
prop_2d_axes, prop_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_2d_axes, pred_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs, calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_3d_axes, draw_score, draw_iou,
gt_classes, draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def main():
"""This demo runs through all samples in the trainval set, and checks
that the 3D box projection of all 'Car', 'Van', 'Pedestrian', and 'Cyclist'
objects are in the correct flipped 2D location after applying
modifications to the stereo p2 matrix.
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_TRAINVAL,
use_defaults=True)
np.set_printoptions(formatter={'float': lambda x: "{0:0.3f}".format(x)})
all_samples = dataset.sample_names
all_pixel_errors = []
all_max_pixel_errors = []
total_flip_time = 0.0
for sample_idx in range(dataset.num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx,
dataset.num_samples - 1))
sample_name = all_samples[sample_idx]
img_idx = int(sample_name)
# Run the main loop to run throughout the images
frame_calibration_info = calib_utils.read_calibration(
dataset.calib_dir, img_idx)
# Load labels
gt_labels = obj_utils.read_labels(dataset.label_dir, img_idx)
gt_labels = dataset.kitti_utils.filter_labels(
gt_labels, ['Car', 'Van', 'Pedestrian', 'Cyclist'])
image = cv2.imread(dataset.get_rgb_image_path(sample_name))
image_size = [image.shape[1], image.shape[0]]
# Flip p2 matrix
calib_p2 = frame_calibration_info.p2
flipped_p2 = np.copy(calib_p2)
flipped_p2[0, 2] = image.shape[1] - flipped_p2[0, 2]
flipped_p2[0, 3] = -flipped_p2[0, 3]
for obj_idx in range(len(gt_labels)):
obj = gt_labels[obj_idx]
# Get original 2D bounding boxes
orig_box_3d = box_3d_encoder.object_label_to_box_3d(obj)
orig_bbox_2d = box_3d_projector.project_to_image_space(
orig_box_3d, calib_p2, truncate=True, image_size=image_size)
# Skip boxes outside image
if orig_bbox_2d is None:
continue
orig_bbox_2d_flipped = flip_box_2d(orig_bbox_2d, image_size)
# Do flipping
start_time = time.time()
flipped_obj = kitti_aug.flip_label_in_3d_only(obj)
flip_time = time.time() - start_time
total_flip_time += flip_time
box_3d_flipped = box_3d_encoder.object_label_to_box_3d(flipped_obj)
new_bbox_2d_flipped = box_3d_projector.project_to_image_space(
box_3d_flipped,
flipped_p2,
truncate=True,
image_size=image_size)
pixel_errors = new_bbox_2d_flipped - orig_bbox_2d_flipped
max_pixel_error = np.amax(np.abs(pixel_errors))
all_pixel_errors.append(pixel_errors)
all_max_pixel_errors.append(max_pixel_error)
if max_pixel_error > 5:
print(' Error > 5px', sample_idx, max_pixel_error)
print(np.round(orig_bbox_2d_flipped, 3),
np.round(new_bbox_2d_flipped, 3))
print('Avg flip time:', total_flip_time / dataset.num_samples)
# Convert to ndarrays
all_pixel_errors = np.asarray(all_pixel_errors)
all_max_pixel_errors = np.asarray(all_max_pixel_errors)
# Print max values
print(np.amax(all_max_pixel_errors))
# Plot pixel errors
fig, axes = plt.subplots(nrows=3, ncols=1)
ax0, ax1, ax2 = axes.flatten()
ax0.hist(all_pixel_errors[:, 0], 50, histtype='bar', facecolor='green')
ax1.hist(all_pixel_errors[:, 2], 50, histtype='bar', facecolor='green')
ax2.hist(all_max_pixel_errors, 50, histtype='bar', facecolor='green')
plt.show()
def setUpClass(cls):
cls.fake_kitti_dir = tests.test_path() + "/datasets/Kitti/object"
cls.dataset = DatasetBuilder.build_kitti_dataset(
DatasetBuilder.KITTI_UNITTEST)
def setUpClass(cls):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.KITTI_UNITTEST)
cls.dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
cls.label_dir = cls.dataset.label_dir
def setUpClass(cls):
dataset_config = DatasetBuilder.copy_config(
DatasetBuilder.PANOPTIC_UNITTEST)
cls.dataset = DatasetBuilder.build_panoptic_dataset(dataset_config)
cls.label_dir = cls.dataset.label_dir
def generate_fake_dataset():
return DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_UNITTEST)
def main():
"""
Visualization of 3D grid anchor generation, showing 2D projections
in BEV and image space, and a 3D display of the anchors
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_TRAIN)
dataset_config.num_clusters[0] = 1
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
label_cluster_utils = LabelClusterUtils(dataset)
clusters, _ = label_cluster_utils.get_clusters()
# Options
img_idx = 1
# fake_clusters = np.array([[5, 4, 3], [6, 5, 4]])
# fake_clusters = np.array([[3, 3, 3], [4, 4, 4]])
fake_clusters = np.array([[4, 2, 3]])
fake_anchor_stride = [5.0, 5.0]
ground_plane = [0, -1, 0, 1.72]
anchor_3d_generator = grid_anchor_3d_generator.GridAnchor3dGenerator()
area_extents = np.array([[-40, 40], [-5, 5], [0, 70]])
# Generate anchors for cars only
start_time = time.time()
anchor_boxes_3d = anchor_3d_generator.generate(
area_3d=dataset.kitti_utils.area_extents,
anchor_3d_sizes=fake_clusters,
anchor_stride=fake_anchor_stride,
ground_plane=ground_plane)
all_anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
end_time = time.time()
print("Anchors generated in {} s".format(end_time - start_time))
# Project into bev
bev_boxes, bev_normalized_boxes = \
anchor_projector.project_to_bev(all_anchors, area_extents[[0, 2]])
bev_fig, (bev_axes, bev_normalized_axes) = \
plt.subplots(1, 2, figsize=(16, 7))
bev_axes.set_xlim(0, 80)
bev_axes.set_ylim(70, 0)
bev_normalized_axes.set_xlim(0, 1.0)
bev_normalized_axes.set_ylim(1, 0.0)
plt.show(block=False)
for box in bev_boxes:
box_w = box[2] - box[0]
box_h = box[3] - box[1]
rect = patches.Rectangle((box[0], box[1]),
box_w,
box_h,
linewidth=2,
edgecolor='b',
facecolor='none')
bev_axes.add_patch(rect)
for normalized_box in bev_normalized_boxes:
box_w = normalized_box[2] - normalized_box[0]
box_h = normalized_box[3] - normalized_box[1]
rect = patches.Rectangle((normalized_box[0], normalized_box[1]),
box_w,
box_h,
linewidth=2,
edgecolor='b',
facecolor='none')
bev_normalized_axes.add_patch(rect)
rgb_fig, rgb_2d_axes, rgb_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir, img_idx)
plt.show(block=False)
image_path = dataset.get_rgb_image_path(dataset.sample_names[img_idx])
image_shape = np.array(Image.open(image_path)).shape
stereo_calib_p2 = calib_utils.read_calibration(dataset.calib_dir,
img_idx).p2
start_time = time.time()
rgb_boxes, rgb_normalized_boxes = \
anchor_projector.project_to_image_space(all_anchors,
stereo_calib_p2,
image_shape)
end_time = time.time()
print("Anchors projected in {} s".format(end_time - start_time))
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir, 0)
p = stereo_calib.p2
# Overlay boxes on images
for anchor_idx in range(len(anchor_boxes_3d)):
anchor_box_3d = anchor_boxes_3d[anchor_idx]
obj_label = box_3d_encoder.box_3d_to_object_label(anchor_box_3d)
# Draw 3D boxes
vis_utils.draw_box_3d(rgb_3d_axes, obj_label, p)
# Draw 2D boxes
rgb_box_2d = rgb_boxes[anchor_idx]
box_x1 = rgb_box_2d[0]
box_y1 = rgb_box_2d[1]
box_w = rgb_box_2d[2] - box_x1
box_h = rgb_box_2d[3] - box_y1
rect = patches.Rectangle((box_x1, box_y1),
box_w,
box_h,
linewidth=2,
edgecolor='b',
facecolor='none')
rgb_2d_axes.add_patch(rect)
if anchor_idx % 32 == 0:
rgb_fig.canvas.draw()
plt.show(block=True)