def test_evaluate_3d(self):
gt_detections = utils.read_labels(self.label_dir, 194)
detections = utils.read_labels(self.results_dir, 194, True)
_, gt_3d_bbs, _ = utils.build_bbs_from_objects(gt_detections,
['Car', 'car'])
_, detected_3d_bbs, scores = utils.build_bbs_from_objects(
detections, ['Car', 'car'])
score_thresholds = np.array([0.5])
iou_threshold = 0.25
car_precision_3d, car_recall_3d = evaluation.evaluate_3d(
[gt_3d_bbs], [detected_3d_bbs], [scores], score_thresholds,
iou_threshold)
np.testing.assert_almost_equal(car_precision_3d, 1.0, 3)
np.testing.assert_almost_equal(car_recall_3d, 1.0, 3)
score_thresholds = np.array([0.994])
car_precision_3d, car_recall_3d = evaluation.evaluate_3d(
[gt_3d_bbs], [detected_3d_bbs], [scores], score_thresholds,
iou_threshold)
np.testing.assert_almost_equal(car_precision_3d, 1.0, 3)
np.testing.assert_almost_equal(car_recall_3d, 0.5, 3)
def main():
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL)
# get proposals
proposal_output_dir = mlod.root_dir() + \
"/data/predictions/rpn_model/proposals_and_scores/" + \
dataset.data_split
global_steps = os.listdir(proposal_output_dir)
print('Checkpoints found ', global_steps)
all_recalls = []
for step in global_steps:
for sample_name in dataset.sample_list:
img_idx = int(sample_name)
# ------------------------------------
# Load proposals and scores from files
# ------------------------------------
proposals_scores_dir = proposal_output_dir + \
"{}/{}/{}.txt".format(dataset.data_split,
step,
sample_name)
if not os.path.exists(proposals_scores_dir):
print('File {} not found, skipping'.format(sample_name))
continue
proposals_scores = np.loadtxt(proposals_scores_dir)
proposals = proposals_scores[:, 0:-1]
proposal_iou_format = \
box_3d_encoder.box_3d_to_3d_iou_format(proposals)
# scores are in the last column
scores = proposals_scores[:, -1]
# -----------------------
# Get ground truth labels
# -----------------------
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
_, gt_3d_bbs, _ = obj_utils.build_bbs_from_objects(
gt_objects, ['Car', 'car'])
score_thresholds = np.array([0.3])
iou_threshold = 0.0025
# calculate RPN recall and precision
precision, recall = evaluation.evaluate_3d([gt_3d_bbs],
[proposal_iou_format],
[scores],
score_thresholds,
iou_threshold)
print('Recall ', recall[0])
print('Precision ', precision[0])
all_recalls.append(recall)
def test_negative_2d_bb_speed(self):
object_labels = od.read_labels(self.test_data_labels, 5258)
first_obj = object_labels[0]
img_class = "All"
boxes2d, _, _ = od.build_bbs_from_objects(object_labels, img_class)
iou_threshold_min = 0.01
iou_threshold_max = 0.7
samples = 5000
self.startTime = time.time()
new_boxes, failed_count = augment.calculate_negative_2d_bb(
first_obj,
boxes2d,
iou_threshold_min,
iou_threshold_max,
samples,
rand_sampl=True)
t = time.time() - self.startTime
print('\n==== Random Sampling ====\n')
print("%s Run Time: \n%.3f" % (self.id(), t))
print('Number of failures ', failed_count)
self.startTime = time.time()
new_boxes, failed_count = augment.calculate_negative_2d_bb(
first_obj,
boxes2d,
iou_threshold_min,
iou_threshold_max,
samples,
rand_sampl=False)
t = time.time() - self.startTime
print("\n==== Calculating and Random Sampling on Failures ====\n")
print("%s Run Time: \n%.3f" % (self.id(), t))
print('Number of failures ', failed_count)
def main():
"""This demo shows RPN proposals and AVOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
dataset_config = DatasetBuilder.copy_config(DatasetBuilder.KITTI_VAL)
##############################
# Options
##############################
dataset_config.data_split = 'val'
fig_size = (10, 6.1)
rpn_score_threshold = 0.1
avod_score_threshold = 0.1
# gt_classes = ['Car']
gt_classes = ['Pedestrian', 'Cyclist']
# gt_classes = ['Car', 'Pedestrian', 'Cyclist']
# Overwrite this to select a specific checkpoint
global_step = None
checkpoint_name = sys.argv[1] #'pyramid_cars_with_aug_example'
# Drawing Toggles
draw_proposals_separate = False
draw_overlaid = False
draw_predictions_separate = True
# Show orientation for both GT and proposals/predictions
draw_orientations_on_prop = False
draw_orientations_on_pred = False
# Draw 2D bounding boxes
draw_projected_2d_boxes = True
# Save images for samples with no detections
save_empty_images = True
draw_score = True
draw_iou = True
##############################
# End of Options
##############################
# Get the dataset
dataset = DatasetBuilder.build_kitti_dataset(dataset_config)
# Setup Paths
predictions_dir = avod.root_dir() + \
'/data/outputs/' + checkpoint_name + '/predictions'
proposals_and_scores_dir = predictions_dir + \
'/proposals_and_scores/' + dataset.data_split
predictions_and_scores_dir = predictions_dir + \
'/final_predictions_and_scores/' + dataset.data_split
# Output images directories
output_dir_base = predictions_dir + '/images_2d'
# Get checkpoint step
steps = os.listdir(proposals_and_scores_dir)
steps.sort(key=int)
print('Available steps: {}'.format(steps))
# Use latest checkpoint if no index provided
if global_step is None:
global_step = steps[-1]
if draw_proposals_separate:
prop_out_dir = output_dir_base + '/proposals/{}/{}/{}'.format(
dataset.data_split, global_step, rpn_score_threshold)
if not os.path.exists(prop_out_dir):
os.makedirs(prop_out_dir)
print('Proposal images saved to:', prop_out_dir)
if draw_overlaid:
overlaid_out_dir = output_dir_base + '/overlaid/{}/{}/{}'.format(
dataset.data_split, global_step, avod_score_threshold)
if not os.path.exists(overlaid_out_dir):
os.makedirs(overlaid_out_dir)
print('Overlaid images saved to:', overlaid_out_dir)
if draw_predictions_separate:
pred_out_dir = output_dir_base + '/predictions/{}/{}/{}'.format(
dataset.data_split, global_step, avod_score_threshold)
if not os.path.exists(pred_out_dir):
os.makedirs(pred_out_dir)
print('Prediction images saved to:', pred_out_dir)
# Rolling average array of times for time estimation
avg_time_arr_length = 10
last_times = np.repeat(time.time(), avg_time_arr_length) + \
np.arange(avg_time_arr_length)
for sample_idx in range(dataset.num_samples):
# Estimate time remaining with 5 slowest times
start_time = time.time()
last_times = np.roll(last_times, -1)
last_times[-1] = start_time
avg_time = np.mean(np.sort(np.diff(last_times))[-5:])
samples_remaining = dataset.num_samples - sample_idx
est_time_left = avg_time * samples_remaining
# Print progress and time remaining estimate
sys.stdout.write('\rSaving {} / {}, Avg Time: {:.3f}s, '
'Time Remaining: {:.2f}s'.format(
sample_idx + 1, dataset.num_samples, avg_time,
est_time_left))
sys.stdout.flush()
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
##############################
# Proposals
##############################
if draw_proposals_separate or draw_overlaid:
# Load proposals from files
proposals_file_path = proposals_and_scores_dir + \
"/{}/{}.txt".format(global_step, sample_name)
if not os.path.exists(proposals_file_path):
print('Sample {}: No proposals, skipping'.format(sample_name))
continue
print('Sample {}: Drawing proposals'.format(sample_name))
proposals_and_scores = np.loadtxt(proposals_file_path)
proposal_boxes_3d = proposals_and_scores[:, 0:7]
proposal_scores = proposals_and_scores[:, 7]
# Apply score mask to proposals
score_mask = proposal_scores > rpn_score_threshold
proposal_boxes_3d = proposal_boxes_3d[score_mask]
proposal_scores = proposal_scores[score_mask]
proposal_objs = \
[box_3d_encoder.box_3d_to_object_label(proposal,
obj_type='Proposal')
for proposal in proposal_boxes_3d]
##############################
# Predictions
##############################
if draw_predictions_separate or draw_overlaid:
predictions_file_path = predictions_and_scores_dir + \
"/{}/{}.txt".format(global_step,
sample_name)
if not os.path.exists(predictions_file_path):
continue
# Load predictions from files
predictions_and_scores = np.loadtxt(
predictions_and_scores_dir +
"/{}/{}.txt".format(global_step, sample_name))
prediction_boxes_3d = predictions_and_scores[:, 0:7]
prediction_scores = predictions_and_scores[:, 7]
prediction_class_indices = predictions_and_scores[:, 8]
# process predictions only if we have any predictions left after
# masking
if len(prediction_boxes_3d) > 0:
# Apply score mask
avod_score_mask = prediction_scores >= avod_score_threshold
prediction_boxes_3d = prediction_boxes_3d[avod_score_mask]
prediction_scores = prediction_scores[avod_score_mask]
prediction_class_indices = \
prediction_class_indices[avod_score_mask]
# # Swap l, w for predictions where w > l
# swapped_indices = \
# prediction_boxes_3d[:, 4] > prediction_boxes_3d[:, 3]
# prediction_boxes_3d = np.copy(prediction_boxes_3d)
# prediction_boxes_3d[swapped_indices, 3] = \
# prediction_boxes_3d[swapped_indices, 4]
# prediction_boxes_3d[swapped_indices, 4] = \
# prediction_boxes_3d[swapped_indices, 3]
##############################
# Ground Truth
##############################
# Get ground truth labels
if dataset.has_labels:
gt_objects = obj_utils.read_labels(dataset.label_dir, img_idx)
else:
gt_objects = []
# Filter objects to desired difficulty
filtered_gt_objs = dataset.kitti_utils.filter_labels(
gt_objects, classes=gt_classes)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
filtered_gt_objs, class_needed=gt_classes)
image_path = dataset.get_rgb_image_path(sample_name)
image = Image.open(image_path)
image_size = image.size
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(dataset.calib_dir, img_idx)
calib_p2 = stereo_calib.p2
##############################
# Reformat and prepare to draw
##############################
if draw_proposals_separate or draw_overlaid:
proposals_as_anchors = box_3d_encoder.box_3d_to_anchor(
proposal_boxes_3d)
proposal_boxes, _ = anchor_projector.project_to_image_space(
proposals_as_anchors, calib_p2, image_size)
num_of_proposals = proposal_boxes_3d.shape[0]
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False)
draw_proposals(filtered_gt_objs, calib_p2, num_of_proposals,
proposal_objs, proposal_boxes, prop_2d_axes,
prop_3d_axes, draw_orientations_on_prop)
if draw_proposals_separate:
# Save just the proposals
filename = prop_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
if not draw_overlaid:
plt.close(prop_fig)
if draw_overlaid or draw_predictions_separate:
if len(prediction_boxes_3d) > 0:
# Project the 3D box predictions to image space
image_filter = []
final_boxes_2d = []
for i in range(len(prediction_boxes_3d)):
box_3d = prediction_boxes_3d[i, 0:7]
img_box = box_3d_projector.project_to_image_space(
box_3d,
calib_p2,
truncate=True,
image_size=image_size,
discard_before_truncation=False)
if img_box is not None:
image_filter.append(True)
final_boxes_2d.append(img_box)
else:
image_filter.append(False)
final_boxes_2d = np.asarray(final_boxes_2d)
final_prediction_boxes_3d = prediction_boxes_3d[image_filter]
final_scores = prediction_scores[image_filter]
final_class_indices = prediction_class_indices[image_filter]
num_of_predictions = final_boxes_2d.shape[0]
# Convert to objs
final_prediction_objs = \
[box_3d_encoder.box_3d_to_object_label(
prediction, obj_type='Prediction')
for prediction in final_prediction_boxes_3d]
for (obj, score) in zip(final_prediction_objs, final_scores):
obj.score = score
else:
if save_empty_images:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
continue
if draw_overlaid:
# Overlay prediction boxes on image
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
prop_2d_axes, prop_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
filename = overlaid_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(prop_fig)
if draw_predictions_separate:
# Now only draw prediction boxes on images
# on a new figure handler
if draw_projected_2d_boxes:
pred_fig, pred_2d_axes, pred_3d_axes = \
vis_utils.visualization(dataset.rgb_image_dir,
img_idx,
display=False,
fig_size=fig_size)
draw_predictions(filtered_gt_objs, calib_p2,
num_of_predictions, final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_2d_axes, pred_3d_axes, draw_score,
draw_iou, gt_classes,
draw_orientations_on_pred)
else:
pred_fig, pred_3d_axes = \
vis_utils.visualize_single_plot(
dataset.rgb_image_dir, img_idx, display=False)
draw_3d_predictions(filtered_gt_objs, calib_p2,
num_of_predictions,
final_prediction_objs,
final_class_indices, final_boxes_2d,
pred_3d_axes, draw_score, draw_iou,
gt_classes, draw_orientations_on_pred)
filename = pred_out_dir + '/' + sample_name + '.png'
plt.savefig(filename)
plt.close(pred_fig)
print('\nDone')
def generate_negative_3d_bb(obj_label, boxes3d, iou_threshold_min,
iou_threshold_max, samples):
"""Generates negative 3D bounding boxes.
This is the 3D version of generate_negative_3d_bb.
Keyword arguments:
obj_label: single label for a detected 3D object
boxes3d: a list of numpy array representing all
3D bounding boxes in the image
iou_threshold_min: determines the min variation
between the original iou and the negative samples
iou_threshold_max: determines the max variation
between the original iou and the negative samples
samples: number of negative samples per detected object
Returns: a list of generated ObjectLabels
"""
box_corners = od.compute_box_corners_3d(obj_label)
# make sure this is not empty
assert (len(box_corners) > 0)
P1 = box_corners[:, 0]
P2 = box_corners[:, 1]
P4 = box_corners[:, 3]
current_samples = 0
new_objects = []
while current_samples < samples:
# Generate random 3D point inside the box
# we keep the same y, only generate x and z
new_xp = float(np.random.uniform(P1[0], P4[0], 1))
new_zp = float(np.random.uniform(P1[2], P2[2], 1))
# create a new ObjectLabel
new_obj = copy.copy(obj_label)
# update the new obj.t
# everything else is the same, only changing the
# centroid point t which remains the same along
# the y direction
new_obj.t = (new_xp, obj_label.t[1], new_zp)
_, box_to_test, _ = od.build_bbs_from_objects([new_obj], 'All')
assert (len(box_to_test) == 1)
# we are dealing with one box here so its the first element
iou_3d = evaluation.three_d_iou(box_to_test[0], boxes3d)
# check if iou_3d is a list, take the largest
# this compares to all the boxes
if isinstance(iou_3d, np.ndarray):
iou_max = np.max(iou_3d)
else:
iou_max = iou_3d
if iou_threshold_min < iou_max < iou_threshold_max:
# keep the new object label
new_objects.append(new_obj)
current_samples += 1
return new_objects
def main():
"""This demo shows RPN proposals and AVOD predictions in 3D
and 2D in image space. Given certain thresholds for proposals
and predictions, it selects and draws the bounding boxes on
the image sample. It goes through the entire proposal and
prediction samples for the given dataset split.
The proposals, overlaid, and prediction images can be toggled on or off
separately in the options section.
The prediction score and IoU with ground truth can be toggled on or off
as well, shown as (score, IoU) above the detection.
"""
fig_size = (10, 6.1)
gt_classes = ['Car', 'Pedestrian', 'Cyclist']
# Output images directories
output_dir_base = 'images_2d'
data_dir = '../../DATA/Kitti/object/'
label_dir = data_dir + 'training/label_2'
image_dir = data_dir + 'training/image_2'
filepath = data_dir + 'val.txt'
calib_dir = data_dir + 'training/calib'
filenames = open(filepath, 'r').readlines()
filenames = [int(filename) for filename in filenames]
i = 0
i_max = len(filenames)
for filename in filenames:
##############################
# Ground Truth
##############################
# Get ground truth labels
gt_objects = obj_utils.read_labels(label_dir, filename)
boxes2d, _, _ = obj_utils.build_bbs_from_objects(
gt_objects, class_needed=gt_classes)
image_path = image_dir + "/%06d.png" % filename
image = Image.open(image_path)
image_size = image.size
prop_fig, prop_2d_axes, prop_3d_axes = \
vis_utils.visualization(image_dir,
filename,
display=False)
# Read the stereo calibration matrix for visualization
stereo_calib = calib_utils.read_calibration(calib_dir, filename)
calib_p2 = stereo_calib.p2
draw_gt(gt_objects, prop_2d_axes, prop_3d_axes, calib_p2)
out_name = output_dir_base + "/%06d.png" % filename
plt.savefig(out_name)
plt.close(prop_fig)
i += 1
print(str(i) + '/' + str(i_max))
print('\nDone')