def get_max_ious_3d(all_gt_boxes_3d, pred_boxes_3d):
"""Helper function to calculate 3D IoU for the given predictions.
Args:
all_gt_boxes_3d: A list of the same ground-truth boxes in box_3d
format.
pred_boxes_3d: A list of predictions in box_3d format.
"""
# Only calculate ious if there are predictions
if pred_boxes_3d:
# Convert to iou format
gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
all_gt_boxes_3d)
pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
pred_boxes_3d)
max_ious_3d = np.zeros(len(all_gt_boxes_3d))
for gt_obj_idx in range(len(all_gt_boxes_3d)):
gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx]
ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt,
pred_objs_iou_fmt)
max_ious_3d[gt_obj_idx] = np.amax(ious_3d)
else:
# No detections, all ious = 0
max_ious_3d = np.zeros(len(all_gt_boxes_3d))
return max_ious_3d
def draw_prediction_info(ax, x, y, pred_obj, pred_class_idx, pred_box,
ground_truth, draw_score, draw_iou, gt_classes,
iou_3d):
label = ""
if draw_score:
label += "{:.2f}".format(pred_obj.score)
if draw_iou and len(ground_truth) > 0:
if draw_score:
label += ', '
if iou_3d:
iou = evaluation.three_d_iou(pred_box, ground_truth)
if len(iou.shape) == 0:
iou = np.array([iou])
else:
iou = evaluation.two_d_iou(pred_box, ground_truth)
label += "{:.3f}".format(max(iou))
box_cls = gt_classes[int(pred_class_idx)]
ax.text(
x,
y - 4,
gt_classes[int(pred_class_idx)] + '\n' + label,
verticalalignment='bottom',
horizontalalignment='center',
color=BOX_COLOUR_SCHEME[box_cls],
fontsize=15,
fontweight='bold',
path_effects=[patheffects.withStroke(linewidth=2, foreground='black')])
def get_iou3d_matches(ego_objs, objs_perspectives, iou_thresh):
all_3d_ious = []
if len(ego_objs) > 0 and \
objs_perspectives is not None and len(objs_perspectives) > 0:
ego_objs_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(ego_obj)
for ego_obj in ego_objs
]
perspect_objs_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(objs_perspective)
for objs_perspective in objs_perspectives
]
# Convert to iou format
ego_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
ego_objs_boxes_3d)
perspect_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
perspect_objs_boxes_3d)
max_ious_3d = np.zeros(len(objs_perspectives))
max_iou_pred_indices = -np.ones(len(objs_perspectives))
for det_idx in range(len(objs_perspectives)):
perspect_obj_iou_fmt = perspect_objs_iou_fmt[det_idx]
ious_3d = evaluation.three_d_iou(perspect_obj_iou_fmt,
ego_objs_iou_fmt)
max_iou_3d = np.amax(ious_3d)
max_ious_3d[det_idx] = max_iou_3d
if max_iou_3d > iou_thresh:
max_iou_pred_indices[det_idx] = np.argmax(ious_3d)
return max_ious_3d, max_iou_pred_indices
def _calculate_anchors_info(self, all_anchor_boxes_3d, empty_anchor_filter,
gt_labels):
"""Calculates the list of anchor information in the format:
N x 8 [max_gt_2d_iou, max_gt_3d_iou, (6 x offsets), class_index]
max_gt_out - highest 3D iou with any ground truth box
offsets - encoded offsets [dx, dy, dz, d_dimx, d_dimy, d_dimz]
class_index - the anchor's class as an index
(e.g. 0 or 1, for "Background" or "Car")
Args:
all_anchor_boxes_3d: list of anchors in box_3d format
N x [x, y, z, l, w, h, ry]
empty_anchor_filter: boolean mask of which anchors are non empty
gt_labels: list of Object Label data format containing ground truth
labels to generate positives/negatives from.
Returns:
list of anchor info
"""
# Check for ground truth objects
if len(gt_labels) == 0:
raise Warning("No valid ground truth label to generate anchors.")
kitti_utils = self._dataset.kitti_utils
# Filter empty anchors
anchor_indices = np.where(empty_anchor_filter)[0]
anchor_boxes_3d = all_anchor_boxes_3d[empty_anchor_filter]
# Convert anchor_boxes_3d to anchor format
anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
# Convert gt to boxes_3d -> anchors -> iou format
gt_boxes_3d = np.asarray([
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in gt_labels
])
gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d,
ortho_rotate=True)
rpn_iou_type = self.mini_batch_utils.rpn_iou_type
if rpn_iou_type == '2d':
# Convert anchors to 2d iou format
anchors_for_2d_iou, _ = np.asarray(
anchor_projector.project_to_bev(anchors,
kitti_utils.bev_extents))
gt_boxes_for_2d_iou, _ = anchor_projector.project_to_bev(
gt_anchors, kitti_utils.bev_extents)
elif rpn_iou_type == '3d':
# Convert anchors to 3d iou format for calculation
anchors_for_3d_iou = box_3d_encoder.box_3d_to_3d_iou_format(
anchor_boxes_3d)
gt_boxes_for_3d_iou = \
box_3d_encoder.box_3d_to_3d_iou_format(gt_boxes_3d)
else:
raise ValueError('Invalid rpn_iou_type {}', rpn_iou_type)
# Initialize sample and offset lists
num_anchors = len(anchor_boxes_3d)
all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length))
# Update anchor indices
all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices
# For each of the labels, generate samples
for gt_idx in range(len(gt_labels)):
gt_obj = gt_labels[gt_idx]
gt_box_3d = gt_boxes_3d[gt_idx]
# Get 2D or 3D IoU for every anchor
if self.mini_batch_utils.rpn_iou_type == '2d':
gt_box_for_2d_iou = gt_boxes_for_2d_iou[gt_idx]
ious = evaluation.two_d_iou(gt_box_for_2d_iou,
anchors_for_2d_iou)
elif self.mini_batch_utils.rpn_iou_type == '3d':
gt_box_for_3d_iou = gt_boxes_for_3d_iou[gt_idx]
ious = evaluation.three_d_iou(gt_box_for_3d_iou,
anchors_for_3d_iou)
# Only update indices with a higher iou than before
update_indices = np.greater(
ious, all_info[:, self.mini_batch_utils.col_ious])
# Get ious to update
ious_to_update = ious[update_indices]
# Calculate offsets, use 3D iou to get highest iou
anchors_to_update = anchors[update_indices]
gt_anchor = box_3d_encoder.box_3d_to_anchor(gt_box_3d,
ortho_rotate=True)
offsets = anchor_encoder.anchor_to_offset(anchors_to_update,
gt_anchor)
# Convert gt type to index
class_idx = kitti_utils.class_str_to_index(gt_obj.type)
# Update anchors info (indices already updated)
# [index, iou, (offsets), class_index]
all_info[update_indices,
self.mini_batch_utils.col_ious] = ious_to_update
all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self.
mini_batch_utils.col_offsets_hi] = offsets
all_info[update_indices,
self.mini_batch_utils.col_class_idx] = class_idx
return all_info
def generate_negative_3d_bb(obj_label, boxes3d, iou_threshold_min,
iou_threshold_max, samples):
"""Generates negative 3D bounding boxes.
This is the 3D version of generate_negative_3d_bb.
Keyword arguments:
obj_label: single label for a detected 3D object
boxes3d: a list of numpy array representing all
3D bounding boxes in the image
iou_threshold_min: determines the min variation
between the original iou and the negative samples
iou_threshold_max: determines the max variation
between the original iou and the negative samples
samples: number of negative samples per detected object
Returns: a list of generated ObjectLabels
"""
box_corners = od.compute_box_corners_3d(obj_label)
# make sure this is not empty
assert (len(box_corners) > 0)
P1 = box_corners[:, 0]
P2 = box_corners[:, 1]
P4 = box_corners[:, 3]
current_samples = 0
new_objects = []
while current_samples < samples:
# Generate random 3D point inside the box
# we keep the same y, only generate x and z
new_xp = float(np.random.uniform(P1[0], P4[0], 1))
new_zp = float(np.random.uniform(P1[2], P2[2], 1))
# create a new ObjectLabel
new_obj = copy.copy(obj_label)
# update the new obj.t
# everything else is the same, only changing the
# centroid point t which remains the same along
# the y direction
new_obj.t = (new_xp, obj_label.t[1], new_zp)
_, box_to_test, _ = od.build_bbs_from_objects([new_obj], 'All')
assert (len(box_to_test) == 1)
# we are dealing with one box here so its the first element
iou_3d = evaluation.three_d_iou(box_to_test[0], boxes3d)
# check if iou_3d is a list, take the largest
# this compares to all the boxes
if isinstance(iou_3d, np.ndarray):
iou_max = np.max(iou_3d)
else:
iou_max = iou_3d
if iou_threshold_min < iou_max < iou_threshold_max:
# keep the new object label
new_objects.append(new_obj)
current_samples += 1
return new_objects
def test_3d_iou(self):
iou = evaluation.three_d_iou(self.gtbb_3d, self.test_cases_3d)
np.testing.assert_almost_equal(self.gt_iou_3d, iou, 3)
iou = evaluation.three_d_iou(self.gtbb_3d, self.test_cases_3d[4])
np.testing.assert_almost_equal(self.gt_iou_3d[4], iou, 3)
def main():
"""Plots detection errors for xyz, lwh, ry, and shows 3D IoU with
ground truth boxes
"""
dataset = DatasetBuilder.build_kitti_dataset(DatasetBuilder.KITTI_VAL_HALF,
use_defaults=True)
difficulty = 2
# Full path to kitti predictions
# (e.g. '.../data/outputs/mlod_exp_example/predictions/'
# 'kitti_predictions_3d/val/0.1/100000/data'
predictions_data_path = 'path_to_detections/data'
# Loop through split and save ious and errors
all_3d_ious = []
all_errors = []
for sample_idx in range(dataset.num_samples):
sys.stdout.write('\r{} / {}'.format(sample_idx + 1,
dataset.num_samples))
sample_name = dataset.sample_names[sample_idx]
img_idx = int(sample_name)
# Get filtered ground truth
all_gt_objs = obj_utils.read_labels(dataset.label_dir, img_idx)
all_gt_objs = dataset.kitti_utils.filter_labels(all_gt_objs,
difficulty=difficulty)
pred_objs = obj_utils.read_labels(predictions_data_path, img_idx)
##############################
# Check IoUs
##############################
if len(all_gt_objs) > 0 and \
pred_objs is not None and len(pred_objs) > 0:
all_gt_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in all_gt_objs
]
pred_boxes_3d = [
box_3d_encoder.object_label_to_box_3d(pred_obj)
for pred_obj in pred_objs
]
# Convert to iou format
gt_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
all_gt_boxes_3d)
pred_objs_iou_fmt = box_3d_encoder.box_3d_to_3d_iou_format(
pred_boxes_3d)
max_ious_3d = np.zeros(len(all_gt_objs))
max_iou_pred_indices = -np.ones(len(all_gt_objs))
for gt_obj_idx in range(len(all_gt_objs)):
gt_obj_iou_fmt = gt_objs_iou_fmt[gt_obj_idx]
ious_3d = evaluation.three_d_iou(gt_obj_iou_fmt,
pred_objs_iou_fmt)
max_iou_3d = np.amax(ious_3d)
max_ious_3d[gt_obj_idx] = max_iou_3d
if max_iou_3d > 0.0:
max_iou_pred_indices[gt_obj_idx] = np.argmax(ious_3d)
for gt_obj_idx in range(len(all_gt_objs)):
max_iou_pred_idx = int(max_iou_pred_indices[gt_obj_idx])
if max_iou_pred_idx >= 0:
error = all_gt_boxes_3d[gt_obj_idx] - \
pred_boxes_3d[max_iou_pred_idx]
all_errors.append(error)
all_3d_ious.extend(max_ious_3d)
print('Done')
all_errors = np.asarray(all_errors)
# Plot Data Histograms
f, ax_arr = plt.subplots(3, 3)
xyzlwh_bins = 51
ry_bins = 31
iou_bins = 51
# xyz
ax_arr[0, 0].hist(all_errors[:, 0],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[0, 1].hist(all_errors[:, 1],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[0, 2].hist(all_errors[:, 2],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
# lwh
ax_arr[1, 0].hist(all_errors[:, 3],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[1, 1].hist(all_errors[:, 4],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
ax_arr[1, 2].hist(all_errors[:, 5],
xyzlwh_bins,
facecolor='green',
alpha=0.75)
# orientation
ax_arr[2, 0].hist(all_errors[:, 6], ry_bins, facecolor='green', alpha=0.75)
# iou
ax_arr[2, 2].hist(all_3d_ious, iou_bins, facecolor='green', alpha=0.75)
plt.show()
