def test_2d_iou(self):
iou = evaluation.two_d_iou(self.gtbb_2d, self.test_cases_2d)
np.testing.assert_allclose(self.gt_iou_2d, iou, 0, 0.01)
gt_box = np.asarray([0.0, 0.0, 0.5, 0.5])
test_boxes = np.asarray([
[-0.5, 0.0, 0.0, 0.5],
[-0.25, 0.0, 0.25, 0.5],
[0.0, 0.0, 0.5, 0.5],
[0.25, 0.0, 0.75, 0.5],
[0.5, 0.0, 1.0, 0.5],
])
exp_ious_2d = [0.0, 0.333, 1.0, 0.333, 0.0]
ious_2d = evaluation.two_d_iou(gt_box, test_boxes)
np.testing.assert_almost_equal(ious_2d, exp_ious_2d)
def draw_proposals_info(ax, x, y, proposal_scores, proposal_box_2d,
ground_truth, draw_score, draw_iou):
label = ""
# print('proposal_scores = ', proposal_scores)
if draw_score:
label += "sc:{:.2f}".format(proposal_scores)
# print('proposal_box_2d = ', proposal_box_2d)
# print('ground_truth = ', ground_truth)
if draw_iou and len(ground_truth) > 0:
if draw_score:
label += ', '
iou = evaluation.two_d_iou(proposal_box_2d, ground_truth)
label += "iou:{:.3f}".format(max(iou))
ax.text(
x,
y + 150,
label,
verticalalignment='bottom',
horizontalalignment='center',
color='cornflowerblue',
fontsize=10,
fontweight='bold',
path_effects=[patheffects.withStroke(linewidth=5, foreground='black')])
def draw_prediction_info(ax, x, y, pred_obj, pred_class_idx, pred_box_2d,
ground_truth, draw_score, draw_iou, gt_classes):
label = ""
if draw_score:
label += "sc:{:.2f}".format(pred_obj.score)
if draw_iou and len(ground_truth) > 0:
if draw_score:
label += ', '
iou = evaluation.two_d_iou(pred_box_2d, ground_truth)
label += "iou:{:.3f}".format(max(iou))
box_cls = gt_classes[int(pred_class_idx)]
ax.text(
x,
y + 54,
gt_classes[int(pred_class_idx)] + '\n' + label,
verticalalignment='bottom',
horizontalalignment='center',
color=BOX_COLOUR_SCHEME[box_cls],
fontsize=10,
fontweight='bold',
path_effects=[patheffects.withStroke(linewidth=5, foreground='black')])
def generate_negative_2d_bb(obj_label, boxes2d, iou_threshold_min,
iou_threshold_max, samples):
"""Generates negative 2D bounding boxes.
This is computed in a brute force fashion. For any given
bounding box, we randomly shift the centroid and generate
new bounding boxes and if it lies within the desired IoU
threshold bound, we will keep it. Otherwise it is thrown
out and this is repeated until number of valid samples is
satisfied.
Keyword arguments:
obj_label: single label for a detected 2D object
boxes2d: a list of numpy array representing all
bounding boxes in the image
iou_threshold_min: determines the min variation
between the original iou and the negative samples
iou_threshold_max: determines the max variation
between the original iou and the negative samples
samples: number of negative samples per detected object
Returns: a list of generated ObjectLabels
"""
x1 = obj_label.x1
y1 = obj_label.y1
x2 = obj_label.x2
y2 = obj_label.y2
diff_x = (x2 - x1) / 2
diff_y = (y2 - y1) / 2
current_samples = 0
new_objects = []
while current_samples < samples:
# Keep trying to generate samples that
# lie within reasonable bound
new_xp = np.random.uniform(x1, x2, 1)
new_yp = np.random.uniform(y1, y2, 1)
new_x1 = float(new_xp - diff_x)
new_x2 = float(new_xp + diff_x)
new_y1 = float(new_yp - diff_y)
new_y2 = float(new_yp + diff_y)
new_obj = od.ObjectLabel()
new_obj.x1 = new_x1
new_obj.x2 = new_x2
new_obj.y1 = new_y1
new_obj.y2 = new_y2
new_box = np.array([new_x1, new_y1, new_x2, new_y2])
# calculate the IoU
iou = evaluation.two_d_iou(new_box, boxes2d)
if iou_threshold_min < max(iou) < iou_threshold_max:
# keep the new object label
current_samples += 1
new_objects.append(new_obj)
return new_objects
def draw_prediction_info(ax, x, y, pred_obj, pred_class_idx, pred_box,
ground_truth, draw_score, draw_iou, gt_classes,
iou_3d):
label = ""
if draw_score:
label += "{:.2f}".format(pred_obj.score)
if draw_iou and len(ground_truth) > 0:
if draw_score:
label += ', '
if iou_3d:
iou = evaluation.three_d_iou(pred_box, ground_truth)
if len(iou.shape) == 0:
iou = np.array([iou])
else:
iou = evaluation.two_d_iou(pred_box, ground_truth)
label += "{:.3f}".format(max(iou))
box_cls = gt_classes[int(pred_class_idx)]
ax.text(
x,
y - 4,
gt_classes[int(pred_class_idx)] + '\n' + label,
verticalalignment='bottom',
horizontalalignment='center',
color=BOX_COLOUR_SCHEME[box_cls],
fontsize=15,
fontweight='bold',
path_effects=[patheffects.withStroke(linewidth=2, foreground='black')])
def _calculate_anchors_info(self, all_anchor_boxes_3d, empty_anchor_filter,
gt_labels):
"""Calculates the list of anchor information in the format:
N x 8 [max_gt_2d_iou, max_gt_3d_iou, (6 x offsets), class_index]
max_gt_out - highest 3D iou with any ground truth box
offsets - encoded offsets [dx, dy, dz, d_dimx, d_dimy, d_dimz]
class_index - the anchor's class as an index
(e.g. 0 or 1, for "Background" or "Car")
Args:
all_anchor_boxes_3d: list of anchors in box_3d format
N x [x, y, z, l, w, h, ry]
empty_anchor_filter: boolean mask of which anchors are non empty
gt_labels: list of Object Label data format containing ground truth
labels to generate positives/negatives from.
Returns:
list of anchor info
"""
# Check for ground truth objects
if len(gt_labels) == 0:
raise Warning("No valid ground truth label to generate anchors.")
kitti_utils = self._dataset.kitti_utils
# Filter empty anchors
anchor_indices = np.where(empty_anchor_filter)[0]
anchor_boxes_3d = all_anchor_boxes_3d[empty_anchor_filter]
# Convert anchor_boxes_3d to anchor format
anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
# Convert gt to boxes_3d -> anchors -> iou format
gt_boxes_3d = np.asarray([
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in gt_labels
])
gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d,
ortho_rotate=True)
rpn_iou_type = self.mini_batch_utils.rpn_iou_type
if rpn_iou_type == '2d':
# Convert anchors to 2d iou format
anchors_for_2d_iou, _ = np.asarray(
anchor_projector.project_to_bev(anchors,
kitti_utils.bev_extents))
gt_boxes_for_2d_iou, _ = anchor_projector.project_to_bev(
gt_anchors, kitti_utils.bev_extents)
elif rpn_iou_type == '3d':
# Convert anchors to 3d iou format for calculation
anchors_for_3d_iou = box_3d_encoder.box_3d_to_3d_iou_format(
anchor_boxes_3d)
gt_boxes_for_3d_iou = \
box_3d_encoder.box_3d_to_3d_iou_format(gt_boxes_3d)
else:
raise ValueError('Invalid rpn_iou_type {}', rpn_iou_type)
# Initialize sample and offset lists
num_anchors = len(anchor_boxes_3d)
all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length))
# Update anchor indices
all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices
# For each of the labels, generate samples
for gt_idx in range(len(gt_labels)):
gt_obj = gt_labels[gt_idx]
gt_box_3d = gt_boxes_3d[gt_idx]
# Get 2D or 3D IoU for every anchor
if self.mini_batch_utils.rpn_iou_type == '2d':
gt_box_for_2d_iou = gt_boxes_for_2d_iou[gt_idx]
ious = evaluation.two_d_iou(gt_box_for_2d_iou,
anchors_for_2d_iou)
elif self.mini_batch_utils.rpn_iou_type == '3d':
gt_box_for_3d_iou = gt_boxes_for_3d_iou[gt_idx]
ious = evaluation.three_d_iou(gt_box_for_3d_iou,
anchors_for_3d_iou)
# Only update indices with a higher iou than before
update_indices = np.greater(
ious, all_info[:, self.mini_batch_utils.col_ious])
# Get ious to update
ious_to_update = ious[update_indices]
# Calculate offsets, use 3D iou to get highest iou
anchors_to_update = anchors[update_indices]
gt_anchor = box_3d_encoder.box_3d_to_anchor(gt_box_3d,
ortho_rotate=True)
offsets = anchor_encoder.anchor_to_offset(anchors_to_update,
gt_anchor)
# Convert gt type to index
class_idx = kitti_utils.class_str_to_index(gt_obj.type)
# Update anchors info (indices already updated)
# [index, iou, (offsets), class_index]
all_info[update_indices,
self.mini_batch_utils.col_ious] = ious_to_update
all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self.
mini_batch_utils.col_offsets_hi] = offsets
all_info[update_indices,
self.mini_batch_utils.col_class_idx] = class_idx
return all_info
def calculate_negative_2d_bb(obj_label,
boxes2d,
iou_threshold_min,
iou_threshold_max,
samples,
rand_sampl=False):
"""Generates negative 2D bounding boxes.
This is computed in a semi-brute force fashion.
For any given bounding box, we first try to calculate
the desired shift based on the selected IoU. If this
failes, we just randomly shift the centroid and generate
new bounding boxes. If it lies within the desired IoU
threshold bound, we will keep it. Otherwise it is thrown
out and this is repeated until number of valid samples is
satisfied.
Keyword arguments:
obj_label: single label for a detected 2D object
boxes2d: a list of numpy array representing all
bounding boxes in the image
iou_threshold_min: determines the min variation
between the original iou and the negative samples
iou_threshold_max: determines the max variation
between the original iou and the negative samples
samples: number of negative samples per detected object
rand_sampl: Flag to switch between calculation vs pure
random sampling. For speed testing purposes.
Returns:
new_objects: a list of randomly generated ObjectLabels
failed_cases: int number of cases it failed to calculate
and opted in for random sampling.
"""
x1 = obj_label.x1
y1 = obj_label.y1
x2 = obj_label.x2
y2 = obj_label.y2
width = x2 - x1
length = y2 - y1
half_w = width / 2
half_l = length / 2
miscalc = False
current_samples = 0
new_objects = []
failed_cases = 0
while current_samples < samples:
# Keep trying to generate samples that
# lie within reasonable bound
if not miscalc and not rand_sampl:
# we will try to to this by calculating
# the shift along l or w given the desired
# IoU and fixing either w or l shift.
# this is sub-optimal since it does not
# guarantee to achive the actual desired
# IoU since we keep one variable fixed!
# First let's try to generate bounding box
# by randomly selecting a desirable IoU
possible_ious = np.linspace(iou_threshold_min, iou_threshold_max,
10)
# pick one randomly
rand_iou = random.choice(possible_ious)
# assuming l and w here are equal, given IoU
# by fixing either delta_w or delta_l we can
# calculate the other. This way we *guess*
# the generated box will satify the IoU bound
# constraint
l_fixed = random.choice([True, False])
if l_fixed:
# Lets keep delta_l fixed
# It just needs to keep it within 0 - l
delta_l = random.uniform(0, length / 2)
l_shift = length - delta_l
w_shift = (rand_iou * length * width) / (l_shift)
delta_w = width - w_shift
else:
# keep delta_w fixed
delta_w = random.uniform(0, width / 2)
w_shift = length - delta_w
l_shift = (rand_iou * length * width) / (w_shift)
delta_l = length - l_shift
# now just shift the l and w
new_xp = x1 + delta_w
new_yp = y1 + delta_l
else:
# that didn't work in the previous iteration
# try generating random points
new_xp = np.random.uniform(x1, x2, 1)
new_yp = np.random.uniform(y1, y2, 1)
# give it another chance in the next iteration
miscalc = False
new_obj, new_box = _construct_new_2d_object(new_xp, half_w, new_yp,
half_l)
# calculate the IoU
iou = evaluation.two_d_iou(new_box, boxes2d)
# check if it generated the desired IoU
if iou_threshold_min < max(iou) < iou_threshold_max:
# keep the new object label
current_samples += 1
new_objects.append(new_obj)
else:
failed_cases += 1
miscalc = True
return new_objects, failed_cases
def _calculate_img_anchors_info(self, all_anchor_boxes_3d,
empty_anchor_filter, gt_labels,
stereo_calib_p2, image_size):
"""Calculates the list of anchor information in the format:
N x 6 [max_gt_img_iou, (4 x offsets), class_index]
max_gt_img_out - highest img iou with any ground truth box
offsets - encoded offsets [dx, dy, d_dimx, d_dimy]
class_index - the anchor's class as an index
(e.g. 0 or 1, for "Background" or "Car")
Args:
all_anchor_boxes_3d: list of anchors in box_3d format
N x [x, y, z, l, w, h, ry]
empty_anchor_filter: boolean mask of which anchors are non empty
gt_labels: list of Object Label data format containing ground truth
labels to generate positives/negatives from.
image_size: [h, w]
Returns:
list of anchor info
"""
# Check for ground truth objects
if len(gt_labels) == 0:
raise Warning("No valid ground truth label to generate anchors.")
kitti_utils = self._dataset.kitti_utils
# Filter empty anchors
anchor_indices = np.where(empty_anchor_filter)[0]
anchor_boxes_3d = all_anchor_boxes_3d[empty_anchor_filter]
# Convert anchor_boxes_3d to anchor format [x,y,z,dx,dy,dz]
anchors = box_3d_encoder.box_3d_to_anchor(anchor_boxes_3d)
## benz, project anchor_boxes_3d into image plane instead of using anchors, TODO remove out_of_bounding points
## img_boxes: [x1, y1, x2, y2]
# img_boxes_anchors, img_boxes_anchors_norm = anchor_projector.project_to_image_space(anchors, stereo_calib_p2, image_size)
img_boxes_anchors, img_boxes_anchors_norm = anchor_projector.project_3d_box_to_image_space(
anchor_boxes_3d, stereo_calib_p2, image_size) ## project 3d_boxes
# Convert gt to boxes_3d -> anchors -> iou format
#gt_boxes_3d = np.asarray(
# [box_3d_encoder.object_label_to_box_3d(gt_obj)
# for gt_obj in gt_labels])
## benz, gt_boxes_3d [x1,y1,x2,y2]
gt_boxes_2d = np.asarray([
box_2d_encoder.object_label_to_box_2d(gt_obj)
for gt_obj in gt_labels
])
#gt_anchors = box_3d_encoder.box_3d_to_anchor(gt_boxes_3d,
# ortho_rotate=True)
rpn_iou_type = self.mini_batch_utils.rpn_iou_type
#if rpn_iou_type == '2d':
# # Convert anchors to 2d iou format
# #anchors_for_2d_iou, _ = np.asarray(anchor_projector.project_to_bev(
# # anchors, kitti_utils.bev_extents))
# gt_boxes_for_2d_iou, _ = anchor_projector.project_to_bev(
# gt_anchors, kitti_utils.bev_extents)
#elif rpn_iou_type == '3d':
# # Convert anchors to 3d iou format for calculation
# anchors_for_3d_iou = box_3d_encoder.box_3d_to_3d_iou_format(
# anchor_boxes_3d)
# gt_boxes_for_3d_iou = \
# box_3d_encoder.box_3d_to_3d_iou_format(gt_boxes_3d)
#else:
# raise ValueError('Invalid rpn_iou_type {}', rpn_iou_type)
# Initialize sample and offset lists
num_anchors = len(anchor_boxes_3d)
all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length))
# Update anchor indices
all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices
# For each of the labels, generate samples
for gt_idx in range(len(gt_labels)):
gt_obj = gt_labels[gt_idx]
gt_box_2d = gt_boxes_2d[gt_idx]
# gt_box_for_2d_iou = gt_boxes_for_2d_iou[gt_idx]
ious = evaluation.two_d_iou(gt_box_2d, img_boxes_anchors)
# Only update indices with a higher iou than before
update_indices = np.greater(
ious, all_info[:, self.mini_batch_utils.col_ious])
# Get ious to update
ious_to_update = ious[update_indices]
# Calculate offsets, use 3D iou to get highest iou
anchors_to_update = img_boxes_anchors[update_indices]
#gt_anchor = box_3d_encoder.box_3d_to_anchor(gt_box_2d,
# ortho_rotate=True)
offsets = anchor_encoder.np_2d_box_to_offset(
anchors_to_update, gt_box_2d)
# Convert gt type to index
class_idx = kitti_utils.class_str_to_index(gt_obj.type)
# Update anchors info (indices already updated)
# [index, iou, (offsets), class_index]
all_info[update_indices,
self.mini_batch_utils.col_ious] = ious_to_update
all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self.
mini_batch_utils.col_offsets_hi] = offsets
all_info[update_indices,
self.mini_batch_utils.col_class_idx] = class_idx
return all_info, img_boxes_anchors, img_boxes_anchors_norm
def _calculate_anchors_info(self, all_anchor_boxes_bev,
empty_anchor_filter, gt_labels):
"""Calculates the list of anchor information in the format:
N x 8 [max_gt_2d_iou_r, max_gt_2d_iou_h, (6 x offsets), class_index]
max_gt_out - highest 2D iou with any ground truth box, using [anchor_r vs gt_r] or [anchor_h vs gt_h]
offsets - encoded offsets [dx, dy, d_dimx, d_dimy, d_angle, angle_face_class_index, (-180,0) or (0,180)]
class_index - the anchor's class as an index
(e.g. 0 or 1, for "Background" or "Car")
Args:
all_anchor_boxes_3d: list of anchors in box_3d format
N x [xc, yc, w, h, angle]
empty_anchor_filter: boolean mask of which anchors are non empty
gt_labels: list of Object Label data format containing ground truth
labels to generate positives/negatives from.
Returns:
list of anchor info
"""
# Check for ground truth objects
if len(gt_labels) == 0:
raise Warning("No valid ground truth label to generate anchors.")
kitti_utils = self._dataset.kitti_utils
# Filter empty anchors
anchor_indices = np.where(empty_anchor_filter)[0]
anchors = all_anchor_boxes_bev[empty_anchor_filter]
# Convert anchor_boxes_3d to anchor format
#anchors = box_bev_encoder.box_bev_to_anchor(anchor_boxes_bev)
# Convert gt to boxes_3d -> anchors -> iou format
gt_boxes_3d = np.asarray([
box_3d_encoder.object_label_to_box_3d(gt_obj)
for gt_obj in gt_labels
])
gt_anchors_norm, _ = box_3d_projector.project_to_bev_box(
gt_boxes_3d, self._area_extents[[0, 2]])
#bev_image_size = kitti_utils.area_extents / kitti_utils.voxel_size
bev_map_h, bev_map_w = self._bev_shape
#(N, 5) , (5, ) coorespondence element multiplification
gt_anchors = np.multiply(
gt_anchors_norm,
np.array([bev_map_w, bev_map_h, bev_map_w, bev_map_h, 1]))
iou_type = self.mini_batch_utils.retinanet_iou_type
if iou_type == '2d_rotate':
# Convert anchors to 2d iou format
anchors_for_2d_iou_r = anchors
gt_boxes_for_2d_iou_r = gt_anchors
elif iou_type == '2d':
# Convert anchors to 3d iou format for calculation
anchors_for_2d_iou_h = box_bev_encoder.box_bev_to_iou_h_format(
anchors)
anchors_for_2d_iou_h = anchors_for_2d_iou_h.astype(np.int32)
gt_boxes_for_2d_iou_h = box_bev_encoder.box_bev_to_iou_h_format(
gt_anchors)
gt_boxes_for_2d_iou_h = gt_boxes_for_2d_iou_h.astype(np.int32)
else:
raise ValueError('Invalid retinanet iou_type {}', iou_type)
# Initialize sample and offset lists
num_anchors = len(anchors)
all_info = np.zeros((num_anchors, self.mini_batch_utils.col_length))
# Update anchor indices
all_info[:, self.mini_batch_utils.col_anchor_indices] = anchor_indices
# For each of the labels, generate samples
for gt_idx in range(len(gt_labels)):
gt_obj = gt_labels[gt_idx]
gt_box_3d = box_3d_encoder.object_label_to_box_3d(gt_obj)
# Get 2D or 3D IoU for every anchor
if self.mini_batch_utils.retinanet_iou_type == '2d':
gt_box_for_2d_iou_h = gt_boxes_for_2d_iou_h[gt_idx]
ious = evaluation.two_d_iou(gt_box_for_2d_iou_h,
anchors_for_2d_iou_h)
elif self.mini_batch_utils.retinanet_iou_type == '2d_rotate':
gt_box_for_2d_iou_r = gt_boxes_for_2d_iou_r[gt_idx]
ious = evaluation.two_d_rotate_iou(gt_box_for_2d_iou_r,
anchors_for_2d_iou_r)
# Only update indices with a higher iou than before
update_indices = np.greater(
ious, all_info[:, self.mini_batch_utils.col_ious])
# Get ious to update
ious_to_update = ious[update_indices]
# Calculate offsets, use 3D iou to get highest iou
anchors_to_update = anchors[update_indices]
facing_obj_head = gt_obj.ry >= 0 #camera facing object's head.
gt_anchor = gt_anchors[gt_idx]
#turns (-pi, pi) to (-pi, 0) for gt_anchor's angle
if facing_obj_head:
gt_anchor[-1] -= np.pi
offsets_boxes = anchor_bev_encoder.anchor_to_offset(
anchors_to_update, gt_anchor)
gt_anchor_pred = anchor_bev_encoder.offset_to_anchor(
anchors_to_update, offsets_boxes)
#y axis 3d value
n_anchor = offsets_boxes.shape[0]
anchor_h = anchor_bev_encoder.get_default_anchor_h(n_anchor, 'np')
gt_h = [gt_obj.t[1], gt_obj.h]
offsets_h = anchor_bev_encoder.anchor_to_offset_h(anchor_h, gt_h)
gt_anchors_angle = np.zeros_like(offsets_boxes[:, 0],
dtype=np.int) + gt_obj.ry
offsets_angle_cls = orientation_encoder.orientation_to_angle_cls(
gt_anchors_angle)
offsets = np.hstack(
[offsets_boxes, offsets_h, offsets_angle_cls[:, np.newaxis]])
# Convert gt type to index
class_idx = kitti_utils.class_str_to_index(gt_obj.type)
# Update anchors info (indices already updated)
# [index, iou, (offsets), class_index]
all_info[update_indices,
self.mini_batch_utils.col_ious] = ious_to_update
all_info[update_indices, self.mini_batch_utils.col_offsets_lo:self.
mini_batch_utils.col_offsets_hi] = offsets
all_info[update_indices,
self.mini_batch_utils.col_class_idx] = class_idx
debug = False #True
if debug:
print(f'gt obj:{gt_box_3d}, gt anchor bev: {gt_anchor}')
print(f'anchors_to_update: {anchors_to_update[:1]}')
print(f'update at all_info: \n{all_info[update_indices][:1]}')
print(f'gt_from_anchor_offsets:\n{gt_anchor_pred[:1]}')
return all_info
