def scipy_proj_error(x, args):
"""Calculates projection error of instance points with a 2D box.
Used for minimizing projection error when varying xz_dist and centroid_y.
Args:
x: array of inputs
xz_dist: distance along viewing angle
centroid_y: box centroid y
args: dict with additional data
'viewing_angle': viewing angle
'inst_points' = (N, 3) instance points
'cam_p' = (3, 4) camera projection matrix
'exp_grid_uv' = expected [u, v] grid projection
'rotate_view' = bool of whether to rotate by viewing angle
Returns:
proj_err_norm: projection error normalized by the number of valid pixels
"""
# Parse inputs from x
xz_dist = x[0]
centroid_y = x[1]
# Parse inputs from args
viewing_angle = args['viewing_angle']
inst_points = args['inst_points']
cam_p = args['cam_p']
exp_grid_uv = args['exp_grid_uv']
rotate_view = args['rotate_view']
pred_points_in_img, valid_points_mask = instance_utils.proj_points(
xz_dist, centroid_y, viewing_angle, inst_points, cam_p, rotate_view=rotate_view)
proj_err_norm = np_proj_error(pred_points_in_img, valid_points_mask, exp_grid_uv)
return proj_err_norm
def scipy_convex_hull_mask_inv_iou_with_viewing_angle(x, args):
"""Computes masks by calculating a convex hull from points. Creates two masks (if possible),
one for the estimated foreground pixels and one for the estimated background pixels.
Minimizes inverted IoU by varying xz_dist, centroid_y, and viewing angle.
Args:
x: array of inputs
xz_dist: distance along viewing angle
centroid_y: box centroid y
viewing_angle: viewing angle
args: dict with additional data
'viewing_angle': viewing angle
'inst_points' = (N, 3) instance points
'cam_p' = (3, 4) camera projection matrix
'im_shape' = image shape [im_height, im_width]
'gt_hull_mask' = expected mask created from instance mask
Returns:
inverted_iou: 1.0 - IoU of the mask computed from the convex hull and the gt hull mask
"""
# Parse inputs from x
xz_dist = x[0]
centroid_y = x[1]
viewing_angle = x[2]
# Parse inputs from args
inst_points = args['inst_points']
cam_p = args['cam_p']
im_shape = args['im_shape']
gt_hull_mask = args['gt_hull_mask']
pred_points_in_img, valid_points_mask = instance_utils.proj_points(
xz_dist, centroid_y, viewing_angle, inst_points, cam_p)
iou = convex_hull_mask_iou(pred_points_in_img, im_shape, gt_hull_mask)
# Invert IoU so it can be minimized
inverted_iou = 1.0 - iou
return inverted_iou
def np_proj_err_rgb(xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p,
inst_rgb, image, valid_mask_map):
# Get instance RGB
inst_rgb_map = inst_rgb.reshape(48, 48, 3)
# Project points to image
proj_uv, _ = instance_utils.proj_points(
xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p)
# Get RGB values of projected pixels
proj_uv_int = np.round(proj_uv).astype(np.int32)
guess_rgb = image[proj_uv_int[1], proj_uv_int[0]]
guess_rgb_map = guess_rgb.reshape(48, 48, 3) * np.expand_dims(valid_mask_map, 2)
# Check image similarity
image_diff_map = abs(inst_rgb_map - guess_rgb_map)
image_diff_map_norm = np.sum(image_diff_map, axis=2) / 255.0
image_diff_total = np.sum(image_diff_map_norm) / np.count_nonzero(valid_mask_map)
return image_diff_total
def np_proj_err_rgb_images(xz_dist, centroid_y, viewing_angle,
cam2_inst_points_local, cam_p,
inst_rgb, inst_mask, image, valid_mask_map, box_2d,
guess_row_col, show_images=False):
"""(Work in progress) Calculates the projection error based on RGB similarity and shows
images for comparison.
Args:
xz_dist: Distance along viewing angle
centroid_y: Object centroid y
viewing_angle: Viewing angle
cam2_inst_points_local: (N, 3) Instance points in local frame
cam_p: (3, 4) Camera projection matrix
inst_rgb: List of instance RGB values
image: Image of sample
valid_mask_map: (H, W) Map mask of valid values
guess_row_col: Guess index, used for numbering images
show_images: (optional) Whether to show comparison images
Returns:
image_diff_total: Lowest image difference
"""
# Get projection into image
proj_uv, valid_points_mask = instance_utils.proj_points(
xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p)
# Get RGB values of projected pixels
proj_uv_int = np.round(proj_uv).astype(np.int32)
guess_rgb = image[proj_uv_int[1], proj_uv_int[0]]
guess_rgb_map = guess_rgb.reshape(48, 48, 3) * np.expand_dims(valid_mask_map, 2)
# Estimated image
est_image = np.copy(image) * np.expand_dims(~inst_mask, 2)
est_image[proj_uv_int[1], proj_uv_int[0]] = inst_rgb
est_image[proj_uv_int[1]-1, proj_uv_int[0]] = inst_rgb
est_image[proj_uv_int[1]+1, proj_uv_int[0]] = inst_rgb
est_image[proj_uv_int[1], proj_uv_int[0]-1] = inst_rgb
est_image[proj_uv_int[1], proj_uv_int[0]+1] = inst_rgb
box_2d_int = np.round(box_2d).astype(np.int32)
est_inst_rgb = est_image[box_2d_int[0]:box_2d_int[2], box_2d_int[1]:box_2d_int[3]]
est_inst_rgb_resized = cv2.resize(est_inst_rgb, (48, 48))
# Check image similarity
inst_rgb_map = inst_rgb.reshape(48, 48, 3)
# image_diff_map = abs(inst_rgb_map - guess_rgb_map)
image_diff_map = abs(inst_rgb_map - est_inst_rgb_resized)
image_diff_map_norm = np.sum(image_diff_map, axis=2) / 255.0
image_diff_total = np.sum(image_diff_map_norm)
if show_images:
# cv2_size = (160, 160)
cv2_size = (90, 90)
cv2_size = (120, 120)
# # Show instance RGB for comparison
# inst_rgb_map_resized = cv2.resize(inst_rgb_map, cv2_size)
# vis_utils.cv2_imshow('inst_rgb_map_resized {}'.format(guess_row_col),
# inst_rgb_map_resized,
# size_wh=cv2_size, row_col=guess_row_col)
#
# # Show guess
# guess_rgb_map_resized = cv2.resize(guess_rgb_map, (200, 200))
# vis_utils.cv2_imshow('guess_rgb_map_resized {}'.format(guess_row_col),
# guess_rgb_map_resized,
# size_wh=cv2_size, row_col=guess_row_col)
vis_utils.cv2_imshow('est_inst_rgb_resized {}'.format(guess_row_col),
est_inst_rgb_resized,
size_wh=cv2_size, row_col=guess_row_col)
# combined = cv2.addWeighted(inst_rgb_map, 0.5, est_inst_rgb_resized, 0.5, 0.0)
# vis_utils.cv2_imshow('combined {}'.format(guess_row_col),
# combined,
# size_wh=cv2_size, row_col=guess_row_col)
# vis_utils.cv2_imshow('image_diff_map_norm {}'.format(guess_row_col),
# image_diff_map_norm,
# size_wh=cv2_size, row_col=guess_row_col)
# vis_utils.cv2_imshow('valid_mask {}'.format(centroid_y),
# (valid_mask_map * 255).astype(np.uint8),
# size_wh=cv2_size, row_col=guess_row_col)
return image_diff_total