def scipy_proj_error(x, args): """Calculates projection error of instance points with a 2D box. Used for minimizing projection error when varying xz_dist and centroid_y. Args: x: array of inputs xz_dist: distance along viewing angle centroid_y: box centroid y args: dict with additional data 'viewing_angle': viewing angle 'inst_points' = (N, 3) instance points 'cam_p' = (3, 4) camera projection matrix 'exp_grid_uv' = expected [u, v] grid projection 'rotate_view' = bool of whether to rotate by viewing angle Returns: proj_err_norm: projection error normalized by the number of valid pixels """ # Parse inputs from x xz_dist = x[0] centroid_y = x[1] # Parse inputs from args viewing_angle = args['viewing_angle'] inst_points = args['inst_points'] cam_p = args['cam_p'] exp_grid_uv = args['exp_grid_uv'] rotate_view = args['rotate_view'] pred_points_in_img, valid_points_mask = instance_utils.proj_points( xz_dist, centroid_y, viewing_angle, inst_points, cam_p, rotate_view=rotate_view) proj_err_norm = np_proj_error(pred_points_in_img, valid_points_mask, exp_grid_uv) return proj_err_norm
def scipy_convex_hull_mask_inv_iou_with_viewing_angle(x, args): """Computes masks by calculating a convex hull from points. Creates two masks (if possible), one for the estimated foreground pixels and one for the estimated background pixels. Minimizes inverted IoU by varying xz_dist, centroid_y, and viewing angle. Args: x: array of inputs xz_dist: distance along viewing angle centroid_y: box centroid y viewing_angle: viewing angle args: dict with additional data 'viewing_angle': viewing angle 'inst_points' = (N, 3) instance points 'cam_p' = (3, 4) camera projection matrix 'im_shape' = image shape [im_height, im_width] 'gt_hull_mask' = expected mask created from instance mask Returns: inverted_iou: 1.0 - IoU of the mask computed from the convex hull and the gt hull mask """ # Parse inputs from x xz_dist = x[0] centroid_y = x[1] viewing_angle = x[2] # Parse inputs from args inst_points = args['inst_points'] cam_p = args['cam_p'] im_shape = args['im_shape'] gt_hull_mask = args['gt_hull_mask'] pred_points_in_img, valid_points_mask = instance_utils.proj_points( xz_dist, centroid_y, viewing_angle, inst_points, cam_p) iou = convex_hull_mask_iou(pred_points_in_img, im_shape, gt_hull_mask) # Invert IoU so it can be minimized inverted_iou = 1.0 - iou return inverted_iou
def np_proj_err_rgb(xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p, inst_rgb, image, valid_mask_map): # Get instance RGB inst_rgb_map = inst_rgb.reshape(48, 48, 3) # Project points to image proj_uv, _ = instance_utils.proj_points( xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p) # Get RGB values of projected pixels proj_uv_int = np.round(proj_uv).astype(np.int32) guess_rgb = image[proj_uv_int[1], proj_uv_int[0]] guess_rgb_map = guess_rgb.reshape(48, 48, 3) * np.expand_dims(valid_mask_map, 2) # Check image similarity image_diff_map = abs(inst_rgb_map - guess_rgb_map) image_diff_map_norm = np.sum(image_diff_map, axis=2) / 255.0 image_diff_total = np.sum(image_diff_map_norm) / np.count_nonzero(valid_mask_map) return image_diff_total
def np_proj_err_rgb_images(xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p, inst_rgb, inst_mask, image, valid_mask_map, box_2d, guess_row_col, show_images=False): """(Work in progress) Calculates the projection error based on RGB similarity and shows images for comparison. Args: xz_dist: Distance along viewing angle centroid_y: Object centroid y viewing_angle: Viewing angle cam2_inst_points_local: (N, 3) Instance points in local frame cam_p: (3, 4) Camera projection matrix inst_rgb: List of instance RGB values image: Image of sample valid_mask_map: (H, W) Map mask of valid values guess_row_col: Guess index, used for numbering images show_images: (optional) Whether to show comparison images Returns: image_diff_total: Lowest image difference """ # Get projection into image proj_uv, valid_points_mask = instance_utils.proj_points( xz_dist, centroid_y, viewing_angle, cam2_inst_points_local, cam_p) # Get RGB values of projected pixels proj_uv_int = np.round(proj_uv).astype(np.int32) guess_rgb = image[proj_uv_int[1], proj_uv_int[0]] guess_rgb_map = guess_rgb.reshape(48, 48, 3) * np.expand_dims(valid_mask_map, 2) # Estimated image est_image = np.copy(image) * np.expand_dims(~inst_mask, 2) est_image[proj_uv_int[1], proj_uv_int[0]] = inst_rgb est_image[proj_uv_int[1]-1, proj_uv_int[0]] = inst_rgb est_image[proj_uv_int[1]+1, proj_uv_int[0]] = inst_rgb est_image[proj_uv_int[1], proj_uv_int[0]-1] = inst_rgb est_image[proj_uv_int[1], proj_uv_int[0]+1] = inst_rgb box_2d_int = np.round(box_2d).astype(np.int32) est_inst_rgb = est_image[box_2d_int[0]:box_2d_int[2], box_2d_int[1]:box_2d_int[3]] est_inst_rgb_resized = cv2.resize(est_inst_rgb, (48, 48)) # Check image similarity inst_rgb_map = inst_rgb.reshape(48, 48, 3) # image_diff_map = abs(inst_rgb_map - guess_rgb_map) image_diff_map = abs(inst_rgb_map - est_inst_rgb_resized) image_diff_map_norm = np.sum(image_diff_map, axis=2) / 255.0 image_diff_total = np.sum(image_diff_map_norm) if show_images: # cv2_size = (160, 160) cv2_size = (90, 90) cv2_size = (120, 120) # # Show instance RGB for comparison # inst_rgb_map_resized = cv2.resize(inst_rgb_map, cv2_size) # vis_utils.cv2_imshow('inst_rgb_map_resized {}'.format(guess_row_col), # inst_rgb_map_resized, # size_wh=cv2_size, row_col=guess_row_col) # # # Show guess # guess_rgb_map_resized = cv2.resize(guess_rgb_map, (200, 200)) # vis_utils.cv2_imshow('guess_rgb_map_resized {}'.format(guess_row_col), # guess_rgb_map_resized, # size_wh=cv2_size, row_col=guess_row_col) vis_utils.cv2_imshow('est_inst_rgb_resized {}'.format(guess_row_col), est_inst_rgb_resized, size_wh=cv2_size, row_col=guess_row_col) # combined = cv2.addWeighted(inst_rgb_map, 0.5, est_inst_rgb_resized, 0.5, 0.0) # vis_utils.cv2_imshow('combined {}'.format(guess_row_col), # combined, # size_wh=cv2_size, row_col=guess_row_col) # vis_utils.cv2_imshow('image_diff_map_norm {}'.format(guess_row_col), # image_diff_map_norm, # size_wh=cv2_size, row_col=guess_row_col) # vis_utils.cv2_imshow('valid_mask {}'.format(centroid_y), # (valid_mask_map * 255).astype(np.uint8), # size_wh=cv2_size, row_col=guess_row_col) return image_diff_total