def compute_correspondence_metrics( keypoints_i1: Keypoints, keypoints_i2: Keypoints, corr_idxs_i1i2: np.ndarray, intrinsics_i1: Cal3Bundler, intrinsics_i2: Cal3Bundler, i2Ti1: Pose3, epipolar_distance_threshold: float, ) -> Tuple[int, float]: """Compute the metrics for the generated verified correspondence. Args: keypoints_i1: detected keypoints in image i1. keypoints_i2: detected keypoints in image i2. corr_idxs_i1i2: indices of correspondences. intrinsics_i1: intrinsics for i1. intrinsics_i2: intrinsics for i2. i2Ti1: relative pose. epipolar_distance_threshold: max epipolar distance to qualify as a correct match. Returns: Number of correct correspondences. Inlier Ratio, i.e. ratio of correspondences which are correct. """ number_correct = metric_utils.count_correct_correspondences( keypoints_i1.extract_indices(corr_idxs_i1i2[:, 0]), keypoints_i2.extract_indices(corr_idxs_i1i2[:, 1]), intrinsics_i1, intrinsics_i2, i2Ti1, epipolar_distance_threshold, ) return number_correct, number_correct / corr_idxs_i1i2.shape[0]
def estimate_F( self, keypoints_i1: Keypoints, keypoints_i2: Keypoints, match_indices: np.ndarray, robust_estimation_type: RobustEstimationType = RobustEstimationType. FM_RANSAC, ) -> Tuple[np.ndarray, np.ndarray]: """Estimate the Fundamental matrix from correspondences. Args: keypoints_i1: detected features in image #i1. keypoints_i2: detected features in image #i2. match_indices: matches as indices of features from both images, of shape (N3, 2), where N3 <= min(N1, N2), given N1 features from image 1, and N2 features from image 2. Returns: i2Fi1: Fundamental matrix, as 3x3 array. inlier_mask: boolean array of shape (N3,) indicating inlier matches. """ i2Fi1, inlier_mask = cv2.findFundamentalMat( keypoints_i1.extract_indices(match_indices[:, 0]).coordinates, keypoints_i2.extract_indices(match_indices[:, 1]).coordinates, method=getattr(cv2, robust_estimation_type.value), ransacReprojThreshold=self._estimation_threshold_px, confidence=RANSAC_SUCCESS_PROB, maxIters=RANSAC_MAX_ITERS, ) return i2Fi1, inlier_mask
def test_extract_indices_valid(self): """Test extraction of indices.""" # test without scales and responses input = Keypoints(coordinates=np.array([[1.3, 5], [20, 10], [5.0, 1.3], [2.1, 4.2]])) indices = np.array([0, 2]) expected = Keypoints(coordinates=np.array([[1.3, 5], [5.0, 1.3]])) computed = input.extract_indices(indices) self.assertEqual(computed, expected) # test without scales and responses input = Keypoints( coordinates=np.array([[1.3, 5], [20, 10], [5.0, 1.3], [2.1, 4.2]]), scales=np.array([0.2, 0.5, 0.3, 0.9]), responses=np.array([2.3, 1.2, 4.5, 0.2]), ) indices = np.array([0, 2]) expected = Keypoints( coordinates=np.array([[1.3, 5], [5.0, 1.3]]), scales=np.array([0.2, 0.3]), responses=np.array([2.3, 4.5]) ) computed = input.extract_indices(indices) self.assertEqual(computed, expected)
def test_extract_indices_empty(self): """Test extraction of indices, which are empty.""" # test without scales and responses input = Keypoints(coordinates=np.array([[1.3, 5], [20, 10], [5.0, 1.3], [2.1, 4.2]])) indices = np.array([]) computed = input.extract_indices(indices) self.assertEqual(len(computed), 0)
def estimate_F(self, keypoints_i1: Keypoints, keypoints_i2: Keypoints, match_indices: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: """Estimate the Fundamental matrix from correspondences. Args: keypoints_i1: detected features in image #i1. keypoints_i2: detected features in image #i2. match_indices: matches as indices of features from both images, of shape (N3, 2), where N3 <= min(N1, N2), given N1 features from image 1, and N2 features from image 2. Returns: i2Fi1: Fundamental matrix, as 3x3 array. inlier_mask: boolean array of shape (N3,) indicating inlier matches. """ i2Fi1, inlier_mask = cv2.findFundamentalMat( keypoints_i1.extract_indices(match_indices[:, 0]).coordinates, keypoints_i2.extract_indices(match_indices[:, 1]).coordinates, method=cv2.FM_LMEDS, ) return i2Fi1, inlier_mask
def compute_correspondence_metrics( keypoints_i1: Keypoints, keypoints_i2: Keypoints, corr_idxs_i1i2: np.ndarray, intrinsics_i1: Cal3Bundler, intrinsics_i2: Cal3Bundler, dist_threshold: float, gt_wTi1: Optional[Pose3] = None, gt_wTi2: Optional[Pose3] = None, gt_scene_mesh: Optional[Trimesh] = None, ) -> Tuple[Optional[np.ndarray], Optional[np.ndarray]]: """Checks the correspondences for epipolar distances and counts ones which are below the threshold. Args: keypoints_i1: keypoints in image i1. keypoints_i2: corr. keypoints in image i2. intrinsics_i1: intrinsics for i1. intrinsics_i2: intrinsics for i2. dist_threshold: max acceptable distance for a correct correspondence. gt_wTi1: ground truth pose of image i1. gt_wTi2: ground truth pose of image i2. gt_scene_mesh: ground truth triangular surface mesh of the scene in the world frame. Raises: ValueError: when the number of keypoints do not match. Returns: Boolean mask of which verified correspondences are classified as correct under Sampson error (using GT epipolar geometry). Reprojection error for every verified correspondence against GT geometry. """ if corr_idxs_i1i2.size == 0: return None, None if gt_wTi1 is None or gt_wTi2 is None: return None, None # Compute ground truth correspondences. matched_keypoints_i1 = keypoints_i1.extract_indices(corr_idxs_i1i2[:, 0]) matched_keypoints_i2 = keypoints_i2.extract_indices(corr_idxs_i1i2[:, 1]) # Check to see if a GT mesh is provided. if gt_scene_mesh is not None: gt_camera_i1 = PinholeCameraCal3Bundler(gt_wTi1, intrinsics_i1) gt_camera_i2 = PinholeCameraCal3Bundler(gt_wTi2, intrinsics_i2) return mesh_inlier_correspondences( matched_keypoints_i1, matched_keypoints_i2, gt_camera_i1, gt_camera_i2, gt_scene_mesh, dist_threshold, ) # If no mesh is provided, use squared Sampson error. gt_i2Ti1 = gt_wTi2.between(gt_wTi1) return epipolar_inlier_correspondences( matched_keypoints_i1, matched_keypoints_i2, intrinsics_i1, intrinsics_i2, gt_i2Ti1, dist_threshold, )
def verify( self, keypoints_i1: Keypoints, keypoints_i2: Keypoints, match_indices: np.ndarray, camera_intrinsics_i1: Cal3Bundler, camera_intrinsics_i2: Cal3Bundler, ) -> Tuple[Optional[Rot3], Optional[Unit3], np.ndarray]: """Performs verification of correspondences between two images to recover the relative pose and indices of verified correspondences. Args: keypoints_i1: detected features in image #i1. keypoints_i2: detected features in image #i2. match_indices: matches as indices of features from both images, of shape (N3, 2), where N3 <= min(N1, N2). camera_intrinsics_i1: intrinsics for image #i1. camera_intrinsics_i2: intrinsics for image #i2. Returns: Estimated rotation i2Ri1, or None if it cannot be estimated. Estimated unit translation i2Ui1, or None if it cannot be estimated. Indices of verified correspondences, of shape (N, 2) with N <= N3. These are subset of match_indices. """ if match_indices.shape[0] < self._min_matches: return self._failure_result if self._use_intrinsics_in_verification: uv_norm_i1 = feature_utils.normalize_coordinates(keypoints_i1.coordinates, camera_intrinsics_i1) uv_norm_i2 = feature_utils.normalize_coordinates(keypoints_i2.coordinates, camera_intrinsics_i2) K = np.eye(3) i2Ei1, inlier_mask = cv2.findEssentialMat( uv_norm_i1[match_indices[:, 0]], uv_norm_i2[match_indices[:, 1]], K, method=cv2.RANSAC, threshold=NORMALIZED_COORD_RANSAC_THRESH, prob=DEFAULT_RANSAC_SUCCESS_PROB, ) else: i2Fi1, inlier_mask = cv2.findFundamentalMat( keypoints_i1.extract_indices(match_indices[:, 0]).coordinates, keypoints_i2.extract_indices(match_indices[:, 1]).coordinates, method=cv2.FM_RANSAC, ransacReprojThreshold=PIXEL_COORD_RANSAC_THRESH, confidence=DEFAULT_RANSAC_SUCCESS_PROB, maxIters=10000, ) i2Ei1 = verification_utils.fundamental_to_essential_matrix( i2Fi1, camera_intrinsics_i1, camera_intrinsics_i2 ) inlier_idxs = np.where(inlier_mask.ravel() == 1)[0] (i2Ri1, i2Ui1) = verification_utils.recover_relative_pose_from_essential_matrix( i2Ei1, keypoints_i1.coordinates[match_indices[inlier_idxs, 0]], keypoints_i2.coordinates[match_indices[inlier_idxs, 1]], camera_intrinsics_i1, camera_intrinsics_i2, ) return i2Ri1, i2Ui1, match_indices[inlier_idxs]