def test_get_top_k_without_responses(self):
"""Tests the selection of top entries in a keypoints w/o responses."""
input_keypoints = Keypoints(
coordinates=np.array(
[
[10.0, 23.2],
[37.1, 50.2],
[90.1, 10.7],
[150.0, 122.0],
[250.0, 49.0],
]
)
)
# test with requested length > current length
requested_length = len(input_keypoints) * 2
computed = input_keypoints.get_top_k(requested_length)
self.assertEqual(computed, input_keypoints)
# test with requested length < current length
requested_length = 2
computed = input_keypoints.get_top_k(requested_length)
expected = Keypoints(coordinates=input_keypoints.coordinates[:requested_length])
self.assertEqual(computed, expected)
def estimate_F(
self,
keypoints_i1: Keypoints,
keypoints_i2: Keypoints,
match_indices: np.ndarray,
robust_estimation_type: RobustEstimationType = RobustEstimationType.
FM_RANSAC,
) -> Tuple[np.ndarray, np.ndarray]:
"""Estimate the Fundamental matrix from correspondences.
Args:
keypoints_i1: detected features in image #i1.
keypoints_i2: detected features in image #i2.
match_indices: matches as indices of features from both images, of shape (N3, 2), where N3 <= min(N1, N2),
given N1 features from image 1, and N2 features from image 2.
Returns:
i2Fi1: Fundamental matrix, as 3x3 array.
inlier_mask: boolean array of shape (N3,) indicating inlier matches.
"""
i2Fi1, inlier_mask = cv2.findFundamentalMat(
keypoints_i1.extract_indices(match_indices[:, 0]).coordinates,
keypoints_i2.extract_indices(match_indices[:, 1]).coordinates,
method=getattr(cv2, robust_estimation_type.value),
ransacReprojThreshold=self._estimation_threshold_px,
confidence=RANSAC_SUCCESS_PROB,
maxIters=RANSAC_MAX_ITERS,
)
return i2Fi1, inlier_mask
def get_dummy_keypoints_list() -> List[Keypoints]:
""" """
img1_kp_coords = np.array([[1, 1], [2, 2], [3, 3]])
img1_kp_scale = np.array([6.0, 9.0, 8.5])
img2_kp_coords = np.array([
[1, 1],
[2, 2],
[3, 3],
[4, 4],
[5, 5],
[6, 6],
[7, 7],
[8, 8],
])
img3_kp_coords = np.array([
[1, 1],
[2, 2],
[3, 3],
[4, 4],
[5, 5],
[6, 6],
[7, 7],
[8, 8],
[9, 9],
[10, 10],
])
keypoints_list = [
Keypoints(coordinates=img1_kp_coords, scales=img1_kp_scale),
Keypoints(coordinates=img2_kp_coords),
Keypoints(coordinates=img3_kp_coords),
]
return keypoints_list
def simulate_two_planes_scene(M: int, N: int) -> Tuple[Keypoints, Keypoints, EssentialMatrix]:
"""Generate a scene where 3D points are on two planes, and projects the points to the 2 cameras. There are M points
on plane 1, and N points on plane 2.
The two planes in this test are:
1. -10x -y -20z +150 = 0
2. 15x -2y -35z +200 = 0
Args:
M: number of points on 1st plane.
N: number of points on 2nd plane.
Returns:
keypoints for image i1, of length (M+N).
keypoints for image i2, of length (M+N).
Essential matrix i2Ei1.
"""
# range of 3D points
range_x_coordinate = (-5, 7)
range_y_coordinate = (-10, 10)
# define the plane equation
plane1_coeffs = (-10, -1, -20, 150)
plane2_coeffs = (15, -2, -35, 200)
# sample the points from planes
plane1_points = sample_points_on_plane(plane1_coeffs, range_x_coordinate, range_y_coordinate, M)
plane2_points = sample_points_on_plane(plane2_coeffs, range_x_coordinate, range_y_coordinate, N)
points_3d = np.vstack((plane1_points, plane2_points))
# define the camera poses and compute the essential matrix
wti1 = np.array([0.1, 0, -20])
wti2 = np.array([1, -2, -20.4])
wRi1 = Rot3.RzRyRx(np.pi / 20, 0, 0.0)
wRi2 = Rot3.RzRyRx(0.0, np.pi / 6, 0.0)
wTi1 = Pose3(wRi1, wti1)
wTi2 = Pose3(wRi2, wti2)
i2Ti1 = wTi2.between(wTi1)
i2Ei1 = EssentialMatrix(i2Ti1.rotation(), Unit3(i2Ti1.translation()))
# project 3D points to 2D image measurements
intrinsics = Cal3Bundler()
camera_i1 = PinholeCameraCal3Bundler(wTi1, intrinsics)
camera_i2 = PinholeCameraCal3Bundler(wTi2, intrinsics)
uv_im1 = []
uv_im2 = []
for point in points_3d:
uv_im1.append(camera_i1.project(point))
uv_im2.append(camera_i2.project(point))
uv_im1 = np.vstack(uv_im1)
uv_im2 = np.vstack(uv_im2)
# return the points as keypoints and the essential matrix
return Keypoints(coordinates=uv_im1), Keypoints(coordinates=uv_im2), i2Ei1
def test_cast_to_opencv_keypoints(self):
"""Tests conversion of GTSFM's keypoints to OpenCV's keypoints."""
gtsfm_keypoints = Keypoints(
coordinates=np.array([[1.3, 5], [20, 10]]),
scales=np.array([1.0, 5.2]),
responses=np.array([4.2, 3.2]),
)
results = gtsfm_keypoints.cast_to_opencv_keypoints()
# check the length of the result
self.assertEqual(len(results), len(gtsfm_keypoints))
# check all the keypoint values
for idx in range(len(gtsfm_keypoints)):
opencv_kp = results[idx]
self.assertAlmostEqual(opencv_kp.pt[0],
gtsfm_keypoints.coordinates[idx, 0],
places=5)
self.assertAlmostEqual(opencv_kp.pt[1],
gtsfm_keypoints.coordinates[idx, 1],
places=5)
self.assertAlmostEqual(opencv_kp.size,
gtsfm_keypoints.scales[idx],
places=5)
self.assertAlmostEqual(opencv_kp.response,
gtsfm_keypoints.responses[idx],
places=5)
def compute_correspondence_metrics(
keypoints_i1: Keypoints,
keypoints_i2: Keypoints,
corr_idxs_i1i2: np.ndarray,
intrinsics_i1: Cal3Bundler,
intrinsics_i2: Cal3Bundler,
i2Ti1: Pose3,
epipolar_distance_threshold: float,
) -> Tuple[int, float]:
"""Compute the metrics for the generated verified correspondence.
Args:
keypoints_i1: detected keypoints in image i1.
keypoints_i2: detected keypoints in image i2.
corr_idxs_i1i2: indices of correspondences.
intrinsics_i1: intrinsics for i1.
intrinsics_i2: intrinsics for i2.
i2Ti1: relative pose.
epipolar_distance_threshold: max epipolar distance to qualify as a correct match.
Returns:
Number of correct correspondences.
Inlier Ratio, i.e. ratio of correspondences which are correct.
"""
number_correct = metric_utils.count_correct_correspondences(
keypoints_i1.extract_indices(corr_idxs_i1i2[:, 0]),
keypoints_i2.extract_indices(corr_idxs_i1i2[:, 1]),
intrinsics_i1,
intrinsics_i2,
i2Ti1,
epipolar_distance_threshold,
)
return number_correct, number_correct / corr_idxs_i1i2.shape[0]
def detect_and_describe(self,
image: Image) -> Tuple[Keypoints, np.ndarray]:
"""Jointly generate keypoint detections and their associated descriptors from a single image."""
# TODO(ayushbaid): fix inference issue #110
device = torch.device("cuda" if self._use_cuda else "cpu")
model = SuperPoint(self._config).to(device)
model.eval()
# Compute features.
image_tensor = torch.from_numpy(
np.expand_dims(
image_utils.rgb_to_gray_cv(image).value_array.astype(
np.float32) / 255.0, (0, 1))).to(device)
with torch.no_grad():
model_results = model({"image": image_tensor})
torch.cuda.empty_cache()
# Unpack results.
coordinates = model_results["keypoints"][0].detach().cpu().numpy()
scores = model_results["scores"][0].detach().cpu().numpy()
keypoints = Keypoints(coordinates, scales=None, responses=scores)
descriptors = model_results["descriptors"][0].detach().cpu().numpy().T
# Filter features.
if image.mask is not None:
keypoints, valid_idxs = keypoints.filter_by_mask(image.mask)
descriptors = descriptors[valid_idxs]
keypoints, selection_idxs = keypoints.get_top_k(self.max_keypoints)
descriptors = descriptors[selection_idxs]
return keypoints, descriptors
def load_argoverse_log_annotated_correspondences():
"""Annotated from Argoverse, ring front-center camera, from vehicle log subdir:
'train1/273c1883-673a-36bf-b124-88311b1a80be/ring_front_center'
Image pair annotated at the following timestamps:
ts1 = 315975640448534784 # nano-second timestamp
ts2 = 315975643412234000
with img_names:
'ring_front_center_315975640448534784.jpg',
'ring_front_center_315975643412234000.jpg'
"""
fname = "argoverse_315975640448534784__315975643412234000.pkl"
pkl_fpath = ARGOVERSE_TEST_DATA_ROOT / f"labeled_correspondences/{fname}"
d = load_pickle_file(pkl_fpath)
X1 = np.array(d["x1"])
Y1 = np.array(d["y1"])
X2 = np.array(d["x2"])
Y2 = np.array(d["y2"])
img1_uv = np.hstack([X1.reshape(-1, 1), Y1.reshape(-1, 1)]).astype(np.float32)
img2_uv = np.hstack([X2.reshape(-1, 1), Y2.reshape(-1, 1)]).astype(np.float32)
keypoints_i1 = Keypoints(img1_uv)
keypoints_i2 = Keypoints(img2_uv)
return keypoints_i1, keypoints_i2
def test_extract_indices_empty(self):
"""Test extraction of indices, which are empty."""
# test without scales and responses
input = Keypoints(coordinates=np.array([[1.3, 5], [20, 10], [5.0, 1.3], [2.1, 4.2]]))
indices = np.array([])
computed = input.extract_indices(indices)
self.assertEqual(len(computed), 0)
def setUp(self):
"""Set up the data association object and test data."""
super().setUp()
self.obj = DummyDataAssociation(0.5, 2)
# set up ground truth data for comparison
self.dummy_corr_idxs_dict = {
(0, 1): np.array([[0, 2]]),
(1, 2): np.array([[2, 3], [4, 5], [7, 9]]),
(0, 2): np.array([[1, 8]]),
}
self.keypoints_list = [
Keypoints(coordinates=np.array([[12, 16], [13, 18], [0, 10]])),
Keypoints(coordinates=np.array([
[8, 2],
[16, 14],
[22, 23],
[1, 6],
[50, 50],
[16, 12],
[82, 121],
[39, 60],
])),
Keypoints(coordinates=np.array([
[1, 1],
[8, 13],
[40, 6],
[82, 21],
[1, 6],
[12, 18],
[15, 14],
[25, 28],
[7, 10],
[14, 17],
])),
]
# Generate two poses for use in triangulation tests
# Looking along X-axis, 1 meter above ground plane (x-y)
upright = Rot3.Ypr(-np.pi / 2, 0.0, -np.pi / 2)
pose1 = Pose3(upright, Point3(0, 0, 1))
# create second camera 1 meter to the right of first camera
pose2 = pose1.compose(Pose3(Rot3(), Point3(1, 0, 0)))
self.poses = Pose3Vector()
self.poses.append(pose1)
self.poses.append(pose2)
# landmark ~5 meters infront of camera
self.expected_landmark = Point3(5, 0.5, 1.2)
def test_extract_indices_valid(self):
"""Test extraction of indices."""
# test without scales and responses
input = Keypoints(coordinates=np.array([[1.3, 5], [20, 10], [5.0, 1.3], [2.1, 4.2]]))
indices = np.array([0, 2])
expected = Keypoints(coordinates=np.array([[1.3, 5], [5.0, 1.3]]))
computed = input.extract_indices(indices)
self.assertEqual(computed, expected)
# test without scales and responses
input = Keypoints(
coordinates=np.array([[1.3, 5], [20, 10], [5.0, 1.3], [2.1, 4.2]]),
scales=np.array([0.2, 0.5, 0.3, 0.9]),
responses=np.array([2.3, 1.2, 4.5, 0.2]),
)
indices = np.array([0, 2])
expected = Keypoints(
coordinates=np.array([[1.3, 5], [5.0, 1.3]]), scales=np.array([0.2, 0.3]), responses=np.array([2.3, 4.5])
)
computed = input.extract_indices(indices)
self.assertEqual(computed, expected)
def test_equality(self):
"""Tests the equality checker."""
# Test with None scales and responses.
obj1 = Keypoints(coordinates=COORDINATES)
obj2 = Keypoints(coordinates=COORDINATES)
self.assertEqual(obj1, obj2)
# test with coordinates and scales.
obj1 = Keypoints(coordinates=COORDINATES,
scales=SCALES,
responses=RESPONSES)
obj2 = Keypoints(coordinates=COORDINATES,
scales=SCALES,
responses=RESPONSES)
self.assertEqual(obj1, obj2)
# Test with one object having scales and other not having scales.
obj1 = Keypoints(coordinates=COORDINATES, scales=SCALES)
obj2 = Keypoints(coordinates=COORDINATES)
print(obj1 != obj2)
self.assertNotEqual(obj1, obj2)
# Test with one object having responses and other not having responses.
obj1 = Keypoints(coordinates=COORDINATES, responses=RESPONSES)
obj2 = Keypoints(coordinates=COORDINATES)
self.assertNotEqual(obj1, obj2)
def setUp(self):
"""Create keypoints."""
num_points = 5
normalized_coordinates_i1 = []
normalized_coordinates_i2 = []
for i in range(num_points):
track = EXAMPLE_DATA.get_track(i)
normalized_coordinates_i1.append(track.measurement(0)[1])
normalized_coordinates_i2.append(track.measurement(1)[1])
normalized_coordinates_i1 = np.array(normalized_coordinates_i1)
normalized_coordinates_i2 = np.array(normalized_coordinates_i2)
self.keypoints_i1 = Keypoints(normalized_coordinates_i1)
self.keypoints_i2 = Keypoints(normalized_coordinates_i2)
self.corr_idxs = np.hstack([np.arange(5).reshape(-1, 1)] * 2)
def test_empty_input(self):
"""Tests the matches when there are no descriptors."""
nonempty_keypoints, _, nonempty_descriptors, _, _, _ = generate_random_input(
)
empty_keypoints = Keypoints(coordinates=np.array([]))
empty_descriptors = np.array([])
im_shape_i1 = (300, 200)
im_shape_i2 = (300, 200)
# no keypoints for just i1
result = self.matcher.match(empty_keypoints, nonempty_keypoints,
empty_descriptors, nonempty_descriptors,
im_shape_i1, im_shape_i2)
self.assertEqual(result.size, 0)
# no keypoints for just i2
result = self.matcher.match(nonempty_keypoints, empty_keypoints,
nonempty_descriptors, empty_descriptors,
im_shape_i1, im_shape_i2)
self.assertEqual(result.size, 0)
# no keypoints for both i1 and i2
result = self.matcher.match(
deepcopy(empty_keypoints),
deepcopy(empty_keypoints),
deepcopy(empty_descriptors),
deepcopy(empty_descriptors),
im_shape_i1,
im_shape_i2,
)
self.assertEqual(result.size, 0)
def detect(self, image: Image) -> Keypoints:
"""Detect the features in an image by using random numbers.
Args:
image: input image.
Returns:
detected keypoints, with maximum length of max_keypoints.
"""
np.random.seed(
int(1000 * np.sum(image.value_array, axis=None) % (2 ^ 32)))
num_detections = np.random.randint(0, high=15, size=(1)).item()
# assign the coordinates
coordinates = np.random.randint(
low=[0, 0],
high=[image.value_array.shape[1], image.value_array.shape[0]],
size=(num_detections, 2),
)
# assign the scale
scales = np.random.rand(num_detections)
# assing responses
responses = np.random.rand(num_detections)
return Keypoints(coordinates, scales, responses)
def test_create_computation_graph(self):
"""Checks the dask computation graph."""
# testing some indices
idxs_under_test = [0, 5]
for idx in idxs_under_test:
test_image = self.loader.get_image(idx)
test_keypoints = Keypoints(coordinates=np.random.randint(
low=[0, 0],
high=[test_image.width, test_image.height],
size=(np.random.randint(5, 10), 2),
))
descriptor_graph = self.descriptor.create_computation_graph(
dask.delayed(test_image),
dask.delayed(test_keypoints),
)
with dask.config.set(scheduler="single-threaded"):
descriptors = dask.compute(descriptor_graph)[0]
expected_descriptors = self.descriptor.describe(
test_image, test_keypoints)
np.testing.assert_allclose(descriptors, expected_descriptors)
def test_compute_keypoint_intersections(self) -> None:
"""Tests `compute_keypoint_intersections()` function."""
# Create cube mesh with side length one centered at origin.
box = trimesh.primitives.Box()
# Create arrangement of 4 cameras in x-z plane pointing at the cube.
fx, k1, k2, u0, v0 = 10, 0, 0, 1, 1
calibration = Cal3Bundler(fx, k1, k2, u0, v0)
cam_pos = [[2, 0, 0], [-2, 0, 0], [0, 0, 2], [0, 0, -2]]
target_pos = [0, 0, 0]
up_vector = [0, -1, 0]
cams = [
PinholeCameraCal3Bundler().Lookat(c, target_pos, up_vector,
calibration) for c in cam_pos
]
# Project keypoint at center of each simulated image and record intersection.
kpt = Keypoints(coordinates=np.array([[1, 1]]).astype(np.float32))
expected_intersections = [[0.5, 0, 0], [-0.5, 0, 0], [0, 0, 0.5],
[0, 0, -0.5]]
estimated_intersections = []
for cam in cams:
_, intersection = metric_utils.compute_keypoint_intersections(
kpt, cam, box, verbose=True)
estimated_intersections.append(intersection.flatten().tolist())
np.testing.assert_allclose(expected_intersections,
estimated_intersections)
def test_constructor_with_all_inputs(self):
"""Tests the construction of keypoints with all data."""
result = Keypoints(coordinates=COORDINATES, scales=SCALES, responses=RESPONSES)
np.testing.assert_array_equal(result.coordinates, COORDINATES)
np.testing.assert_array_equal(result.scales, SCALES)
np.testing.assert_array_equal(result.responses, RESPONSES)
def test_constructor_with_coordinates_only(self):
"""Tests the construction of keypoints with just coordinates."""
result = Keypoints(coordinates=COORDINATES)
np.testing.assert_array_equal(result.coordinates, COORDINATES)
self.assertIsNone(result.responses)
self.assertIsNone(result.scales)
def test_with_no_features(self):
"""Checks that empty feature inputs works well."""
input_image = self.loader.get_image(0)
input_keypoints = Keypoints(coordinates=np.array([]))
result = self.descriptor.describe(input_image, input_keypoints)
self.assertEqual(0, result.size)
def generate_random_keypoints(num_keypoints: int,
image_shape: Tuple[int, int]) -> Keypoints:
"""Generates random keypoints within the image bounds.
Args:
num_keypoints: number of features to generate.
image_shape: size of the image.
Returns:
generated keypoints.
"""
if num_keypoints == 0:
return Keypoints(coordinates=np.array([]))
return Keypoints(coordinates=np.random.randint(
[0, 0], high=image_shape, size=(num_keypoints, 2)).astype(np.float32))
def estimate_F(self, keypoints_i1: Keypoints, keypoints_i2: Keypoints,
match_indices: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
"""Estimate the Fundamental matrix from correspondences.
Args:
keypoints_i1: detected features in image #i1.
keypoints_i2: detected features in image #i2.
match_indices: matches as indices of features from both images, of shape (N3, 2), where N3 <= min(N1, N2),
given N1 features from image 1, and N2 features from image 2.
Returns:
i2Fi1: Fundamental matrix, as 3x3 array.
inlier_mask: boolean array of shape (N3,) indicating inlier matches.
"""
i2Fi1, inlier_mask = cv2.findFundamentalMat(
keypoints_i1.extract_indices(match_indices[:, 0]).coordinates,
keypoints_i2.extract_indices(match_indices[:, 1]).coordinates,
method=cv2.FM_LMEDS,
)
return i2Fi1, inlier_mask
def test_get_top_k_with_responses(self):
"""Tests the selection of top entries in a keypoints with responses."""
input_keypoints = Keypoints(
coordinates=np.array(
[
[10.0, 23.2],
[37.1, 50.2],
[90.1, 10.7],
[150.0, 122.0],
[250.0, 49.0],
]
),
scales=np.array([1, 3, 2, 3.2, 1.8]),
responses=np.array([0.3, 0.7, 0.9, 0.1, 0.2]),
)
# test with requested length > current length
requested_length = len(input_keypoints) * 2
computed = input_keypoints.get_top_k(requested_length)
self.assertEqual(computed, input_keypoints)
# test with requested length < current length
requested_length = 2
computed = input_keypoints.get_top_k(requested_length)
expected = Keypoints(
coordinates=np.array(
[
[37.1, 50.2],
[90.1, 10.7],
]
),
scales=np.array([3, 2]),
responses=np.array([0.7, 0.9]),
)
# compare in an order-insensitive fashion
self.compare_without_ordering(computed, expected)
def test_result_size(self):
"""Check if the number of descriptors are same as number of features."""
input_image = self.loader.get_image(0)
input_keypoints = Keypoints(coordinates=np.random.randint(
low=[0, 0],
high=[input_image.width, input_image.height],
size=(5, 2),
))
result = self.descriptor.describe(input_image, input_keypoints)
self.assertEqual(len(input_keypoints), result.shape[0])
def test_filter_by_mask(self) -> None:
"""Test the `filter_by_mask` method."""
# Create a (9, 9) mask with ones in a (5, 5) square in the center of the mask and zeros everywhere else.
mask = np.zeros((9, 9)).astype(np.uint8)
mask[2:7, 2:7] = 1
# Test coordinates near corners of square of ones and along the diagonal.
coordinates = np.array([
[1.4, 1.4],
[1.4, 6.4],
[6.4, 1.4],
[6.4, 6.4],
[5.0, 5.0],
[0.0, 0.0],
[8.0, 8.0],
])
input_keypoints = Keypoints(coordinates=coordinates)
expected_keypoints = Keypoints(coordinates=coordinates[[3, 4]])
# Create keypoints from coordinates and dummy descriptors.
filtered_keypoints, _ = input_keypoints.filter_by_mask(mask)
assert len(filtered_keypoints) == 2
self.assertEqual(filtered_keypoints, expected_keypoints)
def test_mesh_inlier_correspondences(self) -> None:
"""Tests `compute_keypoint_intersections()` function.
We arrange four cameras in the x-z plane around a cube centered at the origin with side length 1. These cameras
are placed at (2, 0, 0), (-2, 0, 0), (0, 0, 2) and (0, 0, -2). We project a single 3d point located at the
origin into each camera. Since the cube has unit length on each dimension, we expect a keypoint located at the
center of each image to be found at the boundary of the cube -- 0.5 meters from the origin for each side on the
z-x plane.
"""
# Create cube mesh with side length one centered at origin.
box = trimesh.primitives.Box()
# Create arrangement of two cameras pointing at the center of one of the cube's faces.
fx, k1, k2, u0, v0 = 10, 0, 0, 1, 1
calibration = Cal3Bundler(fx, k1, k2, u0, v0)
cam_pos = [[2, 1, 0], [2, -1, 0]]
target_pos = [0.5, 0, 0]
up_vector = [0, -1, 0]
cam_i1 = PinholeCameraCal3Bundler().Lookat(cam_pos[0], target_pos,
up_vector, calibration)
cam_i2 = PinholeCameraCal3Bundler().Lookat(cam_pos[1], target_pos,
up_vector, calibration)
keypoints_i1 = Keypoints(
coordinates=np.array([[1, 1]]).astype(np.float32))
keypoints_i2 = Keypoints(
coordinates=np.array([[1, 1]]).astype(np.float32))
# Project keypoint at center of each simulated image and record intersection.
is_inlier, reproj_err = metric_utils.mesh_inlier_correspondences(
keypoints_i1,
keypoints_i2,
cam_i1,
cam_i2,
box,
dist_threshold=0.1)
assert np.count_nonzero(is_inlier) == 1
assert reproj_err[0] < 1e-4
def detect_and_describe(self, image: Image) -> Tuple[Keypoints, np.ndarray]:
"""Extract keypoints and their corresponding descriptors.
Adapted from:
https://github.com/mihaidusmanu/d2-net/blob/master/extract_features.py
Args:
image: the input image.
Returns:
Detected keypoints, with length N <= max_keypoints.
Corr. descriptors, of shape (N, D) where D is the dimension of each descriptor.
"""
model = D2Net(model_file=self.model_path, use_relu=USE_RELU, use_cuda=self.use_cuda)
model.eval()
# Resize image, and obtain re-scaling factors to postprocess keypoint coordinates.
resized_image, fact_i, fact_j = resize_image(image.value_array)
input_image = preprocess_image(resized_image, preprocessing=PREPROCESSING_METHOD)
scales = PYRAMID_SCALES if USE_MULTISCALE else [1]
with torch.no_grad():
keypoints, scores, descriptors = d2net_pyramid.process_multiscale(
image=torch.tensor(input_image[np.newaxis, :, :, :].astype(np.float32), device=self.device),
model=model,
scales=scales,
)
# Choose the top K keypoints and descriptors.
ordered_idxs = np.argsort(-scores)[: self.max_keypoints]
keypoints = keypoints[ordered_idxs, :]
descriptors = descriptors[ordered_idxs, :]
scores = scores[ordered_idxs]
# Rescale keypoint coordinates from resized image scale, back to provided input image resolution.
keypoints[:, 0] *= fact_i
keypoints[:, 1] *= fact_j
# Convert (y,x) tuples that represented (i, j) indices of image matrix, into (u, v) coordinates.
keypoints = keypoints[:, [1, 0]]
return Keypoints(coordinates=keypoints, responses=scores), descriptors
def generate_noisy_2d_measurements(
world_point: Point3,
calibrations: List[Cal3Bundler],
per_image_noise_vecs: np.ndarray,
poses: Pose3Vector,
) -> Tuple[List[Keypoints], List[Tuple[int, int]], Dict[
int, PinholeCameraCal3Bundler], ]:
"""
Generate PinholeCameras from specified poses and calibrations, and then generate
1 measurement per camera of a given 3d point.
Args:
world_point: 3d coords of 3d landmark in world frame
calibrations: List of calibrations for each camera
noise_params: List of amounts of noise to be added to each measurement
poses: List of poses for each camera in world frame
Returns:
keypoints_list: List of keypoints in all images (projected measurements in all images)
img_idxs: Tuple of indices for all images
cameras: Dictionary mapping image index i to calibrated PinholeCamera object
"""
keypoints_list = []
measurements = Point2Vector()
cameras = dict()
for i in range(len(poses)):
camera = PinholeCameraCal3Bundler(poses[i], calibrations[i])
# Project landmark into two cameras and triangulate
z = camera.project(world_point)
cameras[i] = camera
measurement = z + per_image_noise_vecs[i]
measurements.append(measurement)
keypoints_list += [Keypoints(coordinates=measurement.reshape(1, 2))]
# Create image indices for each pose - only subsequent pairwise matches
# assumed, e.g. between images (0,1) and images (1,2)
img_idxs = []
for i in range(len(poses) - 1):
img_idxs += [(i, i + 1)]
return keypoints_list, img_idxs, cameras
def detect_and_describe(self,
image: Image) -> Tuple[Keypoints, np.ndarray]:
"""Perform feature detection as well as their description.
Refer to detect() in DetectorBase and describe() in DescriptorBase for details about the output format.
Args:
image: the input image.
Returns:
Detected keypoints, with length N <= max_keypoints.
Corr. descriptors, of shape (N, D) where D is the dimension of each descriptor.
"""
# conert to grayscale
gray_image = image_utils.rgb_to_gray_cv(image)
# Creating OpenCV object
opencv_obj = cv.SIFT_create()
# Run the opencv code
cv_keypoints, descriptors = opencv_obj.detectAndCompute(
gray_image.value_array, None)
# convert to GTSFM's keypoints
keypoints = feature_utils.cast_to_gtsfm_keypoints(cv_keypoints)
# sort the features and descriptors by the score
# (need to sort here as we need the sorting order for descriptors)
sort_idx = np.argsort(-keypoints.responses)[:self.max_keypoints]
keypoints = Keypoints(
coordinates=keypoints.coordinates[sort_idx],
scales=keypoints.scales[sort_idx],
responses=keypoints.responses[sort_idx],
)
descriptors = descriptors[sort_idx]
return keypoints, descriptors
def test_data_association_with_missing_camera(self):
"""Tests the data association with input tracks which use a camera index for which the camera doesn't exist."""
triangulation_options = TriangulationOptions(
reproj_error_threshold=5,
mode=TriangulationSamplingMode.NO_RANSAC,
min_num_hypotheses=20)
da = DataAssociation(min_track_len=3,
triangulation_options=triangulation_options)
# add cameras 0 and 2
cameras = {
0:
PinholeCameraCal3Bundler(
Pose3(Rot3.RzRyRx(0, np.deg2rad(20), 0), np.zeros((3, 1)))),
2:
PinholeCameraCal3Bundler(
Pose3(Rot3.RzRyRx(0, 0, 0), np.array([10, 0, 0]))),
}
# just have one track, chaining cams 0->1 , and cams 1->2
corr_idxs_dict = {
(0, 1): np.array([[0, 0]], dtype=np.int32),
(1, 2): np.array([[0, 0]], dtype=np.int32)
}
keypoints_shared = Keypoints(coordinates=np.array([[20.0, 10.0]]))
# will lead to a cheirality exception because keypoints are identical in two cameras
# no track will be formed, and thus connected component will be empty
sfm_data, _ = da.run(
num_images=3,
cameras=cameras,
corr_idxs_dict=corr_idxs_dict,
keypoints_list=[keypoints_shared] * 3,
cameras_gt=[None] * 3,
relative_pose_priors={},
)
self.assertEqual(len(sfm_data.get_valid_camera_indices()), 0)
self.assertEqual(sfm_data.number_tracks(), 0)
