def verify_pydegensac(kps1, kps2, tentatives, th=4.0, n_iter=2000):
src_pts = np.float32([kps1[m.queryIdx].pt
for m in tentatives]).reshape(-1, 2)
dst_pts = np.float32([kps2[m.trainIdx].pt
for m in tentatives]).reshape(-1, 2)
H, mask = pydegensac.findHomography(src_pts, dst_pts, th, 0.99, n_iter)
print('pydegensac found {} inliers'.format(
int(deepcopy(mask).astype(np.float32).sum())))
return H, mask
def test_find_homography(self):
src_pts = np.float32([[0, 0], [0, 1], [1, 1], [1, 0]]).reshape(-1, 2)
dst_pts = np.float32([[0, 0], [0, -1], [-1, -1], [-1,
0]]).reshape(-1, 2)
H, mask = pydegensac.findHomography(src_pts, dst_pts, 4, 1)
self.assertEqual(3, len(H))
self.assertEqual(4, len(mask))
def siftMatching(img1, img2):
# img1 = np.array(cv2.cvtColor(np.array(img1), cv2.COLOR_BGR2RGB))
# img2 = np.array(cv2.cvtColor(np.array(img2), cv2.COLOR_BGR2RGB))
surf = cv2.xfeatures2d.SURF_create(100)
# surf = cv2.xfeatures2d.SIFT_create()
kp1, des1 = surf.detectAndCompute(img1, None)
kp2, des2 = surf.detectAndCompute(img2, None)
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1, des2, k=2)
good = []
for m, n in matches:
if m.distance < 0.7 * n.distance:
good.append(m)
src_pts = np.float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 2)
dst_pts = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 2)
if (src_pts.shape[0] < 5):
return None
# model, inliers = ransac(
# (src_pts, dst_pts),
# AffineTransform, min_samples=4,
# residual_threshold=8, max_trials=10000
# )
H, inliers = pydegensac.findHomography(src_pts, dst_pts, 8.0, 0.99, 10000)
n_inliers = np.sum(inliers)
inlier_keypoints_left = [
cv2.KeyPoint(point[0], point[1], 1) for point in src_pts[inliers]
]
inlier_keypoints_right = [
cv2.KeyPoint(point[0], point[1], 1) for point in dst_pts[inliers]
]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
image3 = cv2.drawMatches(img1, inlier_keypoints_left, img2,
inlier_keypoints_right, placeholder_matches, None)
# cv2.imshow('Matches', image3)
# cv2.waitKey(0)
src_pts = np.float32([
inlier_keypoints_left[m.queryIdx].pt for m in placeholder_matches
]).reshape(-1, 2)
dst_pts = np.float32([
inlier_keypoints_right[m.trainIdx].pt for m in placeholder_matches
]).reshape(-1, 2)
return image3
def cv2D2netMatching(image1, image2, feat1, feat2, matcher="BF"):
if(matcher == "BF"):
t0 = time.time()
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=True)
matches = bf.match(feat1['descriptors'], feat2['descriptors'])
matches = sorted(matches, key=lambda x:x.distance)
t1 = time.time()
print("Time to extract matches: ", t1-t0)
print("Number of raw matches:", len(matches))
match1 = [m.queryIdx for m in matches]
match2 = [m.trainIdx for m in matches]
keypoints_left = feat1['keypoints'][match1, : 2]
keypoints_right = feat2['keypoints'][match2, : 2]
np.random.seed(0)
t0 = time.time()
### Ransac ###
# model, inliers = ransac(
# (keypoints_left, keypoints_right),
# AffineTransform, min_samples=4,
# residual_threshold=8, max_trials=10000
# )
####
H, inliers = pydegensac.findHomography(keypoints_left, keypoints_right, 8.0, 0.99, 10000)
t1 = time.time()
print("Time for ransac: ", t1-t0)
n_inliers = np.sum(inliers)
print('Number of inliers: %d.' % n_inliers)
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_left[inliers]]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_right[inliers]]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
image3 = cv2.drawMatches(image1, inlier_keypoints_left, image2, inlier_keypoints_right, placeholder_matches, None)
#### Visualization ####
# plt.figure(figsize=(20, 20))
# plt.imshow(image3)
# plt.axis('off')
# plt.show()
src_pts = np.float32([ inlier_keypoints_left[m.queryIdx].pt for m in placeholder_matches ]).reshape(-1, 2)
dst_pts = np.float32([ inlier_keypoints_right[m.trainIdx].pt for m in placeholder_matches ]).reshape(-1, 2)
return src_pts, dst_pts
def getPerspKeypoints(rgbFile1, rgbFile2, HFile1, HFile2, model, device):
if HFile1 is None:
igp1, img1 = read_and_process_image(rgbFile1, H=None)
else:
H1 = np.load(HFile1)
igp1, img1 = read_and_process_image(rgbFile1, H=H1)
c,h,w = igp1.shape
if HFile2 is None:
igp2, img2 = read_and_process_image(rgbFile2, H=None)
else:
H2 = np.load(HFile2)
igp2, img2 = read_and_process_image(rgbFile2, H=H2)
feat1 = extractSingle(igp1, model, device)
feat2 = extractSingle(igp2, model, device)
matches = mnn_matcher(
torch.from_numpy(feat1['descriptors']).to(device=device),
torch.from_numpy(feat2['descriptors']).to(device=device),
)
pos_a = feat1["keypoints"][matches[:, 0], : 2]
pos_b = feat2["keypoints"][matches[:, 1], : 2]
H, inliers = pydegensac.findHomography(pos_a, pos_b, 8.0, 0.99, 10000)
pos_a = pos_a[inliers]
pos_b = pos_b[inliers]
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_a]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_b]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(len(pos_a))]
image3 = cv2.drawMatches(img1, inlier_keypoints_left, img2, inlier_keypoints_right, placeholder_matches, None, matchColor=[0, 255, 0])
image3 = cv2.cvtColor(image3, cv2.COLOR_BGR2RGB)
#### Visualization ####
# cv2.imshow('Matches', image3)
# cv2.waitKey()
orgSrc, orgDst = orgKeypoints(pos_a, pos_b, H1, H2)
drawOrg(cv2.imread(rgbFile1), cv2.imread(rgbFile2), orgSrc, orgDst) # Reproject matches to perspective View
return orgSrc, orgDst
def rordMatching(image1, image2, feat1, feat2, matcher="BF"):
if(matcher == "BF"):
t0 = time.time()
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=True)
matches = bf.match(feat1['descriptors'], feat2['descriptors'])
matches = sorted(matches, key=lambda x:x.distance)
t1 = time.time()
print("Time to extract matches: ", t1-t0)
print("Number of raw matches:", len(matches))
match1 = [m.queryIdx for m in matches]
match2 = [m.trainIdx for m in matches]
keypoints_left = feat1['keypoints'][match1, : 2]
keypoints_right = feat2['keypoints'][match2, : 2]
np.random.seed(0)
t0 = time.time()
H, inliers = pydegensac.findHomography(keypoints_left, keypoints_right, 10.0, 0.99, 10000)
t1 = time.time()
print("Time for ransac: ", t1-t0)
n_inliers = np.sum(inliers)
print('Number of inliers: %d.' % n_inliers)
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_left[inliers]]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_right[inliers]]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
draw_params = dict(matchColor = (0,255,0),
singlePointColor = (255,0,0),
# matchesMask = matchesMask,
flags = 0)
image3 = cv2.drawMatches(image1, inlier_keypoints_left, image2, inlier_keypoints_right, placeholder_matches, None, **draw_params)
plt.figure(figsize=(20, 20))
plt.imshow(image3)
plt.axis('off')
plt.show()
def get_single_result(ms, m, method, params):
mask = (ms <= params['match_th']).reshape(-1)
tentatives = m[mask]
tentative_idxs = np.arange(len(mask))[mask]
src_pts = tentatives[:, :2]
dst_pts = tentatives[:, 2:]
if tentatives.shape[0] <= 5:
return np.eye(3), np.array([False] * len(mask))
if method == 'cv2h':
H, mask_inl = cv2.findHomography(src_pts,
dst_pts,
cv2.RANSAC,
params['inl_th'],
maxIters=params['maxiter'],
confidence=params['conf'])
elif method == 'pyransac':
H, mask_inl = pydegensac.findHomography(src_pts,
dst_pts,
params['inl_th'],
conf=params['conf'],
max_iters=params['maxiter'])
elif method == 'sklearn':
try:
#print(src_pts.shape, dst_pts.shape)
H, mask_inl = skransac([src_pts, dst_pts],
ProjectiveTransform,
min_samples=4,
residual_threshold=params['inl_th'],
max_trials=params['maxiter'],
stop_probability=params['conf'])
mask_inl = mask_inl.astype(bool).flatten()
H = H.params
except Exception as e:
print("Fail!", e)
return np.eye(3), np.array([False] * len(mask))
else:
raise ValueError('Unknown method')
final_inliers = np.array([False] * len(mask))
if H is not None:
for i, x in enumerate(mask_inl):
final_inliers[tentative_idxs[i]] = x
return H, final_inliers
def get_num_inliers(test_keypoints, test_descriptors, train_keypoints,
train_descriptors):
"""Returns the number of RANSAC inliers."""
test_match_kp, train_match_kp = get_putative_matching_keypoints(
test_keypoints, test_descriptors, train_keypoints, train_descriptors)
if test_match_kp.shape[0] <= 4:
return 0
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
except np.linalg.LinAlgError:
return 0
return int(copy.deepcopy(mask).astype(np.float32).sum())
def numInliers2(feat1, feat2): bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=True) matches = bf.match(feat1['descriptors'], feat2['descriptors']) matches = sorted(matches, key=lambda x:x.distance) match1 = [m.queryIdx for m in matches] match2 = [m.trainIdx for m in matches] keypoints_left = feat1['keypoints'][match1, : 2] keypoints_right = feat2['keypoints'][match2, : 2] np.random.seed(0) # model, inliers = ransac( # (keypoints_left, keypoints_right), # AffineTransform, min_samples=4, # residual_threshold=8, max_trials=10000 # ) H, inliers = pydegensac.findHomography(keypoints_left, keypoints_right, 8.0, 0.99, 10000) n_inliers = np.sum(inliers) return n_inliers
def getPerspKeypoints2(model1, model2, rgbFile1, rgbFile2, HFile1, HFile2, device):
if HFile1 is None:
igp1, img1 = read_and_process_image(rgbFile1, H=None)
else:
H1 = np.load(HFile1)
igp1, img1 = read_and_process_image(rgbFile1, H=H1)
c,h,w = igp1.shape
if HFile2 is None:
igp2, img2 = read_and_process_image(rgbFile2, H=None)
else:
H2 = np.load(HFile2)
igp2, img2 = read_and_process_image(rgbFile2, H=H2)
with torch.no_grad():
keypoints_a1, scores_a1, descriptors_a1 = process_multiscale(
igp1.to(device).unsqueeze(0),
model1,
scales=[1]
)
keypoints_a1 = keypoints_a1[:, [1, 0, 2]]
keypoints_a2, scores_a2, descriptors_a2 = process_multiscale(
igp1.to(device).unsqueeze(0),
model2,
scales=[1]
)
keypoints_a2 = keypoints_a2[:, [1, 0, 2]]
keypoints_b1, scores_b1, descriptors_b1 = process_multiscale(
igp2.to(device).unsqueeze(0),
model1,
scales=[1]
)
keypoints_b1 = keypoints_b1[:, [1, 0, 2]]
keypoints_b2, scores_b2, descriptors_b2 = process_multiscale(
igp2.to(device).unsqueeze(0),
model2,
scales=[1]
)
keypoints_b2 = keypoints_b2[:, [1, 0, 2]]
# calculating matches for both models
matches1, dist_1 = mnn_matcher_scorer(
torch.from_numpy(descriptors_a1).to(device=device),
torch.from_numpy(descriptors_b1).to(device=device),
# len(matches1)
)
matches2, dist_2 = mnn_matcher_scorer(
torch.from_numpy(descriptors_a2).to(device=device),
torch.from_numpy(descriptors_b2).to(device=device),
# len(matches1)
)
full_matches = torch.cat([matches1, matches2])
full_dist = torch.cat([dist_1, dist_2])
assert len(full_dist)==(len(dist_1)+len(dist_2)), "something wrong"
k_final = len(full_dist)//2
# k_final = len(full_dist)
# k_final = max(len(dist_1), len(dist_2))
top_k_mask = torch.topk(full_dist, k=k_final)[1]
first = []
second = []
for valid_id in top_k_mask:
if valid_id<len(dist_1):
first.append(valid_id)
else:
second.append(valid_id-len(dist_1))
# final_matches = full_matches[top_k_mask]
matches1 = matches1[torch.tensor(first, device=device).long()].data.cpu().numpy()
matches2 = matches2[torch.tensor(second, device=device).long()].data.cpu().numpy()
pos_a1 = keypoints_a1[matches1[:, 0], : 2]
pos_b1 = keypoints_b1[matches1[:, 1], : 2]
pos_a2 = keypoints_a2[matches2[:, 0], : 2]
pos_b2 = keypoints_b2[matches2[:, 1], : 2]
pos_a = np.concatenate([pos_a1, pos_a2], 0)
pos_b = np.concatenate([pos_b1, pos_b2], 0)
# pos_a, pos_b, inliers = apply_ransac(pos_a, pos_b)
H, inliers = pydegensac.findHomography(pos_a, pos_b, 8.0, 0.99, 10000)
pos_a = pos_a[inliers]
pos_b = pos_b[inliers]
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_a]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in pos_b]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(len(pos_a))]
image3 = cv2.drawMatches(img1, inlier_keypoints_left, img2, inlier_keypoints_right, placeholder_matches, None, matchColor=[0, 255, 0])
image3 = cv2.cvtColor(image3, cv2.COLOR_BGR2RGB)
# cv2.imshow('Matches', image3)
# cv2.waitKey()
orgSrc, orgDst = orgKeypoints(pos_a, pos_b, H1, H2)
drawOrg(cv2.imread(rgbFile1), cv2.imread(rgbFile2), orgSrc, orgDst)
return orgSrc, orgDst
def siftMatching(img1, img2, HFile1, HFile2, device):
H1 = np.load(HFile1)
H2 = np.load(HFile2)
img1 = Image.open(img1)
rgbFile1 = img1
if(img1.mode != 'RGB'):
img1 = img1.convert('RGB')
img1 = np.array(img1)
img1 = cv2.warpPerspective(img1, H1, dsize=(400,400))
#### Visualization ####
# cv2.imshow("Image", cv2.cvtColor(img1, cv2.COLOR_BGR2RGB))
# cv2.waitKey(0)
img2 = Image.open(img2)
rgbFile2 = img2
if(img2.mode != 'RGB'):
img2 = img2.convert('RGB')
img2 = np.array(img2)
img2 = cv2.warpPerspective(img2, H2, dsize=(400,400))
#### Visualization ####
# cv2.imshow("Image", cv2.cvtColor(img2, cv2.COLOR_BGR2RGB))
# cv2.waitKey(0)
# surf = cv2.xfeatures2d.SURF_create(100) # SURF
surf = cv2.xfeatures2d.SIFT_create()
kp1, des1 = surf.detectAndCompute(img1, None)
kp2, des2 = surf.detectAndCompute(img2, None)
matches = mnn_matcher(
torch.from_numpy(des1).float().to(device=device),
torch.from_numpy(des2).float().to(device=device)
)
src_pts = np.float32([ kp1[m[0]].pt for m in matches ]).reshape(-1, 2)
dst_pts = np.float32([ kp2[m[1]].pt for m in matches ]).reshape(-1, 2)
if(src_pts.shape[0] < 5 or dst_pts.shape[0] < 5):
return [], []
H, inliers = pydegensac.findHomography(src_pts, dst_pts, 8.0, 0.99, 10000)
n_inliers = np.sum(inliers)
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in src_pts[inliers]]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in dst_pts[inliers]]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
#### Visualization ####
# image3 = cv2.drawMatches(img1, inlier_keypoints_left, img2, inlier_keypoints_right, placeholder_matches, None)
# cv2.imshow('Matches', image3)
# cv2.waitKey()
src_pts = np.float32([ inlier_keypoints_left[m.queryIdx].pt for m in placeholder_matches ]).reshape(-1, 2)
dst_pts = np.float32([ inlier_keypoints_right[m.trainIdx].pt for m in placeholder_matches ]).reshape(-1, 2)
orgSrc, orgDst = orgKeypoints(src_pts, dst_pts, H1, H2)
return orgSrc, orgDst
def numInliers(frontFeat1, rearFeat1, frontFeat2, rearFeat2):
keypoints_a1 = frontFeat1['keypoints']
descriptors_a1 = frontFeat1['descriptors']
keypoints_a2 = frontFeat2['keypoints']
descriptors_a2 = frontFeat2['descriptors']
keypoints_b1 = rearFeat1['keypoints']
descriptors_b1 = rearFeat1['descriptors']
keypoints_b2 = rearFeat2['keypoints']
descriptors_b2 = rearFeat2['descriptors']
keypoints_a1 = keypoints_a1[:, [1, 0, 2]]
keypoints_a2 = keypoints_a2[:, [1, 0, 2]]
keypoints_b1 = keypoints_b1[:, [1, 0, 2]]
keypoints_b2 = keypoints_b2[:, [1, 0, 2]]
use_cuda = torch.cuda.is_available()
device = torch.device('cuda:1' if use_cuda else 'cpu')
# calculating matches for both models
matches1, dist_1 = mnn_matcher_scorer(
torch.from_numpy(descriptors_a1).to(device=device),
torch.from_numpy(descriptors_b1).to(device=device),
# len(matches1)
)
matches2, dist_2 = mnn_matcher_scorer(
torch.from_numpy(descriptors_a2).to(device=device),
torch.from_numpy(descriptors_b2).to(device=device),
# len(matches1)
)
full_matches = torch.cat([matches1, matches2])
full_dist = torch.cat([dist_1, dist_2])
assert len(full_dist) == (len(dist_1) + len(dist_2)), "something wrong"
k_final = len(full_dist) // 2
# k_final = len(full_dist)
# k_final = max(len(dist_1), len(dist_2))
top_k_mask = torch.topk(full_dist, k=k_final)[1]
first = []
second = []
for valid_id in top_k_mask:
if valid_id < len(dist_1):
first.append(valid_id)
else:
second.append(valid_id - len(dist_1))
matches1 = matches1[torch.tensor(first,
device=device).long()].data.cpu().numpy()
matches2 = matches2[torch.tensor(second,
device=device).long()].data.cpu().numpy()
pos_a1 = keypoints_a1[matches1[:, 0], :2]
pos_b1 = keypoints_b1[matches1[:, 1], :2]
pos_a2 = keypoints_a2[matches2[:, 0], :2]
pos_b2 = keypoints_b2[matches2[:, 1], :2]
pos_a = np.concatenate([pos_a1, pos_a2], 0)
pos_b = np.concatenate([pos_b1, pos_b2], 0)
H, inliers = pydegensac.findHomography(pos_a, pos_b, 8.0, 0.99, 10000)
n_inliers = np.sum(inliers)
return n_inliers
def cv2D2netMatching(image1, image2, feat1, feat2, matcher="BF"):
if (matcher == "BF"):
t0 = time.time()
bf = cv2.BFMatcher(cv2.NORM_L2, crossCheck=True)
matches = bf.match(feat1['descriptors'], feat2['descriptors'])
matches = sorted(matches, key=lambda x: x.distance)
t1 = time.time()
print("Time to extract matches: ", t1 - t0)
print("Number of raw matches:", len(matches))
match1 = [m.queryIdx for m in matches]
match2 = [m.trainIdx for m in matches]
keypoints_left = feat1['keypoints'][match1, :2]
keypoints_right = feat2['keypoints'][match2, :2]
np.random.seed(0)
t0 = time.time()
H, inliers = pydegensac.findHomography(keypoints_left, keypoints_right,
10.0, 0.99, 10000)
t1 = time.time()
print("Time for ransac: ", t1 - t0)
n_inliers = np.sum(inliers)
print('Number of inliers: %d.' % n_inliers)
inlier_keypoints_left = [
cv2.KeyPoint(point[0], point[1], 1)
for point in keypoints_left[inliers]
]
inlier_keypoints_right = [
cv2.KeyPoint(point[0], point[1], 1)
for point in keypoints_right[inliers]
]
placeholder_matches = [
cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)
]
draw_params = dict(
matchColor=(0, 255, 0),
singlePointColor=(255, 0, 0),
# matchesMask = matchesMask,
flags=0)
image3 = cv2.drawMatches(image1, inlier_keypoints_left, image2,
inlier_keypoints_right, placeholder_matches,
None, **draw_params)
plt.figure(figsize=(20, 20))
plt.imshow(image3)
plt.axis('off')
plt.show()
elif (matcher == "FLANN"):
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)
t0 = time.time()
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(feat1['descriptors'],
feat2['descriptors'],
k=2)
t1 = time.time()
print("Time to extract matches: ", t1 - t0)
print("Number of raw matches:", len(matches))
t0 = time.time()
good = []
for m, n in matches:
if m.distance < 0.9 * n.distance:
good.append(m)
matches = good
t1 = time.time()
print("Time for outlier rejection: ", t1 - t0)
print("Number of inliers: ", len(matches))
match1 = [m.queryIdx for m in matches]
match2 = [m.trainIdx for m in matches]
keypoints_left = feat1['keypoints'][match1, :2].T
keypoints_right = feat2['keypoints'][match2, :2].T
for i in range(keypoints_left.shape[1]):
image1 = cv2.circle(
image1, (int(keypoints_left[0, i]), int(keypoints_left[1, i])),
2, (0, 0, 255), 4)
for i in range(keypoints_right.shape[1]):
image2 = cv2.circle(
image2,
(int(keypoints_right[0, i]), int(keypoints_right[1, i])), 2,
(0, 0, 255), 4)
im4 = cv2.hconcat([image1, image2])
for i in range(keypoints_left.shape[1]):
im4 = cv2.line(
im4, (int(keypoints_left[0, i]), int(keypoints_left[1, i])),
(int(keypoints_right[0, i]) + image1.shape[1],
int(keypoints_right[1, i])), (0, 255, 0), 1)
cv2.imshow("Image_lines", im4)
cv2.waitKey(0)
def eval_hpatches(
matcher,
data_root,
method='',
task='both',
scale_H=False,
h_solver='degensac',
ransac_thres=2,
thres=[1, 3, 5, 10],
lprint_=print,
print_out=False,
save_npy=None,
debug=False,
):
"""Evaluate a matcher on HPatches sequences for image matching and homogray estimation.
The matching metric is adopted from D2Net paper, i.e., the precentage of correctly matched
keypoints at the given re-projection error thresholds.
For homography estimation, the average distances between the corners transformed using
the estimated and GT homographies are computed. Both percentage of the corner distance at
the given thresholds and the area under the cumulative error curve (AUC) at those thresholds
are reported.
Args:
- matcher: the matching function that inputs an image pair paths and
outputs the matches and keypoints.
- data_root: the folder directory of HPatches dataset.
- method: the description of the evaluated method.
- task: the target task, options = [matching|homography|both]
- ransac_thres: the set of ransac thresholds used by the solver to estimate homographies.
Results under each ransac threshold are printed per line.
- thres: error thresholds in pixels to compute the metrics.
- lprint: the printing function. If needed it can be implemented to outstream to a log file.
- print_out: when set to True, per-pair results are printed during the evaluation.
"""
np.set_printoptions(precision=2)
from PIL import Image
if task == 'both':
task = 'matching+homography'
seq_dirs = sorted(glob.glob('{}/*'.format(data_root)))
lprint_(f'\n>>>>Eval hpatches: task={task} method={method} scale_H={scale_H} rthres={ransac_thres} thres={thres} ')
# Matching
if 'matching' in task:
thres_range = np.arange(1, 16)
i_err = {thr: 0 for thr in thres_range}
v_err = {thr: 0 for thr in thres_range}
n_feats = []
seq_type = []
# Homography
if 'homography' in task:
inlier_ratio = []
h_failed = 0
dists_sa = []
dists_si = []
dists_sv = []
match_failed = 0
n_matches = []
match_time = []
start_time = time.time()
for seq_idx, seq_dir in tqdm(enumerate(seq_dirs[::-1]), total=len(seq_dirs), smoothing=.5):
if debug and seq_idx > 10:
break
sname = seq_dir.split('/')[-1]
im1_path = os.path.join(seq_dir, '1.ppm')
# Eval on composed pairs within seq
for im_idx in range(2, 7):
im2_path = os.path.join(seq_dir, '{}.ppm'.format(im_idx))
H_gt = np.loadtxt(os.path.join(seq_dir, 'H_1_{}'.format(im_idx)))
scale = np.ones(4)
# Predict matches
try:
t0 = time.time()
match_res = matcher(im1_path, im2_path)
match_time.append(time.time() - t0)
matches, p1s, p2s = match_res[0:3]
if scale_H:
# scale = (wo / wt, ho / ht) for im1 & im2
scale = match_res[4]
# Scale gt homoragphies
H_scale_im1 = scale_homography(scale[0], scale[1])
H_scale_im2 = scale_homography(scale[2], scale[3])
H_gt = np.linalg.inv(H_scale_im2) @ H_gt @ H_scale_im1
except:
p1s = p2s = matches = []
match_failed += 1
n_matches.append(len(matches))
if 'matching' in task:
n_feats.append(len(p1s))
n_feats.append(len(p2s))
seq_type.append(sname[0])
if len(matches) == 0:
dist = np.array([float("inf")])
else:
dist = cal_reproj_dists(matches[:, :2], matches[:, 2:], H_gt)
for thr in thres_range:
if sname[0] == 'i':
i_err[thr] += np.mean(dist <= thr)
else:
v_err[thr] += np.mean(dist <= thr)
if 'homography' in task:
try:
if 'cv' in h_solver:
H_pred, inliers = cv2.findHomography(matches[:, :2], matches[:, 2:4], cv2.RANSAC, ransac_thres)
else:
H_pred, inliers = pydegensac.findHomography(matches[:, :2], matches[:, 2:4], ransac_thres)
except:
H_pred = None
if H_pred is None:
corner_dist = np.nan
irat = 0
h_failed += 1
inliers = []
else:
im = Image.open(im1_path)
w, h = im.size
w, h = w / scale[0], h / scale[1]
corners = np.array([[0, 0, 1],
[0, h - 1, 1],
[w - 1, 0, 1],
[w - 1, h - 1, 1]])
real_warped_corners = np.dot(corners, np.transpose(H_gt))
real_warped_corners = real_warped_corners[:, :2] / real_warped_corners[:, 2:]
warped_corners = np.dot(corners, np.transpose(H_pred))
warped_corners = warped_corners[:, :2] / warped_corners[:, 2:]
corner_dist = np.mean(np.linalg.norm(real_warped_corners - warped_corners, axis=1))
irat = np.mean(inliers)
inlier_ratio.append(irat)
dists_sa.append(corner_dist)
if sname[0] == 'i':
dists_si.append(corner_dist)
if sname[0] == 'v':
dists_sv.append(corner_dist)
if print_out:
print(f'Scene {sname}, pair:1-{im_idx} matches:{len(matches)}')
if 'matching' in task:
print(f'Median matching dist:{np.median(dist):.2f} <1px:{np.mean(dist <= 1):.3f}')
if 'homography' in task:
print(f'Corner dist:{corner_dist:.2f} inliers:{np.sum(inliers)}')
lprint_(f'>>Finished, pairs={len(match_time)} match_failed={match_failed} matches={np.mean(n_matches):.1f} match_time={np.mean(match_time):.2f}s')
if 'matching' in task:
results = i_err, v_err, [np.array(seq_type), np.array(n_feats), np.array(n_matches)]
lprint_('==== Image Matching ====')
lprint_(eval_summary_matching(results, thres, save_npy=save_npy))
if 'homography' in task:
lprint_('==== Homography Estimation ====')
lprint_(f'Hest solver={h_solver} est_failed={h_failed} ransac_thres={ransac_thres} inlier_rate={np.mean(inlier_ratio):.2f}')
lprint_(eval_summary_homography(dists_sa, dists_si, dists_sv, thres))
def compute_num_inliers(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
use_ratio_test=False,
draw_matches=True,
query_im_array=None,
index_im_array=None):
"""Returns the number of RANSAC inliers."""
test_match_kp, train_match_kp = \
compute_putative_matching_keypoints(test_keypoints,
test_descriptors,
train_keypoints,
train_descriptors,
use_ratio_test=use_ratio_test)
if test_match_kp.shape[
0] <= 4: # Min keypoints supported by `pydegensac.findHomography()`
return 0, b''
try:
_, mask = pydegensac.findHomography(test_match_kp, train_match_kp,
MAX_REPROJECTION_ERROR,
HOMOGRAPHY_CONFIDENCE,
MAX_RANSAC_ITERATIONS)
except np.linalg.LinAlgError: # When det(H)=0, can't invert matrix.
return 0, b''
inliers = mask if mask is not None else []
match_viz_bytes = b''
if isinstance(query_im_array, np.ndarray) and isinstance(
index_im_array, np.ndarray) and draw_matches:
query_im_scale_factors = [1.0, 1.0]
index_im_scale_factors = [1.0, 1.0]
inlier_idxs = np.nonzero(inliers)[0]
_, ax = plt.subplots()
ax.axis('off')
ax.xaxis.set_major_locator(plt.NullLocator())
ax.yaxis.set_major_locator(plt.NullLocator())
plt.subplots_adjust(top=1,
bottom=0,
right=1,
left=0,
hspace=0,
wspace=0)
plt.margins(0, 0)
feature.plot_matches(ax,
query_im_array,
index_im_array,
test_match_kp * query_im_scale_factors,
train_match_kp * index_im_scale_factors,
np.column_stack((inlier_idxs, inlier_idxs)),
only_matches=False)
match_viz_io = io.BytesIO()
plt.savefig(match_viz_io,
format='jpeg',
bbox_inches='tight',
pad_inches=0)
match_viz_bytes = match_viz_io.getvalue()
return int(copy.deepcopy(mask).astype(np.float32).sum()), match_viz_bytes
