def evaluate_correspondence(img_A, img_B, ground_truth_correspondence_file,
scale_factor, x1_est, y1_est, x2_est, y2_est, matches, confidences, vis, filename="notre_dame_matches.jpg"):
# 'unscale' interest points to compare with ground truth points
x1_est_scaled = x1_est / scale_factor
y1_est_scaled = y1_est / scale_factor
x2_est_scaled = x2_est / scale_factor
y2_est_scaled = y2_est / scale_factor
conf_indices = np.argsort(-confidences, kind='mergesort')
matches = matches[conf_indices,:]
confidences = confidences[conf_indices]
# we want to see how good our matches are, extract the coordinates of each matched
# point
x1_matches = np.zeros(matches.shape[0])
y1_matches = np.zeros(matches.shape[0])
x2_matches = np.zeros(matches.shape[0])
y2_matches = np.zeros(matches.shape[0])
for i in range(matches.shape[0]):
x1_matches[i] = x1_est_scaled[int(matches[i, 0])]
y1_matches[i] = y1_est_scaled[int(matches[i, 0])]
x2_matches[i] = x2_est_scaled[int(matches[i, 1])]
y2_matches[i] = y2_est_scaled[int(matches[i, 1])]
good_matches = np.zeros((matches.shape[0]), dtype=np.bool)
# Loads `ground truth' positions x1, y1, x2, y2
file_contents = scio.loadmat(ground_truth_correspondence_file)
# x1, y1, x2, y2 = scio.loadmat(eval_file)
x1 = file_contents['x1']
y1 = file_contents['y1']
x2 = file_contents['x2']
y2 = file_contents['y2']
pointsA = np.zeros((len(x1), 2))
pointsB = np.zeros((len(x2), 2))
for i in range(len(x1)):
pointsA[i, 0] = x1[i]
pointsA[i, 1] = y1[i]
pointsB[i, 0] = x2[i]
pointsB[i, 1] = y2[i]
correct_matches = 0
F = estimate_fundamental_matrix(pointsA, pointsB)
top50 = 0
top100 = 0
for i in range(x1_matches.shape[0]):
pointA = np.ones((1, 3))
pointB = np.ones((1, 3))
pointA[0,0] = x1_matches[i]
pointA[0,1] = y1_matches[i]
pointB[0,0] = x2_matches[i]
pointB[0,1] = y2_matches[i]
if abs(pointB @ F @ np.transpose(pointA)) < .1:
x_dists = x1 - x1_matches[i]
y_dists = y1 - y1_matches[i]
# computes distances of each interest point to the ground truth point
dists = np.sqrt(np.power(x_dists, 2.0) + np.power(y_dists, 2.0))
closest_ground_truth = np.argmin(dists, axis=0)
offset_x1 = x1_matches[i] - x1[closest_ground_truth]
offset_y1 = y1_matches[i] - y1[closest_ground_truth]
offset_x1 *= img_B.shape[0] / img_A.shape[0]
offset_y1 *= img_B.shape[0] / img_A.shape[0]
offset_x2 = x2_matches[i] - x2[closest_ground_truth]
offset_y2 = y2_matches[i] - y2[closest_ground_truth]
offset_dist = np.sqrt(np.power(offset_x1 - offset_x2, 2) + np.power(offset_y1 - offset_y2, 2))
if offset_dist < 70:
correct_matches += 1
good_matches[i] = True
if i == 49:
print(f'Accuracy on 50 most confident: {int(100 * correct_matches / 50)}%')
top50 = correct_matches
if i == 99:
print(f'Accuracy on 100 most confident: {int(100 * correct_matches / 100)}%')
top100 = correct_matches
print(f'Accuracy on all matches: {int(100 * correct_matches / len(matches))}%')
if vis > 0:
print("Vizualizing...")
visualize.show_correspondences(img_A, img_B, x1_est / scale_factor, y1_est / scale_factor, x2_est / scale_factor, y2_est / scale_factor, matches, good_matches, vis, filename)
return top50, top100, correct_matches
def match_sift_feat(files, names, downscale_factor):
"""
Matches SIFT features extracted for every image pair bi-directionally. Ensures
all matches are unique.
Parameters
----------
files: list of strings
path to input images
names: list of strings
names of the input images
downscale_factor: float
Factor of downscaling (used to reduce computation)
Saves
-------
sift_match.npy, containing:
M: a matrix with each rowing storing the match pairs' indicies and confidence score
F: a matrix with each rowing the corresponding SIFT features
num_matches: int. Total number of matches
"""
print("-----matching SIFT features-----")
num_frames = len(files)
sift_total = np.load('../npy/sift_total.npy') #load previous result
num_features = sift_total[0]
is_vis_match = False #to see the matches or not
# M for matches, F for features
M = [[[0] for i in range(num_frames)] for j in range(num_frames)]
F = [[[0] for i in range(num_frames)] for j in range(num_frames)]
num_matches = 0;
#for every image pair
for i in range(0, num_frames):
for j in range(i+1, num_frames):
#load up feature points and descriptors.
si_f = np.load('../npy/'+names[i]+"_f.npy")
si_d = np.load('../npy/'+names[i]+"_d.npy")
sj_f = np.load('../npy/'+names[j]+"_f.npy")
sj_d = np.load('../npy/'+names[j]+"_d.npy")
############ start bi-directional matching #############
# create BFMatcher object
bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=True)
##### match - direction 1
cv2_matches1 = bf.match(si_d, sj_d)
matches1 = convert_matches(cv2_matches1) #convert matches from DMatches to arrays of [queryIdx, trainIdx]
scores1 = extract_dist_score(cv2_matches1)
##### match - direction 2
cv2_matches2 = bf.match(sj_d, si_d) #note the change in j and i
matches2 = convert_matches(cv2_matches2) #convert matches from DMatches to arrays of [queryIdx, trainIdx]
scores2 = extract_dist_score(cv2_matches2)
############ end bi-directional matching #############
######## find the intersection of two matches
_, mid1, mid2 = np.intersect1d(matches1[:, 0], matches2[:, 1], return_indices=True)
if(mid1.shape[0] == 0):
print('no match intersects')
continue
matches = matches1[mid1, :]
scores = (scores1[mid1] + scores2[mid2]) / 2
######## ensure unique matching pairs
mi, mj = matches[:, 0], matches[:, 1]
s = scores
n = mi.shape[0]
unique_mi, unique_mj = np.unique(mi), np.unique(mj)
num_mi, num_mj = mi.shape[0], mj.shape[0]
num_unique_mi, num_unique_mj = unique_mi.shape[0], unique_mj.shape[0]
if (num_mi == num_unique_mi) and (num_mj == num_unique_mj):
print("all matches are unique")
num_matches = num_matches + s.shape[0]
fi, fj = si_f[mi, :], sj_f[mj, :]
di, dj = si_d[mi, :], sj_d[mj, :]
M[i][j] = np.array([mi, mj, s])
F[i][j] = np.array([fi, fj])
else:
# if matches are not unique, resolve by creating a sparse matrix
print("dealing with non-unique matches")
S_sparse = scipy.sparse.csr_matrix((s, (mi, mj)), shape=(num_features[i].astype(int), num_features[j].astype(int)))
S = S_sparse.todense()
for p in range(0, num_features[i].astype(int)):
if (S_sparse[p, :].nnz > 1):
Ss = S_sparse[p, :]
idx = np.argmax(Ss)
val = Ss[idx]
S_sparse[p, :] = 0;
S_sparse[p, idx] = val
for p in range(0, num_features[j].astype(int)):
if (S_sparse[:, p].nnz > 1):
Ss = S_sparse[:, p]
idx = np.argmax(Ss)
val = Ss[idx]
S_sparse[:, p] = 0;
S_sparse[idx, p] = val
mi, mj = np.where(S != 0)
_,_,s_idx = scipy.sparse.find(S_sparse != 0)
s = s[s_idx]
num_matches = num_matches + s.shape[0]
fi, fj = si_f[mi, :], sj_f[mj, :]
di, dj = si_d[mi, :], sj_d[mj, :]
M[i][j] = np.array([mi, mj, s])
F[i][j] = np.array([fi, fj])
############ visualize matches #############
if is_vis_match:
print("visualizing matches")
img1 = color.rgb2gray(io.imread(files[i]))
img2 = color.rgb2gray(io.imread(files[j]))
img1 = rescale(img1, downscale_factor, anti_aliasing=True, multichannel=False, mode='reflect')
img2 = rescale(img2, downscale_factor, anti_aliasing=True, multichannel=False, mode='reflect')
matches[:, 0] = mi
matches[:, 1] = mj
matches = matches[np.random.permutation(matches.shape[0])][:50]
visualize.show_correspondences(img1, img2, si_f[:, 1], si_f[:, 0], sj_f[:, 1], sj_f[:, 0], matches, mode='arrows')
# saving results to file
M, F = np.array(M), np.array(F)
to_save = np.array([M, F, num_matches])
np.save('../npy/sift_match', to_save)
print("Completed matching SIFT features")
def evaluate_correspondence(img_A,
img_B,
ground_truth_correspondence_file,
scale_factor,
x1_est,
y1_est,
x2_est,
y2_est,
matches,
confidences,
vis,
filename="eval_corr.jpg"):
# Sort the matches by their confidences into descending order of confidence
# (high confidence at low array index)
conf_sorted = -np.sort(-confidences, kind='mergesort')
conf_indices = np.argsort(-confidences, kind='mergesort')
matches = matches[conf_indices, :]
confidences = conf_sorted
# 'unscale' interest points to compare with ground truth points
x1_est_scaled = x1_est / scale_factor
y1_est_scaled = y1_est / scale_factor
x2_est_scaled = x2_est / scale_factor
y2_est_scaled = y2_est / scale_factor
# We want to see how good our matches are;
# extract the coordinates of each matched point
x1_matches = np.zeros(matches.shape[0])
y1_matches = np.zeros(matches.shape[0])
x2_matches = np.zeros(matches.shape[0])
y2_matches = np.zeros(matches.shape[0])
for i in range(matches.shape[0]):
x1_matches[i] = x1_est_scaled[int(matches[i, 0])]
y1_matches[i] = y1_est_scaled[int(matches[i, 0])]
x2_matches[i] = x2_est_scaled[int(matches[i, 1])]
y2_matches[i] = y2_est_scaled[int(matches[i, 1])]
good_matches = np.zeros((matches.shape[0], 1))
# Loads `ground truth' positions x1, y1, x2, y2
file_contents = scio.loadmat(ground_truth_correspondence_file)
# x1, y1, x2, y2 = scio.loadmat(eval_file)
x1 = file_contents['x1']
y1 = file_contents['y1']
x2 = file_contents['x2']
y2 = file_contents['y2']
uniqueness_dist = 150
good_match_dist = 150
good_match_counter = 0
bad_match_counter = 0
top_100_counter = 0
# Used to keep track of which TA points the student has matched
# to so the student only gets credit for matching a TA point once
correct_matches = np.zeros(x2.shape[0])
# for each ground truth point in image 1
for i in range(x1.shape[0]):
# 1. find the student points within uniqueness_dist pixels of the ground truth point
x_dists = x1_matches - x1[i]
y_dists = y1_matches - y1[i]
# computes distances of each interest point to the ground truth point
dists = np.sqrt(np.power(x_dists, 2.0) + np.power(y_dists, 2.0))
# get indices of points where distance is < uniqueness_dist
close_to_truth = dists < uniqueness_dist
# 2. get the points in image1 and their corresponding matches in image2
image1_x = x1_matches[close_to_truth]
image1_y = y1_matches[close_to_truth]
image2_x = x2_matches[close_to_truth]
image2_y = y2_matches[close_to_truth]
# 3. compute the distance of the student's image2 matches to the ground truth match
x_dists_2 = image2_x - x2[i]
y_dists_2 = image2_y - y2[i]
dists_2 = np.sqrt(np.power(x_dists_2, 2.0) + np.power(y_dists_2, 2.0))
# 4. matches within good_match_dist then count it as a correct match
good = dists_2 < good_match_dist
if np.sum(good) >= 1.0:
correct_matches[i] = 1
#good_match_counter += 1
if i < 100:
top_100_counter += 1
else:
bad_match_counter += 1
precision = (np.sum(correct_matches) / x2.shape[0]) * 100.0
accuracy100 = min(
top_100_counter, 100
) # / 100) * 100# If we were testing more than the top 100, then this would be important.
print(
str(np.sum(correct_matches)) + " total good matches, " +
str(bad_match_counter) + " total bad matches.")
print(str(precision) + "% precision")
print(str(accuracy100) + "% accuracy (top 100)")
if vis > 0:
print("Vizualizing...")
# Rescale the points to the scaled input
visualize.show_correspondences(img_A, img_B, \
x1_est, y1_est, \
x2_est, y2_est, \
matches, filename)
return accuracy100