def orb(img):
img1 = rgb2gray(img)
img2 = transform.rotate(img1, 180)
tform = transform.AffineTransform(scale=(1.3, 1.1),
rotation=0.5,
translation=(0, -200))
img3 = transform.warp(img1, tform)
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(img1)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(img2)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(img3)
keypoints3 = descriptor_extractor.keypoints
descriptors3 = descriptor_extractor.descriptors
matches1 = match_descriptors(descriptors1, descriptors2, cross_check=True)
matches2 = match_descriptors(descriptors1, descriptors3, cross_check=True)
return np.hstack(
(keypoints1[matches1[:, 0]].ravel(), keypoints2[matches2[:,
1]].ravel()))
def show_matches(img1, img2, feat1, feat2):
matches12 = match_descriptors(
feat1["descriptors"], feat2["descriptors"], cross_check=True
)
fig, (ax3, ax2) = plt.subplots(1, 2, figsize=(15, 5))
c_matches = match_descriptors(
feat1["descriptors"], feat2["descriptors"], cross_check=True
)
plot_matches(ax3, img1, img2, feat1["keypoints"], feat1["keypoints"], matches12)
ax2.plot(feat1["keypoints"][:, 1], feat1["keypoints"][:, 0], ".", label="Before")
ax2.plot(feat2["keypoints"][:, 1], feat2["keypoints"][:, 0], ".", label="After")
for i, (c_idx, n_idx) in enumerate(c_matches):
x_vec = [feat1["keypoints"][c_idx, 0], feat2["keypoints"][n_idx, 0]]
y_vec = [feat1["keypoints"][c_idx, 1], feat2["keypoints"][n_idx, 1]]
dist = np.sqrt(np.square(np.diff(x_vec)) + np.square(np.diff(y_vec)))
alpha = np.clip(50 / dist, 0, 1)
ax2.plot(y_vec, x_vec, "k-", alpha=alpha, label="Match" if i == 0 else "")
ax2.legend()
ax3.set_title(r"{} $\rightarrow$ {}".format("Before", "After"))
def test_max_distance():
descs1 = np.zeros((10, 128))
descs2 = np.zeros((15, 128))
descs1[0, :] = 1
matches = match_descriptors(descs1,
descs2,
metric='euclidean',
max_distance=0.1,
cross_check=False)
assert len(matches) == 9
matches = match_descriptors(descs1,
descs2,
metric='euclidean',
max_distance=np.sqrt(128.1),
cross_check=False)
assert len(matches) == 10
matches = match_descriptors(descs1,
descs2,
metric='euclidean',
max_distance=0.1,
cross_check=True)
assert_equal(matches, [[1, 0]])
matches = match_descriptors(descs1,
descs2,
metric='euclidean',
max_distance=np.sqrt(128.1),
cross_check=True)
assert_equal(matches, [[1, 0]])
def test_binary_descriptors_unequal_descriptor_sizes_error():
"""Sizes of descriptors of keypoints to be matched should be equal."""
descs1 = np.array([[True, True, False, True], [False, True, False, True]])
descs2 = np.array([[True, False, False, True, False],
[False, True, True, True, False]])
with testing.raises(ValueError):
match_descriptors(descs1, descs2)
def match_robust(src_keypoints,
src_descriptors,
dest_keypoints,
dest_descriptors,
method='brute-force',
model_class=AffineTransform,
min_samples=4,
residual_threshold=1,
max_trials=5000,
**kwargs):
"""Find matches of keypoints between two images and filter outlier with RANSAC.
src_keypoints, dest_keypoints:
(both) `np.array(float)` of the shape (N, 2)
Coordinates of keypoints in format (x, y) each
src_descriptors, dest_descriptors:
(both) `np.array(float)` of the shape (N, P)
Descriptors of the keypoints. P is a descriptor dim
model_class:
`skimage.transform`
Model of plane transformation
min_samples:
int
Number of points needed to construct the transformation.
The harder the model is the more points are needed.
See the docs to know about each particular model.
residual_threshold:
int
Distance in pixel in which the points are considered as "the same".
max_trials:
int
Number of trials to do before stopping.
The more is the number of keypoints, the larger `max_trials` should be.
**kwargs:
dict
Other params, kept unchanged
return:
model, matches
model: transformation of the 1st frame to the second
matches:
`np.array` of shape == (num_matched_keypoints, 2)
List of matches in format: np.array([[src_kp_id1, dest_kp_id1], ...], dtype=int)
"""
if method == 'brute-force':
matches = match_descriptors(src_descriptors, dest_descriptors)
elif method == 'flann':
matches = match_descriptors(src_descriptors, dest_descriptors)
model, inliers = ransac(
(src_keypoints[matches[:, 0]], dest_keypoints[matches[:, 1]]),
model_class=model_class,
min_samples=min_samples,
residual_threshold=residual_threshold,
max_trials=max_trials,
**kwargs)
matches = matches[inliers]
return model, matches
def test_binary_descriptors_unequal_descriptor_sizes_error():
"""Sizes of descriptors of keypoints to be matched should be equal."""
descs1 = np.array([[True, True, False, True],
[False, True, False, True]])
descs2 = np.array([[True, False, False, True, False],
[False, True, True, True, False]])
with testing.raises(ValueError):
match_descriptors(descs1, descs2)
def test_binary_descriptors_rotation_crosscheck_false():
"""Verify matched keypoints and their corresponding masks results between
image and its rotated version with the expected keypoint pairs with
cross_check disabled."""
img = data.astronaut()
img = rgb2gray(img)
tform = transform.SimilarityTransform(scale=1,
rotation=0.15,
translation=(0, 0))
rotated_img = transform.warp(img, tform, clip=False)
extractor = BRIEF(descriptor_size=512)
keypoints1 = corner_peaks(corner_harris(img),
min_distance=5,
threshold_abs=0,
threshold_rel=0.1)
extractor.extract(img, keypoints1)
descriptors1 = extractor.descriptors
keypoints2 = corner_peaks(corner_harris(rotated_img),
min_distance=5,
threshold_abs=0,
threshold_rel=0.1)
extractor.extract(rotated_img, keypoints2)
descriptors2 = extractor.descriptors
matches = match_descriptors(descriptors1, descriptors2, cross_check=False)
exp_matches1 = np.array([
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19,
20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37,
38, 39, 40, 41, 42, 43, 44, 45, 46
])
exp_matches2 = np.array([
0, 31, 2, 3, 1, 4, 6, 4, 38, 5, 27, 7, 13, 10, 9, 27, 7, 11, 15, 8, 23,
14, 12, 16, 10, 25, 18, 19, 21, 20, 41, 24, 25, 26, 28, 27, 22, 23, 29,
30, 31, 32, 35, 33, 34, 30, 36
])
assert_equal(matches[:, 0], exp_matches1)
assert_equal(matches[:, 1], exp_matches2)
# minkowski takes a different code path, therefore we test it explicitly
matches = match_descriptors(descriptors1,
descriptors2,
metric='minkowski',
cross_check=False)
assert_equal(matches[:, 0], exp_matches1)
assert_equal(matches[:, 1], exp_matches2)
# it also has an extra parameter
matches = match_descriptors(descriptors1,
descriptors2,
metric='minkowski',
p=4,
cross_check=False)
assert_equal(matches[:, 0], exp_matches1)
assert_equal(matches[:, 1], exp_matches2)
def epipolar_rectify(imL,imR,show_matches=True):
descriptor_extractor = ORB(n_keypoints=2000)
descriptor_extractor.detect_and_extract(imL)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(imR)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2,metric='hamming', cross_check=True)
pts1=keypoints1[matches12[:,0],:]
pts2=keypoints2[matches12[:,1],:]
F, mask = cv2.findFundamentalMat(pts1,pts2,cv2.FM_RANSAC)
pts1 = pts1[mask.ravel()==1]
pts2 = pts2[mask.ravel()==1]
res,H1,H2=cv2.stereoRectifyUncalibrated(pts1,pts2,F,imL.shape,10)
if show_matches:
fig, ax = plt.subplots(nrows=1, ncols=1)
plot_matches(ax, imL, imR, keypoints1, keypoints2, matches12)
return H1,H2
def drawMatches2(image1, image2, feat1, feat2):
# image1 = cv2.imread(file1)
# image2 = cv2.imread(file2)
image1 = np.array(cv2.cvtColor(np.array(image1), cv2.COLOR_BGR2RGB))
image2 = np.array(cv2.cvtColor(np.array(image2), cv2.COLOR_BGR2RGB))
matches = match_descriptors(feat1['descriptors'],
feat2['descriptors'],
cross_check=True)
keypoints_left = feat1['keypoints'][matches[:, 0], :2].T
keypoints_right = feat2['keypoints'][matches[:, 1], :2].T
# print(keypoints_left.shape, keypoints_right.shape)
for i in range(keypoints_left.shape[1]):
image1 = cv2.circle(
image1, (int(keypoints_left[0, i]), int(keypoints_left[1, i])), 2,
(0, 0, 255), 4)
for i in range(keypoints_right.shape[1]):
image2 = cv2.circle(
image2, (int(keypoints_right[0, i]), int(keypoints_right[1, i])),
2, (0, 0, 255), 4)
im4 = cv2.hconcat([image1, image2])
for i in range(keypoints_left.shape[1]):
im4 = cv2.line(im4,
(int(keypoints_left[0, i]), int(keypoints_left[1, i])),
(int(keypoints_right[0, i]) + image1.shape[1],
int(keypoints_right[1, i])), (0, 255, 0), 1)
cv2.imshow("Image_lines", im4)
cv2.waitKey(0)
def findH(img1, img2):
from skimage.feature import ORB, match_descriptors
# load image
img1_gray = skimage.color.rgb2gray(img1)
img2_gray = skimage.color.rgb2gray(img2)
# extract points
detector_extractor1 = ORB(n_keypoints=3000)
detector_extractor1.detect_and_extract(img1_gray)
detector_extractor2 = ORB(n_keypoints=3000)
detector_extractor2.detect_and_extract(img2_gray)
matches = match_descriptors(detector_extractor1.descriptors,
detector_extractor2.descriptors)
match_pts1 = detector_extractor1.keypoints[matches[:, 0]].astype(int)
match_pts2 = detector_extractor2.keypoints[matches[:, 1]].astype(int)
# call RANSAC
match_pts1 = np.flip(match_pts1, axis=1)
match_pts2 = np.flip(match_pts2, axis=1)
H_2to1, _ = computeHransac(match_pts1, match_pts2)
H_2to1 = H_2to1 / H_2to1[2, 2]
print('tranform H:')
print(H_2to1)
return H_2to1
def match_features(self):
self.tforms = [ProjectiveTransform()]
self.new_corners = np.copy(self.corners)
for i in range(1, self.num_imgs):
# Find correspondences between I(n) and I(n-1).
matches = match_descriptors(self.descriptors[i - 1],
self.descriptors[i],
cross_check=True)
# Estimate the transformation between I(n) and I(n-1).
src = self.keypoints[i][matches[:, 1]][:, ::-1]
dst = self.keypoints[i - 1][matches[:, 0]][:, ::-1]
model, _ = ransac((src, dst),
ProjectiveTransform,
4,
residual_threshold=2,
max_trials=2000)
self.tforms.append(
ProjectiveTransform(model.params @ self.tforms[-1].params))
# Compute new corners transformed by models
self.new_corners[i] = self.tforms[-1](self.corners[i])
corners_min = np.min(self.new_corners, axis=1)
corners_max = np.max(self.new_corners, axis=1)
self.xLim = corners_max[:, 0] - corners_min[:, 0]
self.yLim = corners_max[:, 1] - corners_min[:, 1]
def CENSURETransform(img1, img2, fp1, fp2):
bw_img1 = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY).astype("double")
bw_img2 = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY).astype("double")
cen = CENSURE()
cen.detect(bw_img1)
kp1 = cen.keypoints
cen.detect(bw_img2)
kp2 = cen.keypoints
kp1 = np.append(kp1, fp1, axis=0)
kp2 = np.append(kp2, fp2, axis=0)
matches = match_descriptors(kp1, kp2, cross_check=True)
# fig, ax = plt.subplots(nrows=1, ncols=1)
# plot_matches(ax, img1, img2, kp1, kp2, matches)
# plt.show()
points1 = kp1[matches[:, 0]]
points2 = kp2[matches[:, 1]]
M, mask = cv2.findHomography(points1, points2, cv2.RANSAC)
if M is None:
return
else:
dst = cv2.warpPerspective(img1, M, (img1.shape[1], img1.shape[0]))
return dst
def iris_scan_orb(request):
from skimage import io
from skimage.feature import (match_descriptors, ORB)
from skimage.color import rgb2gray
from .settings import MEDIA_ROOT
img1 = rgb2gray(io.imread(MEDIA_ROOT + '/IRIS3.jpg')) # Query
img2 = rgb2gray(io.imread(MEDIA_ROOT + '/IRIS6.jpg')) # Comparing to
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(img1)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors # Query Descriptor
descriptor_extractor.detect_and_extract(img2)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors # Comparing To Descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# print("Matched: ", len(matches12), " of ", len(descriptors1))
percent = len(matches12) / len(descriptors1) * 100
# print("Percent Match - ", percent, "%")
"""if percent > 80:
print("Matched!")
else:
print("Not Matched!")"""
return render(request, 'scan.html', {'percent': percent})
def iris_scan_orb_android(file_name):
from skimage import io
from skimage.feature import (match_descriptors, ORB)
from skimage.color import rgb2gray
from .settings import MEDIA_ROOT
img1 = rgb2gray(io.imread(MEDIA_ROOT + '/' + file_name)) # Query
img2 = rgb2gray(io.imread(MEDIA_ROOT + '/IRIS9.jpg')) # Comparing to
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(img1)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors # Query Descriptor
descriptor_extractor.detect_and_extract(img2)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors # Comparing To Descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
percent = len(matches12) / len(descriptors1) * 100
return percent
def featurePointMatching(image0, image1, decimation=1, n_keypoints=750):
"""
Get a transformation model knowing 2 images
"""
# 2 images, possibly with a certain decimation factor (reduce the size of the image)
image0 = transform.rescale(image0, 1 / float(decimation))
image1 = transform.rescale(image1, 1 / float(decimation))
orb = ORB(n_keypoints=n_keypoints,
fast_threshold=0.05) # definition of the ORB detector
# Get the keypoints from the first image
orb.detect_and_extract(image0)
keypoints1 = orb.keypoints
descriptors1 = orb.descriptors
# Get the keypoints from the second image
orb.detect_and_extract(image1)
keypoints2 = orb.keypoints
descriptors2 = orb.descriptors
# Matching of descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# Select keypoints from both images with RANSAC algorithm
src = keypoints1[matches12[:, 0]][:, ::-1]
dst = keypoints2[matches12[:, 1]][:, ::-1]
model_robust, inliers = \
ransac((src, dst), transform.EuclideanTransform,
min_samples=6, residual_threshold=2)
# Get the inliners
outliers = inliers == False
return model_robust, inliers, outliers, src, dst
def orb(img_path):
image = PIL.Image.open(img_path).convert('L')
img1 = np.array(image)
img2 = tf.rotate(img1, 180)
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(img1)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(img2)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
fig, ax = plt.subplots(nrows=2, ncols=1)
plt.gray()
plot_matches(ax[0], img1, img2, keypoints1, keypoints2, matches12)
ax[0].axis('off')
ax[0].set_title(img_path)
plt.show()
return matches12.shape[0]
def iris_scan_orb_android(file_name):
from skimage import io
from skimage.feature import (match_descriptors, ORB)
from skimage.color import rgb2gray
from .settings import MEDIA_ROOT
img1 = rgb2gray(io.imread(MEDIA_ROOT + '/'+ file_name)) # Query
img2 = rgb2gray(io.imread(MEDIA_ROOT + '/IRIS9.jpg')) # Comparing to
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(img1)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors # Query Descriptor
descriptor_extractor.detect_and_extract(img2)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors # Comparing To Descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
percent = len(matches12) / len(descriptors1) * 100
return percent
def drawMatches(image1, image2, feat1, feat2):
image1 = np.array(image1)
image2 = np.array(image2)
matches = match_descriptors(feat1['descriptors'], feat2['descriptors'], cross_check=True)
print('Number of raw matches: %d.' % matches.shape[0])
keypoints_left = feat1['keypoints'][matches[:, 0], : 2]
keypoints_right = feat2['keypoints'][matches[:, 1], : 2]
np.random.seed(0)
model, inliers = ransac(
(keypoints_left, keypoints_right),
ProjectiveTransform, min_samples=4,
residual_threshold=8, max_trials=10000
)
n_inliers = np.sum(inliers)
print('Number of inliers: %d.' % n_inliers)
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_left[inliers]]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_right[inliers]]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
image3 = cv2.drawMatches(image1, inlier_keypoints_left, image2, inlier_keypoints_right, placeholder_matches, None)
plt.figure(figsize=(20, 20))
plt.imshow(image3)
plt.axis('off')
plt.show()
def test_binary_descriptors():
descs1 = np.array([[True, True, False, True, True],
[False, True, False, True, True]])
descs2 = np.array([[True, False, False, True, False],
[False, False, True, True, True]])
matches = match_descriptors(descs1, descs2)
assert_equal(matches, [[0, 0], [1, 1]])
def iris_scan_orb(request):
from skimage import io
from skimage.feature import (match_descriptors, ORB)
from skimage.color import rgb2gray
from .settings import MEDIA_ROOT
img1 = rgb2gray(io.imread(MEDIA_ROOT + '/IRIS3.jpg')) # Query
img2 = rgb2gray(io.imread(MEDIA_ROOT + '/IRIS6.jpg')) # Comparing to
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(img1)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors # Query Descriptor
descriptor_extractor.detect_and_extract(img2)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors # Comparing To Descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# print("Matched: ", len(matches12), " of ", len(descriptors1))
percent = len(matches12) / len(descriptors1) * 100
# print("Percent Match - ", percent, "%")
"""if percent > 80:
print("Matched!")
else:
print("Not Matched!")"""
return render(request, 'scan.html', {'percent': percent})
def feature_registration(src, dst):
"""Register dst to src using feature detection."""
# First convert both image to grayscale.
src_gray = rgb2gray(src)
dst_gray = rgb2gray(dst)
# Scale down.
src_scaled = transform.rescale(src_gray, 0.25)
dst_scaled = transform.rescale(src_gray, 0.25)
orb = ORB(n_keypoints=1000, fast_threshold=0.05)
orb.detect_and_extract(src_scaled)
src_keypoints = orb.keypoints
src_descriptors = orb.descriptors
orb.detect_and_extract(dst_scaled)
dst_keypoints = orb.keypoints
dst_descriptors = orb.descriptors
matches = match_descriptors(src_descriptors, dst_descriptors, cross_check=True)
src_points = src_keypoints[matches[:,0]][:, ::-1]
dst_points = dst_keypoints[matches[:,0]][:, ::,-1]
model, inlier = ransac(
(src_points, dst_points), ProjectiveTransform, min_samples=4, residual_threshold=2)
)
def get_feature_displacment(self, img1, img2):
# if cfg.shot_c == 3:
# img1 = rgb2gray(img1)
# img2 = rgb2gray(img2)
keypoints1 = corner_peaks(corner_harris(img1),
min_distance=5,
threshold_rel=0.02)
keypoints2 = corner_peaks(corner_harris(img2),
min_distance=5,
threshold_rel=0.02)
self.extractor.extract(img1, keypoints1)
keypoints1 = keypoints1[self.extractor.mask]
descriptors1 = self.extractor.descriptors
self.extractor.extract(img2, keypoints2)
keypoints2 = keypoints2[self.extractor.mask]
descriptors2 = self.extractor.descriptors
if descriptors1.shape[0] == 0 or descriptors2.shape[0] == 0:
return None # no feature found
matches = match_descriptors(descriptors1,
descriptors2,
cross_check=True,
max_ratio=0.85)
if matches.shape[0] < 4: return None
dist = np.sum(
(keypoints1[matches[:, 0]] - keypoints2[matches[:, 1]])**2, axis=1)
return np.mean(self.median_outlier_filter(dist))
def test_binary_descriptors_lena_rotation_crosscheck_true():
"""Verify matched keypoints and their corresponding masks results between
lena image and its rotated version with the expected keypoint pairs with
cross_check enabled."""
img = data.lena()
img = rgb2gray(img)
tform = tf.SimilarityTransform(scale=1, rotation=0.15, translation=(0, 0))
rotated_img = tf.warp(img, tform, clip=False)
extractor = BRIEF(descriptor_size=512)
keypoints1 = corner_peaks(corner_harris(img), min_distance=5)
extractor.extract(img, keypoints1)
descriptors1 = extractor.descriptors
keypoints2 = corner_peaks(corner_harris(rotated_img), min_distance=5)
extractor.extract(rotated_img, keypoints2)
descriptors2 = extractor.descriptors
matches = match_descriptors(descriptors1, descriptors2, cross_check=True)
exp_matches1 = np.array([
0, 1, 2, 4, 6, 7, 9, 10, 11, 12, 13, 15, 16, 17, 19, 20, 21, 24, 26,
27, 28, 29, 30, 35, 36, 38, 39, 40, 42, 44, 45
])
exp_matches2 = np.array([
33, 0, 35, 1, 3, 2, 6, 4, 9, 11, 10, 7, 8, 5, 14, 13, 15, 16, 17, 18,
19, 21, 22, 24, 23, 26, 27, 25, 28, 29, 30
])
assert_equal(matches[:, 0], exp_matches1)
assert_equal(matches[:, 1], exp_matches2)
def drawMatches(image1, image2, feat1, feat2):
t0 = time.time()
matches = match_descriptors(feat1['descriptors'], feat2['descriptors'], cross_check=True)
t1 = time.time()
print("Time to extract matches: ", t1-t0)
print('Number of raw matches: %d.' % matches.shape[0])
keypoints_left = feat1['keypoints'][matches[:, 0], : 2]
keypoints_right = feat2['keypoints'][matches[:, 1], : 2]
# print(type(matches), matches.shape, feat1['keypoints'].shape)
# print(feat1['keypoints'][0:4, :])
np.random.seed(0)
t0 = time.time()
model, inliers = ransac(
(keypoints_left, keypoints_right),
AffineTransform, min_samples=4,
residual_threshold=8, max_trials=10000
)
t1 = time.time()
print("Time for ransac: ", t1-t0)
n_inliers = np.sum(inliers)
print('Number of inliers: %d.' % n_inliers)
inlier_keypoints_left = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_left[inliers]]
inlier_keypoints_right = [cv2.KeyPoint(point[0], point[1], 1) for point in keypoints_right[inliers]]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
image3 = cv2.drawMatches(image1, inlier_keypoints_left, image2, inlier_keypoints_right, placeholder_matches, None)
plt.figure(figsize=(20, 20))
plt.imshow(image3)
plt.axis('off')
plt.show()
def MatchPics(I1,I2):
if I1.ndim == 3:
I1 = rgb2gray(I1)
if I2.ndim == 3:
I2 = rgb2gray(I2)
points1 = corner_peaks(corner_fast(I1,n=12,threshold=0.15),min_distance=1)
points2 = corner_peaks(corner_fast(I2,n=12,threshold=0.15),min_distance=1)
extractor = BRIEF()
extractor.extract(I1,points1)
points1 = points1[extractor.mask]
descriptors1 = extractor.descriptors
extractor.extract(I2,points2)
points2 = points2[extractor.mask]
descriptors2 = extractor.descriptors
matches = match_descriptors(descriptors1,descriptors2,metric = 'hamming',cross_check=True)
#these points are y,x (row,col)
locs1 = points1[matches[:,0]]
locs2 = points2[matches[:,1]]
#Change to x,y (col,row)
xy1 = np.array([locs1[:,1],locs1[:,0]])
xy1 = xy1.transpose()
xy2 = np.array([locs2[:,1],locs2[:,0]])
xy2 = xy2.transpose()
fig, ax = plt.subplots()
plot_matches(ax,I1,I2,points1,points2,matches,keypoints_color='r',only_matches=True)#,matches_color='y')
return [xy1,xy2]
def test_binary_descriptors_rotation_crosscheck_true():
"""Verify matched keypoints and their corresponding masks results between
image and its rotated version with the expected keypoint pairs with
cross_check enabled."""
img = data.astronaut()
img = rgb2gray(img)
tform = tf.SimilarityTransform(scale=1, rotation=0.15, translation=(0, 0))
rotated_img = tf.warp(img, tform, clip=False)
extractor = BRIEF(descriptor_size=512)
keypoints1 = corner_peaks(corner_harris(img), min_distance=5,
threshold_abs=0, threshold_rel=0.1)
extractor.extract(img, keypoints1)
descriptors1 = extractor.descriptors
keypoints2 = corner_peaks(corner_harris(rotated_img), min_distance=5,
threshold_abs=0, threshold_rel=0.1)
extractor.extract(rotated_img, keypoints2)
descriptors2 = extractor.descriptors
matches = match_descriptors(descriptors1, descriptors2, cross_check=True)
exp_matches1 = np.array([ 0, 2, 3, 4, 5, 6, 9, 11, 12, 13, 14, 17,
18, 19, 21, 22, 23, 26, 27, 28, 29, 31, 32, 33,
34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46])
exp_matches2 = np.array([ 0, 2, 3, 1, 4, 6, 5, 7, 13, 10, 9, 11,
15, 8, 14, 12, 16, 18, 19, 21, 20, 24, 25, 26,
28, 27, 22, 23, 29, 30, 31, 32, 35, 33, 34, 36])
assert_equal(matches[:, 0], exp_matches1)
assert_equal(matches[:, 1], exp_matches2)
def test_binary_descriptors():
descs1 = np.array([[True, True, False, True, True],
[False, True, False, True, True]])
descs2 = np.array([[True, False, False, True, False],
[False, False, True, True, True]])
matches = match_descriptors(descs1, descs2)
assert_equal(matches, [[0, 0], [1, 1]])
def main():
baseImg = loadResized("base.jpg", 600, 410)
atlasImg = loadResized("atlas.jpg", 600, 410)
orb = (ORB(n_keypoints=800,
fast_threshold=0.05), ORB(n_keypoints=800, fast_threshold=0.05))
orb[0].detect_and_extract(baseImg)
orb[1].detect_and_extract(atlasImg)
baseData = [orb[0].keypoints, orb[0].descriptors]
atlasData = [orb[1].keypoints, orb[1].descriptors]
match = match_descriptors(baseData[1], atlasData[1])
dst = baseData[0][match[:, 0]][:, ::-1]
src = atlasData[0][match[:, 1]][:, ::-1]
robust, inliers = ransac((src, dst),
ProjectiveTransform,
min_samples=4,
residual_threshold=1,
max_trials=300)
r, c = baseImg.shape[:2]
corners = np.array([[0, 0], [0, r], [c, 0], [c, r]])
warpedCorners = robust(corners)
allCorners = np.vstack((warpedCorners, corners))
cornerMin = np.min(allCorners, axis=0)
cornerMax = np.max(allCorners, axis=0)
outputShape = (cornerMax - cornerMin)
outputShape = np.ceil(outputShape[::-1]).astype(int)
offSet = SimilarityTransform(translation=cornerMin)
atlasWarped = warp(atlasImg,
offSet.inverse,
order=3,
output_shape=outputShape,
cval=-1)
atlasMask = (atlasWarped != -1)
atlasWarped[~atlasMask] = 0
fig, ax = plt.subplots(figsize=(12, 12))
diffImg = atlasWarped - baseImg
ax.imshow(diffImg, cmap="gray")
ax.axis("off")
plt.show()
compare(atlasWarped, baseImg, figsize=(12, 10))
costs = generateCosts(np.abs(atlasWarped, baseImg), atlasWarped & baseImg)
fig, ax = plt.subplots(figsize=(15, 12))
ax.imshow(costs, cmap="gray", interpolation="none")
ax.axis("off")
outputImg = cv2.addWeighted(baseImg, .3, atlasImg, 1, 0)
cv2.imshow("Output", outputImg)
cv2.waitKey(0)
cv2.destroyAllWindows()
def recognize(pattern, name, scene):
zipped_matches = []
match = match_descriptors(scene,
pattern,
cross_check=True,
max_distance=0.2)
zipped_matches.append([match.size, name, match])
return zipped_matches
def get_pair(outputdir,
imgs,
p,
offsets,
ds,
overlap_fraction,
orb,
plotpairs=0,
res_th=10,
num_inliers=100,
transformname="EuclideanTransform"):
"""Create inlier keypoint pairs."""
pair_tstart = time()
overlap_pixels = [
int(math.ceil(d * of * 1 / ds))
for d, of in zip(imgs[0][0].shape, overlap_fraction)
]
f1, f2 = downsample_images(p, imgs, ds)
p1, p2 = select_imregions(p[2], f1, f2, overlap_pixels)
kp1, de1 = get_keypoints(orb, p1)
kp2, de2 = get_keypoints(orb, p2)
kp1, kp2 = reset_imregions(p[2], kp1, kp2, overlap_pixels, f1.shape)
matches = match_descriptors(de1, de2, cross_check=True)
dst = kp1[matches[:, 0]][:, ::-1]
src = kp2[matches[:, 1]][:, ::-1]
transform = eval("tf.%s" % transformname)
model, inliers = ransac((src, dst),
transform,
min_samples=4,
residual_threshold=res_th,
max_trials=1000,
stop_sample_num=num_inliers)
# get the weighing kernel in z
k = gaussian(offsets * 2 + 1, 1, sym=True)
w = k[offsets - (p[1][0] - p[0][0])]
# transform from downsampled space to full
S = np.array([[ds, 0, 0], [0, ds, 0], [0, 0, 1]])
s = np.c_[src, np.ones(src.shape[0])].dot(S)[inliers, :2]
d = np.c_[dst, np.ones(dst.shape[0])].dot(S)[inliers, :2]
pair = (p, s, d, model, w)
pairstring = generate_pairstring(offsets, ds, p)
pairfile = path.join(outputdir, pairstring + '.pickle')
pickle.dump(pair, open(pairfile, 'wb'))
if plotpairs:
plot_pair_ransac(outputdir, pairstring, p, f1, f2, kp1, kp2, matches,
inliers)
print('%s done in: %6.2f s; matches: %05d; inliers: %05d' %
(pairstring, time() - pair_tstart, len(matches), np.sum(inliers)))
return pair
def register_image_pair(idx, path_img_target, path_img_source, path_out):
""" register two images together
:param int idx: empty parameter for using the function in parallel
:param str path_img_target: path to the target image
:param str path_img_source: path to the source image
:param str path_out: path for exporting the output
:return tuple(str,float):
"""
start = time.time()
# load and denoise reference image
img_target = io.imread(path_img_target)[..., :3]
img_target = denoise_wavelet(img_target,
wavelet_levels=7,
multichannel=True)
img_target_gray = rgb2gray(img_target)
# load and denoise moving image
img_source = io.imread(path_img_source)[..., :3]
img_source = denoise_bilateral(img_source,
sigma_color=0.05,
sigma_spatial=2,
multichannel=True)
img_source_gray = rgb2gray(img_source)
# detect ORB features on both images
detector_target = ORB(n_keypoints=150)
detector_source = ORB(n_keypoints=150)
detector_target.detect_and_extract(img_target_gray)
detector_source.detect_and_extract(img_source_gray)
matches = match_descriptors(detector_target.descriptors,
detector_source.descriptors)
# robustly estimate affine transform model with RANSAC
model, _ = ransac(
(detector_target.keypoints[matches[:, 0]],
detector_source.keypoints[matches[:, 1]]),
AffineTransform,
min_samples=25,
max_trials=500,
residual_threshold=0.9,
)
# warping source image with estimated transformations
path_img_warped = os.path.join(path_out, NAME_IMAGE_WARPED % idx)
if model:
img_warped = warp(img_target,
model.inverse,
output_shape=img_target.shape[:2])
try:
io.imsave(path_img_warped, img_warped)
except Exception:
traceback.print_exc()
else:
warnings.warn("Image registration failed.", RuntimeWarning)
path_img_warped = None
# summarise experiment
execution_time = time.time() - start
return path_img_warped, execution_time
def drawMatches(file1, file2, feat1, feat2):
image1 = np.array(Image.open(file1).convert('RGB'))
image2 = np.array(Image.open(file2).convert('RGB'))
matches = match_descriptors(feat1['descriptors'],
feat2['descriptors'],
cross_check=True)
print('Number of raw matches: %d.' % matches.shape[0])
keypoints_left = feat1['keypoints'][matches[:, 0], :2]
keypoints_right = feat2['keypoints'][matches[:, 1], :2]
keypoints_left_new = []
keypoints_right_new = []
for i in range(keypoints_left.shape[0]):
if np.all(image1[int(keypoints_left[i, 1]),
int(keypoints_left[i, 0])]) == 0:
continue
keypoints_left_new.append(keypoints_left[i])
keypoints_left_new = np.array(keypoints_left_new)[:]
for i in range(0, keypoints_right.shape[0]):
if np.all(image2[int(keypoints_right[i, 1]),
int(keypoints_right[i, 0])]) == 0:
continue
keypoints_right_new.append(keypoints_right[i])
keypoints_right_new = np.array(keypoints_right_new)[:]
print(keypoints_left_new.shape, keypoints_right_new.shape)
np.random.seed(0)
model, inliers = ransac((keypoints_left_new, keypoints_right_new),
ProjectiveTransform,
min_samples=4,
residual_threshold=8,
max_trials=10000)
n_inliers = np.sum(inliers)
print('Number of inliers: %d.' % n_inliers)
inlier_keypoints_left = [
cv2.KeyPoint(point[0], point[1], 1)
for point in keypoints_left_new[inliers]
]
inlier_keypoints_right = [
cv2.KeyPoint(point[0], point[1], 1)
for point in keypoints_right_new[inliers]
]
placeholder_matches = [cv2.DMatch(idx, idx, 1) for idx in range(n_inliers)]
image3 = cv2.drawMatches(image1, inlier_keypoints_left, image2,
inlier_keypoints_right, placeholder_matches, None)
#image3 = Image.fromarray(image3)
#image3.save('/home/udit/d2-net/media/rcar_Pairs_overcast/9_extract.jpg')
cv2.imwrite('/home/udit/d2-net/extract.jpg', image3)
plt.figure(figsize=(20, 20))
plt.imshow(image3)
plt.axis('off')
plt.show()
def ransac_transform(src_keypoints,
src_descriptors,
dest_keypoints,
dest_descriptors,
max_trials=N_TRIALS,
residual_threshold=1,
return_matches=False):
"""Match keypoints of 2 images and find ProjectiveTransform using RANSAC algorithm.
src_keypoints ((N, 2) np.ndarray) : source coordinates
src_descriptors ((N, 256) np.ndarray) : source descriptors
dest_keypoints ((N, 2) np.ndarray) : destination coordinates
dest_descriptors ((N, 256) np.ndarray) : destination descriptors
max_trials (int) : maximum number of iterations for random sample selection.
residual_threshold (float) : maximum distance for a data point to be classified as an inlier.
return_matches (bool) : if True function returns matches
Returns:
skimage.transform.ProjectiveTransform : transform of source image to destination image
(Optional)(N, 2) np.ndarray : inliers' indexes of source and destination images
"""
# your code here
matches = match_descriptors(src_descriptors, dest_descriptors)
n = matches.shape[0]
res_inds = [-1, -1, -1, -1]
res_kol = 0
for trial in range(max_trials):
inds = random.sample(range(n), 4)
h = find_homography(src_keypoints[matches[inds, 0]],
dest_keypoints[matches[inds, 1]])
projected = ProjectiveTransform(h)(src_keypoints[matches[:, 0]])
dist = np.sqrt(
np.power(projected[:, 0] - dest_keypoints[matches[:, 1], 0], 2) +
np.power(projected[:, 1] - dest_keypoints[matches[:, 1], 1], 2))
kol = np.sum(dist < residual_threshold)
if kol > res_kol:
print("trial: {}, kol: {}".format(trial, kol))
res_kol = kol
res_inds = inds
if res_kol > len(src_keypoints) / 10 and trial > max_trials / 10:
break
h = find_homography(src_keypoints[matches[res_inds, 0]],
dest_keypoints[matches[res_inds, 1]])
transform = ProjectiveTransform(h)
projected = transform(src_keypoints[matches[:, 0]])
dist = np.sqrt(
np.power(projected[:, 0] - dest_keypoints[matches[:, 1], 0], 2) +
np.power(projected[:, 1] - dest_keypoints[matches[:, 1], 1], 2))
inliers = matches[dist < residual_threshold]
print("{} inliers matched".format(inliers.shape[0]))
transform = ProjectiveTransform(
find_homography(src_keypoints[inliers[:, 0]],
dest_keypoints[inliers[:, 1]]))
if return_matches:
return transform, inliers
else:
return transform
def get_displacement(image0, image1):
"""
Gets displacement (in pixels I think) difference between 2 images using scikit-image
not as accurate as the opencv version i think.
:param image0: reference image
:param image1: target image
:return:
"""
from skimage.feature import (match_descriptors, ORB, plot_matches)
from skimage.color import rgb2gray
from scipy.spatial.distance import hamming
from scipy import misc
image0_gray = rgb2gray(image0)
image1_gray = rgb2gray(image1)
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(image0_gray)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(image1_gray)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# Sort the matches based on distance. Least distance
# is better
distances12 = []
for match in matches12:
distance = hamming(descriptors1[match[0]], descriptors2[match[1]])
distances12.append(distance)
indices = np.arange(len(matches12))
indices = [index for (_, index) in sorted(zip(distances12, indices))]
matches12 = matches12[indices]
# collect displacement from the first 10 matches
dx_list = []
dy_list = []
for mat in matches12[:10]:
# Get the matching key points for each of the images
img1_idx = mat[0]
img2_idx = mat[1]
# x - columns
# y - rows
(x1, y1) = keypoints1[img1_idx]
(x2, y2) = keypoints2[img2_idx]
dx_list.append(abs(x1 - x2))
dy_list.append(abs(y1 - y2))
dx_median = np.median(np.asarray(dx_list, dtype=np.double))
dy_median = np.median(np.asarray(dy_list, dtype=np.double))
# plot_matches(image0, image1, descriptors1, descriptors2, matches12[:10])
return dx_median, dy_median
def get_pairs(imgs, unique_pairs, offsets, subsample_factor, overlap_pixels,
n_kp):
"""Create inlier keypoint pairs."""
orb = ORB(n_keypoints=n_kp, fast_threshold=0.05)
k = gaussian(offsets * 2 + 1, 1, sym=True)
tf = SimilarityTransform # tf = RigidTransform
tf0 = SimilarityTransform()
# FIXME: is there no rigid model in scikit-image???
# this is needed for input to RANSAC
pairs = []
init_tfs = np.empty([n_slcs, n_tiles, 3])
for p in unique_pairs:
pair_tstart = time()
full_im1, full_im2 = subsample_images(p, imgs, subsample_factor)
part_im1, part_im2 = select_imregions(p[2], full_im1, full_im2,
overlap_pixels)
keyp_im1, desc_im1 = get_keypoints(orb, part_im1)
keyp_im2, desc_im2 = get_keypoints(orb, part_im2)
keyp_im1, keyp_im2 = reset_imregions(p[2], keyp_im1, keyp_im2,
overlap_pixels, full_im1.shape)
matches = match_descriptors(desc_im1, desc_im2, cross_check=True)
dst = keyp_im1[matches[:, 0]][:, ::-1]
src = keyp_im2[matches[:, 1]][:, ::-1]
model, inliers = ransac((src, dst),
tf,
min_samples=4,
residual_threshold=2,
max_trials=300)
w = k[offsets - (p[1][0] - p[0][0])]
pairs.append((p, src[inliers], dst[inliers], model, w))
if (p[0][1] == 0) & (p[0][0] == p[1][0]
): # referenced to tile 0 within the same slice
tf1 = tf0.__add__(model)
itf = [
math.acos(min(tf1.params[0, 0],
1)), # FIXME!!! with RigidTransform
tf1.params[0, 2],
tf1.params[1, 2]
]
init_tfs[p[1][0], p[1][1], :] = np.array(itf)
if (p[0][1] == p[1][1] == 0) & (
p[1][0] - p[0][0] == 1): # if [slcX,tile0] to [slcX-1,tile0]
tf0 = tf0.__add__(model)
plot_pair_ransac(p, full_im1, full_im2, keyp_im1, keyp_im2, matches,
inliers)
print('Pair done in: %.2f s' % (time() - pair_tstart, ))
return pairs, init_tfs
def get_matrix(image_tif_bgrn, image_jpg_bgr, verbose=False):
"""Get similarity transform matrix
ORB Limitation: https://github.com/scikit-image/scikit-image/issues/1472 """
im_tif_adjusted = match_color_curve_tif2jpg(image_tif_bgrn, image_jpg_bgr)
jpg_gray = cv2.cvtColor(image_jpg_bgr, cv2.COLOR_BGR2GRAY).astype(np.uint8)
tif_gray = cv2.cvtColor(im_tif_adjusted,
cv2.COLOR_BGR2GRAY).astype(np.uint8)
number_of_keypoints = 100
# Initialize ORB
# This number of keypoints is large enough for robust results,
# but low enough to run quickly.
orb = ORB(n_keypoints=number_of_keypoints, fast_threshold=0.05)
orb2 = ORB(n_keypoints=number_of_keypoints, fast_threshold=0.05)
try:
# Detect keypoints
orb.detect_and_extract(jpg_gray)
keypoints_jpg = orb.keypoints
descriptors_jpg = orb.descriptors
orb2.detect_and_extract(tif_gray)
keypoints_tif = orb2.keypoints
descriptors_tif = orb2.descriptors
except IndexError:
raise KeypointDetectionException('ORB Keypoint detection failed')
# Match descriptors between images
matches = match_descriptors(descriptors_jpg,
descriptors_tif,
cross_check=True)
# Select keypoints from
# * source (image to be registered)
# * target (reference image)
src = keypoints_jpg[matches[:, 0]][:, ::-1]
dst = keypoints_tif[matches[:, 1]][:, ::-1]
model_robust, inliers = ransac((src, dst),
TranslationTransform,
min_samples=4,
residual_threshold=1,
max_trials=300)
if verbose:
print(inliers)
print("number of matching keypoints", np.sum(inliers))
if inliers is None or np.sum(inliers) < 3 or model_robust is None:
raise ValueError('Possible mismatched JPG and TIF')
if is_translational(model_robust):
# we assume src and dst are not rotated relative to each other
# get rid of any rotational noise introduced during normalization/centering in transform estimate function
model_robust.params[0, 0] = 1.0
model_robust.params[1, 1] = 1.0
return model_robust
else:
raise ValueError('Invalid Model')
def produceMatches(imL1, imR1, panoramas=False, overlap_size=None):
imL = imL1.copy()
imR = imR1.copy()
if (panoramas):
if (overlap_size == None):
overlap_size = int(imL.shape[1] * 0.4)
imL[:, :-overlap_size, :] = 0
imR[:, overlap_size:, :] = 0
imLgray = rgb2gray(imL)
imRgray = rgb2gray(imR)
keypointsL = corner_peaks(corner_harris(imLgray),
threshold_rel=0.001,
min_distance=10)
keypointsR = corner_peaks(corner_harris(imRgray),
threshold_rel=0.001,
min_distance=10)
extractor = BRIEF()
extractor.extract(imLgray, keypointsL)
keypointsL = keypointsL[extractor.mask]
descriptorsL = extractor.descriptors
extractor.extract(imRgray, keypointsR)
keypointsR = keypointsR[extractor.mask]
descriptorsR = extractor.descriptors
matchesLR = match_descriptors(descriptorsL, descriptorsR, cross_check=True)
src = []
dst = []
for coord in matchesLR:
src.append(keypointsL[coord[0]])
dst.append(keypointsR[coord[1]])
src = np.array(src)
dst = np.array(dst)
src_c = src.copy()
dst_c = dst.copy()
src_c[:, 1] = src[:, 0]
src_c[:, 0] = src[:, 1]
dst_c[:, 1] = dst[:, 0]
dst_c[:, 0] = dst[:, 1]
# robustly estimate affine transform model with RANSAC
model_robust, inliers = ransac((src_c, dst_c),
ProjectiveTransform,
min_samples=4,
residual_threshold=8,
max_trials=250)
return (matchesLR, model_robust, inliers)
def process(self, img2, image_gray):
# img2 = warp(img2)
patch_size = [640]
img2 = rgb2gray(img2)
image_gray = rgb2gray(img2)
blobs_dog = blob_dog(image_gray, min_sigma=0.2, max_sigma=225, sigma_ratio=1.6, threshold=.5)
blobs_dog[:, 2] = blobs_dog[:, 2]
blobs = [blobs_dog]
colors = ['black']
titles = ['Difference of Gaussian']
sequence = zip(blobs, colors, titles)
# plt.imshow(img2)
# plt.axis("equal")
# plt.show()
for blobs, color, title in sequence:
print(len(blobs))
for blob in blobs:
y, x, r = blob
plotx = x
ploty = y
for i in range (3):
keypoints1 = corner_peaks(corner_harris(Array.image_arr[i]), min_distance=1)
keypoints2 = corner_peaks(corner_harris(img2), min_distance=1)
extractor = BRIEF(patch_size=30, mode="uniform")
extractor.extract(Array.image_arr[i], keypoints1)
keypoints1 = keypoints1[extractor.mask]
descriptors1 = extractor.descriptors
extractor.extract(img2, keypoints2)
keypoints2 = keypoints2[extractor.mask]
descriptors2 = extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# print(keypoints1, keypoints2)
# print(matches12)
#FUCKGGGPLAYT
for pizdezh in matches12:
X = keypoints2[pizdezh[1]][1]
Y = keypoints2[pizdezh[1]][0]
if sqrt((plotx - X)**2 + (ploty - Y)**2) < r:
seen = [{
"type": Array.type_arr[i],
"center_shift": (plotx - 160/2) * -0.02,
"distance": image_gray[y][x] / 0.08
}]
print seen
data.seen.add(seen)
break
def run(img1_path, img2_path, type='ORB'):
img1, img2 = load_images(img1_path, img2_path)
if type == 'ORB':
keyp1, keyp2, desc1, desc2 = extract_by_orb(img1, img2, 100)
elif type == 'BRIEF':
keyp1, keyp2, desc1, desc2 = extract_by_brief(img1, img2, 10)
matches = match_descriptors(desc1, desc2, cross_check=True)
plot(img1, img2, keyp1, keyp2, matches)
def test_max_distance():
descs1 = np.zeros((10, 128))
descs2 = np.zeros((15, 128))
descs1[0, :] = 1
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_distance=0.1, cross_check=False)
assert len(matches) == 9
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_distance=np.sqrt(128.1),
cross_check=False)
assert len(matches) == 10
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_distance=0.1,
cross_check=True)
assert_equal(matches, [[1, 0]])
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_distance=np.sqrt(128.1),
cross_check=True)
assert_equal(matches, [[1, 0]])
def test_max_ratio():
descs1 = 10 * np.arange(10)[:, None].astype(np.float32)
descs2 = 10 * np.arange(15)[:, None].astype(np.float32)
descs2[0] = 5.0
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=1.0, cross_check=False)
assert_equal(len(matches), 10)
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=0.6, cross_check=False)
assert_equal(len(matches), 10)
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=0.5, cross_check=False)
assert_equal(len(matches), 9)
descs1[0] = 7.5
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=0.5, cross_check=False)
assert_equal(len(matches), 9)
descs2 = 10 * np.arange(1)[:, None].astype(np.float32)
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=1.0, cross_check=False)
assert_equal(len(matches), 10)
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=0.5, cross_check=False)
assert_equal(len(matches), 10)
descs1 = 10 * np.arange(1)[:, None].astype(np.float32)
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=1.0, cross_check=False)
assert_equal(len(matches), 1)
matches = match_descriptors(descs1, descs2, metric='euclidean',
max_ratio=0.5, cross_check=False)
assert_equal(len(matches), 1)
def getDisplacement(Image0, Image1):
Image0Gray = rgb2gray(Image0)
Image1Gray = rgb2gray(Image1)
descriptor_extractor = ORB(n_keypoints=200)
descriptor_extractor.detect_and_extract(Image0Gray)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(Image1Gray)
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# Sort the matches based on distance. Least distance
# is better
distances12 = []
for match in matches12:
distance = hamming(descriptors1[match[0]], descriptors2[match[1]])
distances12.append(distance)
indices = np.range(len(matches12))
indices = [index for (_, index) in sorted(zip(distances12, indices))]
matches12 = matches12[indices]
# collect displacement from the first 10 matches
dxList = []
dyList = []
for mat in matches12[:10]:
# Get the matching keypoints for each of the images
img1_idx = mat[0]
img2_idx = mat[1]
# x - columns
# y - rows
(x1, y1) = keypoints1[img1_idx]
(x2, y2) = keypoints2[img2_idx]
dxList.append(abs(x1 - x2))
dyList.append(abs(y1 - y2))
dxMedian = np.median(np.asarray(dxList, dtype=np.double))
dyMedian = np.median(np.asarray(dyList, dtype=np.double))
plot_matches(Image0, Image1, descriptors1, descriptors2, matches12[:10])
return dxMedian, dyMedian
def NextGID(self,image):
""" Calculates the next Group ID for the input image """
NewImg = self.LoadImage(image,Greyscale=True,scale=0.25)
self.orb.detect_and_extract(NewImg)
NewImgKeyDescr = (self.orb.keypoints, self.orb.descriptors)
for PreImgKeyDescr in reversed(self.ImagesKeypointsDescriptors):
# Check for overlap
matcheOfDesc = match_descriptors(PreImgKeyDescr[1], NewImgKeyDescr[1], cross_check=True)
# Select keypoints from the source (image to be registered)
# and target (reference image)
src = NewImgKeyDescr[0][matcheOfDesc[:, 1]][:, ::-1]
dst = PreImgKeyDescr[0][matcheOfDesc[:, 0]][:, ::-1]
model_robust, inliers = ransac((src, dst), ProjectiveTransform,
min_samples=4, residual_threshold=1, max_trials=300)
NumberOfTrueMatches = np.sum(inliers) #len(inliers[inliers])
if NumberOfTrueMatches > 100 :
# Image has overlap
logger.debug('Image {0} found a match! (No: of Matches={1})'.format(image,NumberOfTrueMatches))
break
else :
logger.debug('Image {0} not matching..(No: of Matches={1})'.format(image,NumberOfTrueMatches))
continue
else:
# None of the images in the for loop has any overlap...So this is a new Group
self.ImagesKeypointsDescriptors = [] # Erase all previous group items
# self.ImagesWithOverlap = []
# Increment Group ID
self.CurrentGroupID += 1
logger.debug('Starting a new Panorama group (GID={0})'.format(self.CurrentGroupID))
# Append the latest image to the current group
self.ImagesKeypointsDescriptors.append(NewImgKeyDescr)
# self.ImagesWithOverlap.append(NewImg)
# Return the current group ID
return self.CurrentGroupID
def _orb_ransac_shift(self, im1, im2, template):
descriptor_extractor = ORB() #n_keypoints=self.parameters['n_keypoints'])
key1, des1 = self._find_key_points(descriptor_extractor, im1)
key2, des2 = self._find_key_points(descriptor_extractor, im2)
matches = match_descriptors(des1, des2, cross_check=True)
# estimate affine transform model using all coordinates
src = key1[matches[:, 0]]
dst = key2[matches[:, 1]]
# robustly estimate affine transform model with RANSAC
model_robust, inliers = ransac((src, dst), AffineTransform,
min_samples=3, residual_threshold=1,
max_trials=100)
# diff = []
# for p1, p2 in zip(src[inliers], dst[inliers]):
# diff.append(p2-p1)
# return np.mean(diff, axis=0)
return model_robust.translation
def find_two_matches(base_img, img, base_k, img_k, base_d, img_d, min_matches=10):
matches = match_descriptors(base_d, img_d, cross_check=True)
# * src (image to be registered):
# * dst (reference image):
src = img_k[matches[:,1]][:,::-1]
dst = base_k[matches[:,0]][:,::-1]
if matches.shape[0] > min_matches:
# model_robust, inliers = ransac((src, dst), ProjectiveTransform,
# min_samples=10, residual_threshold=10,
# stop_sample_num=100, max_trials=300)
#
model_robust, inliers = ransac((src, dst), AffineTransform,
min_samples=6, residual_threshold=3,
max_trials=100)
ransac_matches = matches[inliers]
inlierRatio = ransac_matches.shape[0]/float(matches.shape[0])
return model_robust, ransac_matches, inlierRatio
else:
return np.zeros((0, 2)), np.zeros((0, 2)), 0.0
def match_from_to_compare(fk, fd, tk, td, min_matches=10):
# get matching keypoints between images (from to) or (previous, base) or (next, base)
ransac_matches = np.zeros((0, 2))
matches = np.zeros((0, 2))
inliers = np.zeros((0, 2))
try:
# skimage way
# may need to reverse
matches = match_descriptors(fd, td, cross_check=True)
src = tk[matches[:, 1]][::-1]
dst = fk[matches[:, 0]][::-1]
logging.info("STARTING MATCH src: %d dst %d" % (src.shape[0], dst.shape[0]))
if src.shape[0] > min_matches:
# TODO - select which transform to use based on sensor data?
try:
model_robust, inliers = ransac(
(src, dst),
AffineTransform,
min_samples=min_matches,
stop_sample_num=100,
max_trials=2000,
stop_probability=0.995,
residual_threshold=2,
)
except Exception, e:
logging.error(e)
logging.info("FOUND inliers %d" % inliers.shape[0])
if inliers.shape[0]:
num_correct = inliers.shape[0]
num_matches = src.shape[0]
num_false = num_matches - num_correct
ransac_matches = matches[inliers]
perc_correct = 1 - float(num_false) / float(num_matches)
return model_robust, ransac_matches, matches, inliers, perc_correct
else:
def get_translation_tool(self, n_keypoints=1000):
# Convert images to grayscale
src_image = rgb2gray(self.src_image)
dst_image = rgb2gray(self.dst_image)
# Initiate an ORB class object which can extract features & descriptors from images.
# Set the amount of features that should be found (more = more accurate)
descriptor_extractor = ORB(n_keypoints=n_keypoints)
# Extract features and descriptors from source image
descriptor_extractor.detect_and_extract(src_image)
self.keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
# Extract features and descriptors from destination image
descriptor_extractor.detect_and_extract(dst_image)
self.keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
# Matches the descriptors and gives them rating as to how similar they are
self.matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
# Selects the coordinates from source image and destination image based on the
# indices given from the match_descriptors function.
src = self.keypoints1[self.matches12[:, 0]][:, ::-1]
dst = self.keypoints2[self.matches12[:, 1]][:, ::-1]
# Filters out the outliers and generates the transformation matrix based on only the inliers
model_robust, inliers = \
ransac((src, dst), ProjectiveTransform,
min_samples=4, residual_threshold=2)
# This returns the object "model_robust" which contains the tranformation matrix and
# uses that to translate any coordinate point from source to destination image.
return model_robust, inliers
keypoints3 = corner_peaks(corner_harris(img3), min_distance=5)
extractor = BRIEF()
extractor.extract(img1, keypoints1)
keypoints1 = keypoints1[extractor.mask]
descriptors1 = extractor.descriptors
extractor.extract(img2, keypoints2)
keypoints2 = keypoints2[extractor.mask]
descriptors2 = extractor.descriptors
extractor.extract(img3, keypoints3)
keypoints3 = keypoints3[extractor.mask]
descriptors3 = extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
matches13 = match_descriptors(descriptors1, descriptors3, cross_check=True)
fig, ax = plt.subplots(nrows=2, ncols=1)
plt.gray()
plot_matches(ax[0], img1, img2, keypoints1, keypoints2, matches12)
ax[0].axis('off')
plot_matches(ax[1], img1, img3, keypoints1, keypoints3, matches13)
ax[1].axis('off')
plt.show()
def main():
image_base_dir = '/home/dek/makerfaire-booth/2018/burger/experimental/dek/train_object_detector/decoded'
canonical_dir = 'canonical'
# template = os.path.join(image_base_dir, 'bottombun.0.00.27.34.-24.61.0.81.png')
template = os.path.join(canonical_dir, 'patty.png')
img1 = imread(template)
# img1_padded = numpy.zeros( (256, 256,3), dtype=numpy.uint8)
img1_padded = numpy.resize( [255,255,255], (256, 256, 3))
s = img1.shape
img1_padded[:s[0], :s[1]] = img1
img1_gray = rgb2gray(img1)
descriptor_extractor = ORB()
descriptor_extractor.detect_and_extract(img1_gray)
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
# g = glob.glob(os.path.join(image_base_dir, 'patty*.nobox.png'))
# for moving in g:
while True:
rot, tx, ty, scale = get_random_orientation()
# img2 = imread(moving)
img2 = draw_example('patty', 256, 256, rot, tx, ty, scale)
img2_gray = rgb2gray(img2)
try:
descriptor_extractor.detect_and_extract(img2_gray)
except RuntimeError:
continue
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
src = keypoints2[matches12[:, 1]][:, ::-1]
dst = keypoints1[matches12[:, 0]][:, ::-1]
model_robust, inliers = \
ransac((src, dst), SimilarityTransform,
min_samples=4, residual_threshold=2)
if not model_robust:
print "bad"
continue
img2_transformed = transform.warp(img2, model_robust.inverse, mode='constant', cval=1)
img1_padded_float = img1_padded.astype(numpy.float64)/255.
sub = img2_transformed - img1_padded_float
print compare_ssim(img2_transformed, img1_padded_float, win_size=5, multichannel=True)
fig, axes = plt.subplots(2, 2, figsize=(7, 6), sharex=True, sharey=True)
ax = axes.ravel()
ax[0].imshow(img1_padded_float)
ax[1].imshow(img2)
ax[1].set_title("Template image")
ax[2].imshow(img2_transformed)
ax[2].set_title("Matched image")
ax[3].imshow(sub)
ax[3].set_title("Subtracted image")
# plt.gray()
# ax = plt.gca()
# plot_matches(ax, img1, img2, keypoints1, keypoints2, matches12)
plt.show()
def find_all_matches(unmatched, run_num, min_to_match=40):
channel = 2
num_keypoints=1000
# get previous base
print("=====================================")
## setup detector
orb = ORB(n_keypoints=num_keypoints, downscale=1.2, n_scales=20, harris_k=.04,
fast_threshold=0.05)
base_num = 0
unmatched.sort()
init_num_unmatched = len(unmatched)
while len(unmatched):
print("Working on run_%05i" %run_num)
# get next unmatched image
base_path = unmatched.pop(0)
# read the image
base_img = imread(base_path)
# if the image has gps data, remove it for now
if base_img.shape[0] == 2:
base_img = base_img[0]
base_gray = base_img[:,:,channel]
ikdname = os.path.join(base_path.replace('.'+output_image_type, ''))
#base_k, base_d = detect_and_extract(orb, base_gray)
base_k, base_d = get_keypoints_and_descriptors(ikdname, base_gray, orb)
match_num = get_matches(base_path)
base_name = get_basename(run_num, base_num, match_num)
bad_match = 0
matched_files = [base_path]
print("New base file is: %s" %os.path.split(base_path)[1])
#print('new unmatched', unmatched)
for xx, img_path in enumerate(unmatched):
# get new keypoints for the updated base
img = imread(img_path)
if img.shape[0] == 2:
img = img[0]
img_name = os.path.split(img_path)[1]
print("working on img: %s" %img_name)
img_gray = img[:,:,channel]
ikdname = os.path.join(img_path.replace('.'+output_image_type, ''))
img_k, img_d = get_keypoints_and_descriptors(ikdname, img_gray, orb)
matches = match_descriptors(base_d, img_d, cross_check=True)
model_robust, ransac_matches = find_two_matches(base_gray,
img_gray,
base_k, img_k,
base_d, img_d)
num_ransac_matches = ransac_matches.shape[0]
if num_ransac_matches < min_to_match:
print("------------ could not match %s, only %s ransac_matches out of %s"
%(img_name, num_ransac_matches, matches.shape[0]))
# couldn't match with this base, save where we are and
# get new base
#fig, ax = plt.subplots(nrows=2, ncols=1)
#plt.title('run %s' %run_num)
#plot_matches(ax[0], base_gray, img_gray, base_k, img_k, matches)
#plot_matches(ax[1], base_gray, img_gray, base_k, img_k, ransac_matches)
#plt.show()
bad_match += 1
if bad_match > 1:
print("quiting this base img: %s" %base_path)
break
else:
match_num += 1
print("*********** matched %s with %s" %(img_name,
num_ransac_matches))
base_img = find_mask(base_name, base_img, img_name,
img, model_robust, channel)
base_gray = base_img[:,:,channel]
base_name = get_basename(run_num, base_num, match_num)
matched_files.append(img_path)
base_k, base_d = detect_and_extract(orb, base_gray)
base_out = os.path.join(outdir, base_name)
print("WRITING", base_out)
plt.imsave(base_out, base_img)
[os.remove(f) for f in matched_files]
unmatched = get_unmatched(run_num-1)
# remove original file
# increase to use next base_num
base_num += 1
def calculate_rate(it, jt):
mx = feature.match_descriptors(it, jt)
return float(len(mx) * 2) / (len(it) + len(jt))
def match(desc):
desc1, desc2 = desc[0], desc[1]
matches = match_descriptors(desc1, desc2, cross_check=True)
return matches
def main():
image_base_dir = '/home/dek/makerfaire-booth/2018/burger/experimental/dek/train_object_detector/decoded'
canonical_dir = 'canonical'
# template = os.path.join(image_base_dir, 'bottombun.0.00.27.34.-24.61.0.81.png')
fig, axes = plt.subplots(7, 7, figsize=(7, 6), sharex=True, sharey=True)
fig.delaxes(axes[0][0])
ssims = numpy.zeros( (len(BurgerElement.__members__), len(BurgerElement.__members__)), dtype=float)
mses = numpy.zeros( (len(BurgerElement.__members__), len(BurgerElement.__members__)), dtype=float)
for i, layer in enumerate(BurgerElement.__members__):
template = os.path.join(canonical_dir, '%s.png' % layer)
img1 = imread(template)
# img1_padded = numpy.zeros( (WIDTH, HEIGHT,3), dtype=numpy.uint8)
img1_padded = numpy.resize( [255,255,255], (WIDTH, HEIGHT, 3))
s = img1.shape
w = s[0]
h = s[1]
nb = img1_padded.shape[0]
na = img1.shape[0]
lower1 = (nb) // 2 - (na // 2)
upper1 = (nb // 2) + (na // 2)
nb = img1_padded.shape[1]
na = img1.shape[1]
lower2 = (nb) // 2 - (na // 2)
upper2 = (nb // 2) + (na // 2)
img1_padded[lower1:upper1, lower2:upper2] = img1
img1_padded_float = img1_padded.astype(numpy.float64)/255.
print img1_padded_float.shape
img1_gray = rgb2gray(img1_padded_float)
descriptor_extractor = ORB()
try:
descriptor_extractor.detect_and_extract(img1_gray)
except RuntimeError:
continue
keypoints1 = descriptor_extractor.keypoints
descriptors1 = descriptor_extractor.descriptors
axes[i][0].imshow(img1_padded_float)
axes[i][0].set_title("Template image")
for j, layer2 in enumerate(BurgerElement.__members__):
rot, tx, ty, scale = get_random_orientation()
img2 = draw_example(layer2, WIDTH, HEIGHT, rot, tx, ty, scale)
# match = os.path.join(canonical_dir, '%s.png' % layer2)
# img2 = imread(match)
img2_padded = numpy.resize( [255,255,255], (WIDTH, HEIGHT, 3))
s = img2.shape
img2_padded[:s[0], :s[1]] = img2
img2_padded_float = img2_padded.astype(numpy.float64)/255.
img2_gray = rgb2gray(img2_padded_float)
try:
descriptor_extractor.detect_and_extract(img2_gray)
except RuntimeError:
continue
keypoints2 = descriptor_extractor.keypoints
descriptors2 = descriptor_extractor.descriptors
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
src = keypoints2[matches12[:, 1]][:, ::-1]
dst = keypoints1[matches12[:, 0]][:, ::-1]
model_robust, inliers = \
ransac((src, dst), SimilarityTransform,
min_samples=4, residual_threshold=2)
if not model_robust:
print "bad"
continue
img2_transformed = transform.warp(img2_padded_float, model_robust.inverse, mode='constant', cval=1)
sub = img2_transformed - img1_padded_float
ssim = compare_ssim(img2_transformed, img1_padded_float, win_size=5, multichannel=True)
mse = compare_mse(img2_transformed, img1_padded_float)
ssims[i,j] = ssim
mses[i,j] = mse
axes[0][j].imshow(img2_padded_float)
axes[0][j].set_title("Match image")
axes[i][j].imshow(img2_transformed)
axes[i][j].set_title("Transformed image")
axes[i][j].set_xlabel("SSIM: %9.4f MSE: %9.4f" % (ssim, mse))
# ax = plt.gca()
# plot_matches(ax, img1, img2, keypoints1, keypoints2, matches12)
print ssims
print numpy.argmax(ssims, axis=1)
print numpy.argmin(mses, axis=1)
plt.show()
def match_descriptors(descriptor1, descriptor2):
matches12 = match_descriptors(descriptors1, descriptors2, cross_check=True)
return(matches12)
def match_template(im, template, *args, **kw):
'''see skimage.feature.match_template'''
return feature.match_descriptors(im, template, *args, **kw)
try:
#Retrieve Keypoint Features
keypoints_database = pickle.load( open("features/" + folder1 + "/orb/gray/" + file_names[i][:-5] + "_orb.pkl", "rb") )
kp, des = unpickle_keypoints(keypoints_database[0])
K.append(kp)
D.append(np.array(des))
print(len(D))
print(len(D[0]))
print(len(D[0][0]))
except IOError:
print("Couldn't load " + file_names[i] + " due to IOError.")
except TypeError:
print("Couldn't load " + file_names[i] + " due to TypeError.")
# print("Progress: " + str(i) + "/" + str(N))
# distance = np.zeros((len(K), len(K)))
for i in range(len(K)):
top1 = [0, 0]
for j in range(len(K)):
matches = match_descriptors(D[i], D[j])
if matches.shape[1] > top1[0]:
top1[0] = matches.shape[1]
top1[1] = j
# distance[i][j] = matches.shape[1]
print(top1)
top1 = [0, 0]
print("Keypoints: " + str(K[i][matches[:,0]]) + ', ' + str(K[j][matches[:,1]]))
print("Progress: " + str(i) + "/" + str(len(K)))
def match_descriptors(d1, d2, *args, **kw):
'''see skimage.feature.match_descriptors'''
return feature.match_descriptors(d1, d2, *args, **kw)
img_left, img_right, groundtruth_disp = data.stereo_motorcycle()
img_left, img_right = map(rgb2gray, (img_left, img_right))
# Find sparse feature correspondences between left and right image.
descriptor_extractor = ORB()
descriptor_extractor.detect_and_extract(img_left)
keypoints_left = descriptor_extractor.keypoints
descriptors_left = descriptor_extractor.descriptors
descriptor_extractor.detect_and_extract(img_right)
keypoints_right = descriptor_extractor.keypoints
descriptors_right = descriptor_extractor.descriptors
matches = match_descriptors(descriptors_left, descriptors_right,
cross_check=True)
# Estimate the epipolar geometry between the left and right image.
model, inliers = ransac((keypoints_left[matches[:, 0]],
keypoints_right[matches[:, 1]]),
FundamentalMatrixTransform, min_samples=8,
residual_threshold=1, max_trials=5000)
inlier_keypoints_left = keypoints_left[matches[inliers, 0]]
inlier_keypoints_right = keypoints_right[matches[inliers, 1]]
print("Number of matches:", matches.shape[0])
print("Number of inliers:", inliers.sum())
# Compare estimated sparse disparities to the dense ground-truth disparities.
def recognize(pattern, name, scene):
zipped_matches = []
match = match_descriptors(scene, pattern, cross_check=True, max_distance=0.5)
zipped_matches.append([match.size, name, match])
return zipped_matches
