def find_matching_points(image1, image2, n_levels = 3, distance_threshold = 300):
"""
:param image1 and image2 must be RGB images
:param n_levels: number of scales
:param distance_threshold: a threshold to accept a given match
:return: two numpy lists, each with keypoints in [x,y]
"""
# TODO
'''
Important note : you might need to change the parameters (sift parameters) inside this function to
have more or better matches
'''
matches_1 = []
matches_2 = []
image1 = np.array(image1.convert('L'))
image2 = np.array(image2.convert('L'))
'''
Each column of keypoints is a feature frame and has the format [X;Y;S;TH], where X,Y is the (fractional) center of
the frame, S is the scale and TH is the orientation (in radians).
AND each column of features is the descriptor of the corresponding frame in F.
A descriptor is a 128-dimensional vector of class UINT8
'''
keypoints_1, features_1 = sift.sift(image1, compute_descriptor = True, n_levels = n_levels)
keypoints_2, features_2 = sift.sift(image2, compute_descriptor = True, n_levels = n_levels)
pairwise_dist = cdist(features_1, features_2) # len(features_1) * len(features_2)
closest_1_to_2 = np.argmin(pairwise_dist, axis = 1)
for i, idx in enumerate(closest_1_to_2):
if pairwise_dist[i, idx] <= distance_threshold:
matches_1.append([keypoints_1[i][1], keypoints_1[i][0]])
matches_2.append([keypoints_2[idx][1], keypoints_2[idx][0]])
return np.array(matches_1), np.array(matches_2)
def get_SIFT_descriptor(image2D, **kwargs):
"""
Perform SIFT on a single image.
frames[i] -> (x,y)
descrs[i] -> (f1,f2,f3...f128)
Configure SIFT algorithm with followings:
dense: Apply d-SIFT (Default: True)
float: Descriptor are returned in floating point (Default: False)
fast: Fast approximation for d-SIFT (Default: False)
Returns np.array
"""
dense = kwargs.get('dense', True)
frames = descrs = None
if dense:
fast = kwargs.get('fast', False)
step = kwargs.get('step', 1)
# print("Using dsift with fast:", fast)
frames, descrs = dsift(image2D, step=step,
fast=fast) # Might be useful: verbose=False
else:
# print("Using regular sift")
frames, descrs = sift(
image2D, compute_descriptor=True) # Might be useful: verbose=False
# For debugging purposes
# if(np.any(np.isnan(descrs))):
# print(descrs)
# input("NaN detected, proceed?")
# if(np.any(np.isinf(descrs))):
# print(descrs)
# input("inf detected, proceed?")
return sample_descriptors(descrs)
def find_interest_points(self, img):
"""
Find interest points in greyscale image img
Inputs: img=greyscale input image (H, W, 1)
Outputs: ip=interest points of shape (2, N)
"""
# ip_fun = self.corner_function(img)
# row, col = self.find_local_maxima(ip_fun)
# ip = np.stack((row,col))
frames,desc=sift.sift(img,
compute_descriptor=True,
n_levels=1,
peak_thresh=0.1,
edge_thresh=10.0
)
ip=(frames.T)[0:2,:]
desc=desc.astype(np.float)
return ip
def main(image_list="images.dat"):
cwd = os.getcwd()
data_path = cwd + "/dataset/"
sift_path = cwd + "/sift_descriptor/"
dsift_path = cwd + "/dsift_descriptor/"
with open("images.dat", "r") as im:
images = im.readlines()
i = 1
total = len(images)
for image in images:
print(i, "/", total)
image_matrice = imread(data_path + image.strip(), mode='F')
sift_frame, sift_desc = sift(image_matrice, compute_descriptor=True)
dsift_frame, dsift_desc = dsift(image_matrice, step=10)
sift_image_path = sift_path + image.strip("\n")
dsift_image_path = dsift_path + image.strip("\n")
os.makedirs( os.path.dirname(sift_image_path), exist_ok=True)
os.makedirs( os.path.dirname(dsift_image_path), exist_ok=True)
np.save(sift_image_path, sift_desc)
np.save(dsift_image_path, dsift_desc)
i+=1
def computeFeatures(img):
# Reshape for PCA
copy = img.reshape(-1, 3).copy()
# PCA
mean, eigenvectors = cv2.PCACompute(copy, np.array([]), maxComponents=3)
pca = cv2.PCAProject(copy, mean, eigenvectors)
pca = pca.reshape(img.shape)
# Extract first component
pca = pca[:, :, 0]
pca = (pca - np.min(pca)) / (np.max(pca) - np.min(pca)) * 255
pca = cv2.convertScaleAbs(pca)
# Truncate all sides by 5% since they tend to be background
pca = pca[round(pca.shape[0] * 0.05):round(pca.shape[0] * 0.95),
round(pca.shape[1] * 0.05):round(pca.shape[1] * 0.95)]
# Extract SIFT features and descriptors
(frame, desc) = cysift.sift(pca,
peak_thresh=10,
edge_thresh=20,
compute_descriptor=True)
# CLAHE
clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8, 8))
pca = clahe.apply(pca)
# Load codebook
codebook = pickle.load(open("codebook.pkl", "rb"))
# Construct visual word histogram
code, distortion = vq(desc, codebook)
featvect, bins = np.histogram(code, codebook.shape[0], normed=True)
return featvect
for idx, imgPath in enumerate(sorted(imgList)):
print(idx, imgPath)
fileName = os.path.basename(imgPath)
img = Image.open(imgPath, 'r')
if (max(img.size) > 1000 or len(imgList) > 10):
img.thumbnail((np.asarray(img.size) * RESIZE).astype('int'),
Image.ANTIALIAS)
Images.update({idx: img})
print('Images loaded. Beginning feature detection...')
#%% Feature detection
Descriptor = {}
PointInImg = {}
for idx, (key, img) in enumerate(sorted(Images.items())):
I = np.asarray(img.convert('L')).astype('single')
[f, d] = sift(I, compute_descriptor=True, float_descriptors=True)
pointsInImage = swapcolumn(f[:, 0:2])
PointInImg.update({idx: pointsInImage})
Descriptor.update({idx: d})
#%% Compute Transformation
Transform = {}
for idx in range(len(imgList) - 1):
print('fitting transformation from ' + str(idx) + ' to ' + str(idx + 1) +
'\t')
M = SIFTSimpleMatcher(Descriptor[idx], Descriptor[idx + 1], Thre)
print('matching points:', len(M, ), '\n')
Transform.update({idx: RANSACFit(PointInImg[idx], PointInImg[idx + 1], M)})
#%% Make Panoramic image
print('Stitching images...')
def sift_test(img):
img = np.asarray(img.convert('L')).astype('single')
f, descriptor = sift(img, compute_descriptor=True, float_descriptors=True)
return swapcolumn(f[:, 0:2]), descriptor
testfeatures = []
with open(files[5], 'rb') as fo:
dict = pickle.load(fo, encoding='bytes')
fo.close()
for j in dict[b'labels']:
testlabels.append(j)
for j in dict[b'data']:
testfeatures.append(j)
testfeatures = numpy.array(testfeatures)
testlabels = numpy.array(testlabels)
trainvectors = []
for i in trainfeatures:
a=numpy.array(Image.fromarray(i.reshape((32,32,3),order='F'))\
.transpose(Image.TRANSPOSE).convert('L'))
trainvectors.append(sift.sift(a, compute_descriptor='True')[1])
bag = []
for i in trainvectors:
for j in i:
bag.append(j)
bag = numpy.array(bag)
bag = bag.astype(numpy.float32)
num_of_words = 8
words = numpy.array(kmeans.kmeans(bag, num_centers=num_of_words))
trainwords = []
testvectors = []
for i in testfeatures:
a=numpy.array(Image.fromarray(i.reshape((32,32,3),order='F'))\
.transpose(Image.TRANSPOSE).convert('L'))
def get_descriptors(self, img, ip):
"""
Extact descriptors from grayscale image img at interest points ip
Inputs: img=grayscale input image (H, W, 1)
ip=interest point coordinates (2, N)
Returns: descriptors=vectorized descriptors (N, num_dims)
"""
# patch_size=self.params['patch_size']
# patch_size_div2=int(patch_size/2)
# num_dims=patch_size**2
#
# print('Patch size: {} Num_dims: {} , ip {}, img {}'.format(patch_size, num_dims, ip.shape, img.shape))
#
# H,W,_=img.shape
# num_ip=ip.shape[1]
#
# sample_spacing = 2
# if sample_spacing > 1:
# # We want to keep the size of the image as a square, since the plotting method squares them.
# num_dims = int(np.sqrt(patch_size**2 / sample_spacing))**2
# else:
# num_dims = patch_size**2
# descriptors=np.zeros((num_ip,num_dims))
#
#
# for i in range(num_ip):
# row=ip[0,i]
# col=ip[1,i]
#
# # FORNOW: random image patch
# # patch=np.random.randn(patch_size,patch_size)
#
# """
# ******************************************************
# *** TODO: write code to extract descriptor at row, col
# ******************************************************
# """
# row_start = 0 if row-patch_size_div2 < 0 else row-patch_size_div2
# row_end = H if row+patch_size_div2+1 > H else row+patch_size_div2+1
# col_start = 0 if col-patch_size_div2 < 0 else col-patch_size_div2
# col_end = W if col+patch_size_div2+1 > W else col+patch_size_div2+1
# patch = img[row_start:row_end, col_start:col_end ]
#
# """
# ******************************************************
# """
# if sample_spacing > 1:
# patch = np.ndarray.flatten(patch)
# patch = patch[::sample_spacing]
# patch = patch[:num_dims]
# descriptors[i, :]= patch
# else:
# descriptors[i, :]=np.reshape(patch,num_dims)
#
# # normalise descriptors to 0 mean, unit length
# mn=np.mean(descriptors,1,keepdims=True)
# sd=np.std(descriptors,1,keepdims=True)
# small_val = 1e-6
# descriptors = (descriptors-mn)/(sd+small_val)
#
# return descriptors
frames,desc=sift.sift(img,
compute_descriptor=True,
n_levels=1,
peak_thresh=0.1,
edge_thresh=10.0
)
ip=(frames.T)[0:2,:]
desc=desc.astype(np.float)
return desc
def extract_image_features(image):
_, descriptors = cysift.sift(img_gray,
peak_thresh=8,
edge_thresh=5,
compute_descriptor=True)
return descriptors
def extract_sift_features(img,
peak_thresh=0.9,
edge_thresh=30,
boundary_pct=0.05,
scale=0.26,
num_keypoints=200):
"""
Extracts key points and their SIFT feature representations.
Finds key points in the images and save them as frames ``f``, then compute
the SIFT descriptor for these key points and save them as descriptors ``d``
. Finally, compute the number of key points in the image and save it as
``num_features``.
Parameters
----------
img: Image
An image to be analyzed
step_size: int
Steps for cyvlfeat dsift function
boundary_pct: float
Percentage of image to be seen as boundary
scale: float
Scale of rescaling (used to reduce computation)
Returns
-------
num_features: int
Number of feature points in the image
fd: Set(f, d)
- **f** (numpy.ndarray[float]) - Frames (key points) of the result
- **d** (numpy.ndarray[uint8]) - Descriptor of corresponding frames
"""
# make sure image is grayscale
img = color.rgb2gray(img)
# downscale image to extract less features
img = rescale(img,
scale=scale,
anti_aliasing=True,
multichannel=False,
mode='reflect')
img_h, img_w = img.shape[0], img.shape[1]
f, d = sift(img,
peak_thresh=peak_thresh,
edge_thresh=edge_thresh,
compute_descriptor=True)
# remove features near boundary
if boundary_pct > 0:
in_boundary = ((f[:, 1] > (img_w * boundary_pct)) *
(f[:, 1] < (img_w * (1 - boundary_pct))) *
(f[:, 0] > (img_h * boundary_pct)) *
(f[:, 0] < (img_h * (1 - boundary_pct))))
f = f[in_boundary]
d = d[in_boundary]
num_keypoints = min(num_keypoints, f.shape[0])
keep_idx = np.random.permutation(f.shape[0])[:num_keypoints]
f = f[keep_idx]
d = d[keep_idx]
assert len(f) == len(d)
num_features = len(f)
return num_features, (f, d)
def im_stitch(image_left, image_right, patch_size, k, ransac_iters,
ransac_threshold, ransac_num_inliers):
# # HARRIS
# coords_1, coord_subpix_1 = harris_detector(image_left)
# coords_2, coord_subpix_2 = harris_detector(image_right)
# # DESCRIBE KEYPOINTS
# descriptors_1 = patch_descriptor(image_left, coords_1, patch_size)
# descriptors_2 = patch_descriptor(image_right, coords_2, patch_size)
# print('patch_size =', patch_size, ', descriptor size =', patch_size[0] * patch_size[1])
# print('#keypoints img1 =', descriptors_1.shape[0])
# print('#keypoints img2 =', descriptors_2.shape[0])
# print('#potential matches =', descriptors_1.shape[0] * descriptors_2.shape[0])
keypoints_1, descriptors_1 = sift.sift(rgb2grey(image_left),
compute_descriptor=True)
keypoints_2, descriptors_2 = sift.sift(rgb2grey(image_right),
compute_descriptor=True)
coords_1 = keypoints_1[:, :2]
coords_2 = keypoints_2[:, :2]
# _, (ax1, ax2) = plt.subplots(1, 2)
# ax1.imshow(image_left, cmap='gray')
# ax1.plot(keypoints_1[:, 1], keypoints_1[:, 0], 'ro')
# ax2.imshow(image_right, cmap='gray')
# ax2.plot(keypoints_2[:, 1], keypoints_2[:, 0], 'ro')
# plt.show()
# return 0
# CALCULATE DISTS AND CHOOSE K BEST MATCHES
# the ones with closest dist between descriptors
if k > np.min([descriptors_1.shape[0], descriptors_2.shape[0]]):
k = np.min([descriptors_1.shape[0], descriptors_2.shape[0]])
top_matched_points = choose_matching_points(descriptors_1, coords_1,
coords_2, descriptors_2, k)
print(k, 'best matches chosen')
# add x_max to x' to get proper distances in transformation
img1_x_max = image_left.shape[1]
top_matched_points[:, 5] += img1_x_max
# ESTIMATE AFFINE TRANSFORM USING RANSAC
minrq = 3
iters = ransac_iters
thresh = ransac_threshold
mininl = ransac_num_inliers
tform_mat, inliers = estimate_tranform_ransac(top_matched_points, minrq,
iters, thresh, mininl)
print('ransac: minreq, iters, threshold, mininl =', minrq, iters, thresh,
mininl)
print('affine transform matrix:\n', tform_mat)
print('num inliers =', inliers.shape[0], 'out of', k)
# CALCULATE ERRORS OF INLIERS
errors = calculate_errors(inliers, tform_mat)
print('mean errors = ', np.mean(errors))
print('std errors =', np.std(errors))
# To plot matching points, reverse the addition of x_max to x'
inliers[:, 2] -= img1_x_max
sorted_inliers_by_err = inliers[np.argsort(errors)]
matches_to_show = 100
if matches_to_show > inliers.shape[0]:
matches_to_show = inliers.shape[0]
plot_matching_points(image_left, image_right, sorted_inliers_by_err,
matches_to_show)
stitched_image = stitch_images(image_left, image_right, tform_mat)
return stitched_image
# Extract first component
pca = pca[:, :, 0]
pca = (pca - np.min(pca)) / (np.max(pca) - np.min(pca)) * 255
pca = cv2.convertScaleAbs(pca)
pca = pca[round(pca.shape[0] * 0.05):round(pca.shape[0] * 0.95),
round(pca.shape[1] * 0.05):round(pca.shape[1] * 0.95)]
# CLAHE
clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8, 8))
pca = clahe.apply(pca)
# Extract SIFT features and descriptors
(frame, descriptor) = cysift.sift(pca,
peak_thresh=5,
edge_thresh=8,
compute_descriptor=True)
# Append descriptors to global descriptors
feat.append((frame, descriptor))
# Vertical stack descriptors
descriptors = [item[1] for item in feat]
descriptors = np.vstack(descriptors)
# Clean up
del temp
del img
del copy
del mean
del eigenvectors
