def find_matching_points(image1, image2, n_levels = 3, distance_threshold = 300): """ :param image1 and image2 must be RGB images :param n_levels: number of scales :param distance_threshold: a threshold to accept a given match :return: two numpy lists, each with keypoints in [x,y] """ # TODO ''' Important note : you might need to change the parameters (sift parameters) inside this function to have more or better matches ''' matches_1 = [] matches_2 = [] image1 = np.array(image1.convert('L')) image2 = np.array(image2.convert('L')) ''' Each column of keypoints is a feature frame and has the format [X;Y;S;TH], where X,Y is the (fractional) center of the frame, S is the scale and TH is the orientation (in radians). AND each column of features is the descriptor of the corresponding frame in F. A descriptor is a 128-dimensional vector of class UINT8 ''' keypoints_1, features_1 = sift.sift(image1, compute_descriptor = True, n_levels = n_levels) keypoints_2, features_2 = sift.sift(image2, compute_descriptor = True, n_levels = n_levels) pairwise_dist = cdist(features_1, features_2) # len(features_1) * len(features_2) closest_1_to_2 = np.argmin(pairwise_dist, axis = 1) for i, idx in enumerate(closest_1_to_2): if pairwise_dist[i, idx] <= distance_threshold: matches_1.append([keypoints_1[i][1], keypoints_1[i][0]]) matches_2.append([keypoints_2[idx][1], keypoints_2[idx][0]]) return np.array(matches_1), np.array(matches_2)
def get_SIFT_descriptor(image2D, **kwargs): """ Perform SIFT on a single image. frames[i] -> (x,y) descrs[i] -> (f1,f2,f3...f128) Configure SIFT algorithm with followings: dense: Apply d-SIFT (Default: True) float: Descriptor are returned in floating point (Default: False) fast: Fast approximation for d-SIFT (Default: False) Returns np.array """ dense = kwargs.get('dense', True) frames = descrs = None if dense: fast = kwargs.get('fast', False) step = kwargs.get('step', 1) # print("Using dsift with fast:", fast) frames, descrs = dsift(image2D, step=step, fast=fast) # Might be useful: verbose=False else: # print("Using regular sift") frames, descrs = sift( image2D, compute_descriptor=True) # Might be useful: verbose=False # For debugging purposes # if(np.any(np.isnan(descrs))): # print(descrs) # input("NaN detected, proceed?") # if(np.any(np.isinf(descrs))): # print(descrs) # input("inf detected, proceed?") return sample_descriptors(descrs)
def find_interest_points(self, img): """ Find interest points in greyscale image img Inputs: img=greyscale input image (H, W, 1) Outputs: ip=interest points of shape (2, N) """ # ip_fun = self.corner_function(img) # row, col = self.find_local_maxima(ip_fun) # ip = np.stack((row,col)) frames,desc=sift.sift(img, compute_descriptor=True, n_levels=1, peak_thresh=0.1, edge_thresh=10.0 ) ip=(frames.T)[0:2,:] desc=desc.astype(np.float) return ip
def main(image_list="images.dat"): cwd = os.getcwd() data_path = cwd + "/dataset/" sift_path = cwd + "/sift_descriptor/" dsift_path = cwd + "/dsift_descriptor/" with open("images.dat", "r") as im: images = im.readlines() i = 1 total = len(images) for image in images: print(i, "/", total) image_matrice = imread(data_path + image.strip(), mode='F') sift_frame, sift_desc = sift(image_matrice, compute_descriptor=True) dsift_frame, dsift_desc = dsift(image_matrice, step=10) sift_image_path = sift_path + image.strip("\n") dsift_image_path = dsift_path + image.strip("\n") os.makedirs( os.path.dirname(sift_image_path), exist_ok=True) os.makedirs( os.path.dirname(dsift_image_path), exist_ok=True) np.save(sift_image_path, sift_desc) np.save(dsift_image_path, dsift_desc) i+=1
def computeFeatures(img): # Reshape for PCA copy = img.reshape(-1, 3).copy() # PCA mean, eigenvectors = cv2.PCACompute(copy, np.array([]), maxComponents=3) pca = cv2.PCAProject(copy, mean, eigenvectors) pca = pca.reshape(img.shape) # Extract first component pca = pca[:, :, 0] pca = (pca - np.min(pca)) / (np.max(pca) - np.min(pca)) * 255 pca = cv2.convertScaleAbs(pca) # Truncate all sides by 5% since they tend to be background pca = pca[round(pca.shape[0] * 0.05):round(pca.shape[0] * 0.95), round(pca.shape[1] * 0.05):round(pca.shape[1] * 0.95)] # Extract SIFT features and descriptors (frame, desc) = cysift.sift(pca, peak_thresh=10, edge_thresh=20, compute_descriptor=True) # CLAHE clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8, 8)) pca = clahe.apply(pca) # Load codebook codebook = pickle.load(open("codebook.pkl", "rb")) # Construct visual word histogram code, distortion = vq(desc, codebook) featvect, bins = np.histogram(code, codebook.shape[0], normed=True) return featvect
for idx, imgPath in enumerate(sorted(imgList)): print(idx, imgPath) fileName = os.path.basename(imgPath) img = Image.open(imgPath, 'r') if (max(img.size) > 1000 or len(imgList) > 10): img.thumbnail((np.asarray(img.size) * RESIZE).astype('int'), Image.ANTIALIAS) Images.update({idx: img}) print('Images loaded. Beginning feature detection...') #%% Feature detection Descriptor = {} PointInImg = {} for idx, (key, img) in enumerate(sorted(Images.items())): I = np.asarray(img.convert('L')).astype('single') [f, d] = sift(I, compute_descriptor=True, float_descriptors=True) pointsInImage = swapcolumn(f[:, 0:2]) PointInImg.update({idx: pointsInImage}) Descriptor.update({idx: d}) #%% Compute Transformation Transform = {} for idx in range(len(imgList) - 1): print('fitting transformation from ' + str(idx) + ' to ' + str(idx + 1) + '\t') M = SIFTSimpleMatcher(Descriptor[idx], Descriptor[idx + 1], Thre) print('matching points:', len(M, ), '\n') Transform.update({idx: RANSACFit(PointInImg[idx], PointInImg[idx + 1], M)}) #%% Make Panoramic image print('Stitching images...')
def sift_test(img): img = np.asarray(img.convert('L')).astype('single') f, descriptor = sift(img, compute_descriptor=True, float_descriptors=True) return swapcolumn(f[:, 0:2]), descriptor
testfeatures = [] with open(files[5], 'rb') as fo: dict = pickle.load(fo, encoding='bytes') fo.close() for j in dict[b'labels']: testlabels.append(j) for j in dict[b'data']: testfeatures.append(j) testfeatures = numpy.array(testfeatures) testlabels = numpy.array(testlabels) trainvectors = [] for i in trainfeatures: a=numpy.array(Image.fromarray(i.reshape((32,32,3),order='F'))\ .transpose(Image.TRANSPOSE).convert('L')) trainvectors.append(sift.sift(a, compute_descriptor='True')[1]) bag = [] for i in trainvectors: for j in i: bag.append(j) bag = numpy.array(bag) bag = bag.astype(numpy.float32) num_of_words = 8 words = numpy.array(kmeans.kmeans(bag, num_centers=num_of_words)) trainwords = [] testvectors = [] for i in testfeatures: a=numpy.array(Image.fromarray(i.reshape((32,32,3),order='F'))\ .transpose(Image.TRANSPOSE).convert('L'))
def get_descriptors(self, img, ip): """ Extact descriptors from grayscale image img at interest points ip Inputs: img=grayscale input image (H, W, 1) ip=interest point coordinates (2, N) Returns: descriptors=vectorized descriptors (N, num_dims) """ # patch_size=self.params['patch_size'] # patch_size_div2=int(patch_size/2) # num_dims=patch_size**2 # # print('Patch size: {} Num_dims: {} , ip {}, img {}'.format(patch_size, num_dims, ip.shape, img.shape)) # # H,W,_=img.shape # num_ip=ip.shape[1] # # sample_spacing = 2 # if sample_spacing > 1: # # We want to keep the size of the image as a square, since the plotting method squares them. # num_dims = int(np.sqrt(patch_size**2 / sample_spacing))**2 # else: # num_dims = patch_size**2 # descriptors=np.zeros((num_ip,num_dims)) # # # for i in range(num_ip): # row=ip[0,i] # col=ip[1,i] # # # FORNOW: random image patch # # patch=np.random.randn(patch_size,patch_size) # # """ # ****************************************************** # *** TODO: write code to extract descriptor at row, col # ****************************************************** # """ # row_start = 0 if row-patch_size_div2 < 0 else row-patch_size_div2 # row_end = H if row+patch_size_div2+1 > H else row+patch_size_div2+1 # col_start = 0 if col-patch_size_div2 < 0 else col-patch_size_div2 # col_end = W if col+patch_size_div2+1 > W else col+patch_size_div2+1 # patch = img[row_start:row_end, col_start:col_end ] # # """ # ****************************************************** # """ # if sample_spacing > 1: # patch = np.ndarray.flatten(patch) # patch = patch[::sample_spacing] # patch = patch[:num_dims] # descriptors[i, :]= patch # else: # descriptors[i, :]=np.reshape(patch,num_dims) # # # normalise descriptors to 0 mean, unit length # mn=np.mean(descriptors,1,keepdims=True) # sd=np.std(descriptors,1,keepdims=True) # small_val = 1e-6 # descriptors = (descriptors-mn)/(sd+small_val) # # return descriptors frames,desc=sift.sift(img, compute_descriptor=True, n_levels=1, peak_thresh=0.1, edge_thresh=10.0 ) ip=(frames.T)[0:2,:] desc=desc.astype(np.float) return desc
def extract_image_features(image): _, descriptors = cysift.sift(img_gray, peak_thresh=8, edge_thresh=5, compute_descriptor=True) return descriptors
def extract_sift_features(img, peak_thresh=0.9, edge_thresh=30, boundary_pct=0.05, scale=0.26, num_keypoints=200): """ Extracts key points and their SIFT feature representations. Finds key points in the images and save them as frames ``f``, then compute the SIFT descriptor for these key points and save them as descriptors ``d`` . Finally, compute the number of key points in the image and save it as ``num_features``. Parameters ---------- img: Image An image to be analyzed step_size: int Steps for cyvlfeat dsift function boundary_pct: float Percentage of image to be seen as boundary scale: float Scale of rescaling (used to reduce computation) Returns ------- num_features: int Number of feature points in the image fd: Set(f, d) - **f** (numpy.ndarray[float]) - Frames (key points) of the result - **d** (numpy.ndarray[uint8]) - Descriptor of corresponding frames """ # make sure image is grayscale img = color.rgb2gray(img) # downscale image to extract less features img = rescale(img, scale=scale, anti_aliasing=True, multichannel=False, mode='reflect') img_h, img_w = img.shape[0], img.shape[1] f, d = sift(img, peak_thresh=peak_thresh, edge_thresh=edge_thresh, compute_descriptor=True) # remove features near boundary if boundary_pct > 0: in_boundary = ((f[:, 1] > (img_w * boundary_pct)) * (f[:, 1] < (img_w * (1 - boundary_pct))) * (f[:, 0] > (img_h * boundary_pct)) * (f[:, 0] < (img_h * (1 - boundary_pct)))) f = f[in_boundary] d = d[in_boundary] num_keypoints = min(num_keypoints, f.shape[0]) keep_idx = np.random.permutation(f.shape[0])[:num_keypoints] f = f[keep_idx] d = d[keep_idx] assert len(f) == len(d) num_features = len(f) return num_features, (f, d)
def im_stitch(image_left, image_right, patch_size, k, ransac_iters, ransac_threshold, ransac_num_inliers): # # HARRIS # coords_1, coord_subpix_1 = harris_detector(image_left) # coords_2, coord_subpix_2 = harris_detector(image_right) # # DESCRIBE KEYPOINTS # descriptors_1 = patch_descriptor(image_left, coords_1, patch_size) # descriptors_2 = patch_descriptor(image_right, coords_2, patch_size) # print('patch_size =', patch_size, ', descriptor size =', patch_size[0] * patch_size[1]) # print('#keypoints img1 =', descriptors_1.shape[0]) # print('#keypoints img2 =', descriptors_2.shape[0]) # print('#potential matches =', descriptors_1.shape[0] * descriptors_2.shape[0]) keypoints_1, descriptors_1 = sift.sift(rgb2grey(image_left), compute_descriptor=True) keypoints_2, descriptors_2 = sift.sift(rgb2grey(image_right), compute_descriptor=True) coords_1 = keypoints_1[:, :2] coords_2 = keypoints_2[:, :2] # _, (ax1, ax2) = plt.subplots(1, 2) # ax1.imshow(image_left, cmap='gray') # ax1.plot(keypoints_1[:, 1], keypoints_1[:, 0], 'ro') # ax2.imshow(image_right, cmap='gray') # ax2.plot(keypoints_2[:, 1], keypoints_2[:, 0], 'ro') # plt.show() # return 0 # CALCULATE DISTS AND CHOOSE K BEST MATCHES # the ones with closest dist between descriptors if k > np.min([descriptors_1.shape[0], descriptors_2.shape[0]]): k = np.min([descriptors_1.shape[0], descriptors_2.shape[0]]) top_matched_points = choose_matching_points(descriptors_1, coords_1, coords_2, descriptors_2, k) print(k, 'best matches chosen') # add x_max to x' to get proper distances in transformation img1_x_max = image_left.shape[1] top_matched_points[:, 5] += img1_x_max # ESTIMATE AFFINE TRANSFORM USING RANSAC minrq = 3 iters = ransac_iters thresh = ransac_threshold mininl = ransac_num_inliers tform_mat, inliers = estimate_tranform_ransac(top_matched_points, minrq, iters, thresh, mininl) print('ransac: minreq, iters, threshold, mininl =', minrq, iters, thresh, mininl) print('affine transform matrix:\n', tform_mat) print('num inliers =', inliers.shape[0], 'out of', k) # CALCULATE ERRORS OF INLIERS errors = calculate_errors(inliers, tform_mat) print('mean errors = ', np.mean(errors)) print('std errors =', np.std(errors)) # To plot matching points, reverse the addition of x_max to x' inliers[:, 2] -= img1_x_max sorted_inliers_by_err = inliers[np.argsort(errors)] matches_to_show = 100 if matches_to_show > inliers.shape[0]: matches_to_show = inliers.shape[0] plot_matching_points(image_left, image_right, sorted_inliers_by_err, matches_to_show) stitched_image = stitch_images(image_left, image_right, tform_mat) return stitched_image
# Extract first component pca = pca[:, :, 0] pca = (pca - np.min(pca)) / (np.max(pca) - np.min(pca)) * 255 pca = cv2.convertScaleAbs(pca) pca = pca[round(pca.shape[0] * 0.05):round(pca.shape[0] * 0.95), round(pca.shape[1] * 0.05):round(pca.shape[1] * 0.95)] # CLAHE clahe = cv2.createCLAHE(clipLimit=4.0, tileGridSize=(8, 8)) pca = clahe.apply(pca) # Extract SIFT features and descriptors (frame, descriptor) = cysift.sift(pca, peak_thresh=5, edge_thresh=8, compute_descriptor=True) # Append descriptors to global descriptors feat.append((frame, descriptor)) # Vertical stack descriptors descriptors = [item[1] for item in feat] descriptors = np.vstack(descriptors) # Clean up del temp del img del copy del mean del eigenvectors