def get_points(c_img1, c_img2): # convert to gray img1 = cvtColor(c_img1, COLOR_BGR2GRAY) img2 = cvtColor(c_img2, COLOR_BGR2GRAY) surf = SURF() # Initiate SURF detector # find the key points and descriptors with SURF kp1, des1 = surf.detectAndCompute(img1, None) kp2, des2 = surf.detectAndCompute(img2, None) my_flan_index_tree = 0 index_params = dict(algorithm=my_flan_index_tree, trees=6) search_params = dict(checks=50) my_flan = FlannBasedMatcher(index_params, search_params) matches = my_flan.knnMatch(des1, des2, k=2) # store all the good matches as per Lowe's ratio test. good = [] pts2 = [] pts1 = [] for m, n in matches: if m.distance < 0.9 * n.distance: good.append(m) pts1 = float32([kp1[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) pts2 = float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) # get color of key points lengths = len(pts1) - 1 color1 = zeros((len(pts1), 1, 3)) color2 = zeros((len(pts1), 1, 3)) color = zeros((len(pts1), 1, 3), dtype=int) for i in range(1, lengths): color1[i] = c_img1[int(pts1[i][0][1]), int(pts1[i][0][0])] color2[i] = c_img2[int(pts2[i][0][1]), int(pts2[i][0][0])] color[i] = (color1[i] + color2[i]) / 2 # avg of colors # convert the 2D features into homogeneous coordinates into array of 3x51 dimension pt1 = pts1.reshape((pts1.shape[0], 2)).T pt1 = vstack((pt1, ones(pt1.shape[1]))) pt2 = pts2.reshape((pts2.shape[0], 2)).T pt2 = vstack((pt2, ones(pt2.shape[1]))) return pt1, pt2, color
def fit_cv2(data, algorithm): logger.info('Fitting cv2 FLANN...') from cv2 import FlannBasedMatcher KDTREE = 0 index_params = { 'algorithm': KDTREE, 'trees': 5, #'target_precision': 0.9, #'build_weight': 0.01, #'memory_weight': 0, #'sample_fraction': 0.1, } search_params = {'checks': 5} flann = FlannBasedMatcher(index_params, search_params) flann.add(np.float32(data)) flann.train() return flann
def match_descriptors(descriptors1, descriptors2, matcher='flann', max_ratio=0.8): """ Args: descriptors1: descriptors2: matcher: max_ratio: Returns: [1] https://stackoverflow.com/questions/30716610/how-to-get-pixel-coordinates-from-feature-matching-in-opencv-python """ if matcher is 'flann': # FLANN parameters FLANN_INDEX_KDTREE = 0 index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) search_params = dict(checks=50) # or pass empty dictionary flann = FlannBasedMatcher(index_params, search_params) matches = flann.knnMatch(descriptors1, descriptors2, k=2) # Need to draw only good matches, so create a mask using ratio test as per Lowe's paper. good_matches = [] for i, (m, n) in enumerate(matches): if m.distance < max_ratio * n.distance: good_matches.append([n.queryIdx, m.trainIdx ]) # Keep indexes of matched keypoints good_matches = array(good_matches) return good_matches
def __init__(self): # Initiate ORB detectors self.orb = cv2.ORB_create() # If we want to filter ourselves the maches from bfmatches, crosscheck = # false self.bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False) self.Flann_index_lsh = 6 self.index_params = dict(algorithm=self.Flann_index_lsh, table_number=6, key_size=12, multi_probe_level=1) self.search_params = dict(checks=50) self.flann_matcher = FlannBasedMatcher(self.index_params, self.search_params) self.kp1 = KeyPoint() self.kp2 = KeyPoint() self.desc1 = None self.desc2 = None self.matches = None self.ratio = 0.65 self.matches1 = None self.matches2 = None self.good_matches = None self.good_kp1 = [] self.good_kp2 = [] self.n_matches = 0 # The following variables are used to work with numpy functions self.global_matches = [] # Numpy array self.global_kpts1 = [] self.global_kpts2 = [] # Create lists where we store the keypoints in their original format for # future uses. Also, store the descriptors self.curr_kp = [] # List of keypoints self.prev_kp = [] self.curr_dsc = [] # List of descriptors self.prev_dsc = [] self.is_minkp = None self.is_minmatches = None self.lk_params = dict( winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03) )
def algorithm_SURF(self, photo, screen, screen_colored, hessianThreshold=3500, descMatcher=1): t1 = time.perf_counter() # Init algorithm surf = SURF_create(hessianThreshold) surf.setUpright(True) t2 = time.perf_counter() self.writeLog('Created SURF object - {}ms'.format( self.formatTimeDiff(t1, t2))) # Detect and compute kp_photo, des_photo = surf.detectAndCompute(photo, None) kp_screen, des_screen = surf.detectAndCompute(screen, None) t3 = time.perf_counter() self.writeLog('Detected keypoints - {}ms'.format( self.formatTimeDiff(t2, t3))) # Descriptor Matcher try: index_params = dict(algorithm=descMatcher, trees=5) search_params = dict(checks=50) flann = FlannBasedMatcher(index_params, search_params) except: return False t4 = time.perf_counter() self.writeLog('Initialized Flann Matcher - {}ms'.format( self.formatTimeDiff(t3, t4))) # Calc knn Matches try: matches = flann.knnMatch(des_photo, des_screen, k=2) except: return False logging.info('knn {}'.format(len(matches))) t5 = time.perf_counter() self.writeLog('Calced knn matches - {}ms'.format( self.formatTimeDiff(t4, t5))) if not matches or len(matches) == 0: return False # store all the good matches as per Lowe's ratio test. good = [] for m, n in matches: if m.distance < 0.75 * n.distance: good.append(m) logging.info('good {}'.format(len(good))) t6 = time.perf_counter() self.writeLog('Filtered good matches - {}ms'.format( self.formatTimeDiff(t5, t6))) if not good or len(good) < 10: return False photo_pts = np.float32([kp_photo[m.queryIdx].pt for m in good]).reshape(-1, 1, 2) # pylint: disable=too-many-function-args screen_pts = np.float32([kp_screen[m.trainIdx].pt for m in good]).reshape(-1, 1, 2) # pylint: disable=too-many-function-args M, _ = findHomography(photo_pts, screen_pts, RANSAC, 5.0) t7 = time.perf_counter() self.writeLog('Found Homography - {}ms'.format( self.formatTimeDiff(t6, t7))) if M is None or not M.any() or len(M) == 0: return False h, w = photo.shape pts = np.float32([[0, 0], [0, h - 1], [w - 1, h - 1], [w - 1, 0]]).reshape(-1, 1, 2) # pylint: disable=too-many-function-args dst = perspectiveTransform(pts, M) t8 = time.perf_counter() self.writeLog('Perspective Transform - {}ms'.format( self.formatTimeDiff(t7, t8))) minX = dst[0][0][0] minY = dst[0][0][1] maxX = dst[0][0][0] maxY = dst[0][0][1] for i in range(4): if dst[i][0][0] < minX: minX = dst[i][0][0] if dst[i][0][0] > maxX: maxX = dst[i][0][0] if dst[i][0][1] < minY: minY = dst[i][0][1] if dst[i][0][1] > maxY: maxY = dst[i][0][1] minX = int(minX) minY = int(minY) maxX = int(maxX) maxY = int(maxY) if minX < 0: minX = 0 if minY < 0: minY = 0 logging.info('minY {}'.format(int(minY))) logging.info('minX {}'.format(int(minX))) logging.info('maxY {}'.format(int(maxY))) logging.info('maxX {}'.format(int(maxX))) if maxX - minX <= 0: return False if maxY - minY <= 0: return False imwrite(self.match_dir + '/result.png', screen_colored[minY:maxY, minX:maxX]) t9 = time.perf_counter() self.writeLog('Wrote Image - {}ms'.format(self.formatTimeDiff(t8, t9))) return True
def getmarker(self, image: object): def findmacth(image_file): print('\nprocessing', str(image_file)) start = timer() desc_1 = get_points(image_file, "", size, brisk) if desc_1 is not None: titles = [] similarity = [] titles_append = titles.append similarity_append = similarity.append for title, desc_2, len_desc_2 in root: good_points = 0 matches = flann.knnMatch(desc_1, desc_2, k=2) for m_n in matches: if len(m_n) != 2: continue elif m_n[0].distance < self.threshold * m_n[1].distance: good_points += 1 percentage_similarity = good_points / len_desc_2 * 100 if percentage_similarity > 2: titles_append(title) similarity_append(percentage_similarity) if similarity: idx = argmax(similarity) # for idx1, t in enumerate(titles): print("Info: " + str(titles[idx])) # print("percentage_similarity: {0}".format(str(similarity[idx]))) end = timer() print('find_match_time: ', (end - start)) if not self.isfolder: return str(titles[idx]) else: end = timer() print("{0} no similarity".format(str(self.image))) print('find_match_time: ', (end - start)) else: print("\n{0} has not points".format(str(image))) if not self.isfolder: return None if not os.path.exists(self.descriptions): print("\nDid not find file {}".format(self.descriptions)) return None with open(self.descriptions, 'rb') as handle: root = load(handle) flann = FlannBasedMatcher(self.index_params, {}) if self.iscover: thresh, octaves, size, ext_of_files = get_parameters("covers") else: thresh, octaves, size, ext_of_files = get_parameters(self.current_book) if not thresh: return None brisk = BRISK_create(thresh, octaves) # norm = cv.NORM_HAMMING (70,2) 30days if self.isfolder: files = os.listdir(self.image) for f in files: if f.endswith(ext_of_files): findmacth(str(image) + "/" + f) # cv_file.release() else: cur_book = findmacth(image) # cv_file.release() return cur_book return None
return error if __name__ == "__main__": img1 = imread('rect_left.jpeg') img2 = imread('rect_right.jpeg') # find the keypoints and descriptors with SIFT sift = xfeatures2d.SIFT_create() kp1, des1 = sift.detectAndCompute(img1,None) kp2, des2 = sift.detectAndCompute(img2,None) # FLANN parameters for points match FLANN_INDEX_KDTREE = 0 index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5) search_params = dict(checks=50) flann = FlannBasedMatcher(index_params,search_params) matches = flann.knnMatch(des1,des2,k=2) good = [] pts1 = [] pts2 = [] dis_ratio = [] for i,(m,n) in enumerate(matches): if m.distance < 0.3*n.distance: good.append(m) dis_ratio.append(m.distance/n.distance) pts2.append(kp2[m.trainIdx].pt) pts1.append(kp1[m.queryIdx].pt) min_idx = np.argmin(dis_ratio) # calculate fundamental matrix and check error fundMat = rectify(pts1, pts2)
class Matcher(object): def __init__(self): # Initiate ORB detectors self.orb = cv2.ORB_create() # If we want to filter ourselves the maches from bfmatches, crosscheck = # false self.bf = cv2.BFMatcher(cv2.NORM_HAMMING, crossCheck=False) self.Flann_index_lsh = 6 self.index_params = dict(algorithm=self.Flann_index_lsh, table_number=6, key_size=12, multi_probe_level=1) self.search_params = dict(checks=50) self.flann_matcher = FlannBasedMatcher(self.index_params, self.search_params) self.kp1 = KeyPoint() self.kp2 = KeyPoint() self.desc1 = None self.desc2 = None self.matches = None self.ratio = 0.65 self.matches1 = None self.matches2 = None self.good_matches = None self.good_kp1 = [] self.good_kp2 = [] self.n_matches = 0 # The following variables are used to work with numpy functions self.global_matches = [] # Numpy array self.global_kpts1 = [] self.global_kpts2 = [] # Create lists where we store the keypoints in their original format for # future uses. Also, store the descriptors self.curr_kp = [] # List of keypoints self.prev_kp = [] self.curr_dsc = [] # List of descriptors self.prev_dsc = [] self.is_minkp = None self.is_minmatches = None self.lk_params = dict( winSize=(15, 15), maxLevel=2, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03) ) def match(self, img_new, img_prev): # Compute the matches for the two images using the Brute Force matcher # @param img_new: The current image # @param img_prev: The reference image # Initialize control variables self.is_minkp = None self.is_minmatches = None self.good_kp1 = [] self.good_kp2 = [] self.good_matches = None self.kp1, self.desc1 = self.orb.detectAndCompute(img_prev, None) self.kp2, self.desc2 = self.orb.detectAndCompute(img_new, None) # Check descriptors # There are any keypoints? if not self.kp1 or not self.kp2: self.is_minkp = False print "The list of keypoints is emtpy" else: print "The list of keypoints is NOT empty" if np.size(self.desc1) == 0 or np.size(self.desc2) == 0: print "NO DESCRIPTORS" self.is_minkp = False else: self.is_minkp = True # Store the keypoints # print "len original", len(self.kp1) # for i in range(len(self.kp1)): # self.curr_kp.append(self.kp1[i]) # for i in range(len(self.kp2)): # self.prev_kp.append(self.kp2[i]) if self.is_minkp: print " Matching..." print self.is_minkp self.matches1 = self.bf.knnMatch(self.desc1, self.desc2, 2) self.matches2 = self.bf.knnMatch(self.desc2, self.desc1, 2) # Check now if there are any matches if self.matches1 is None or self.matches2 is None: print "There are keypoints, but no one matches" else: self.is_minmatches = True self.good_matches = self.filter_matches(self.matches1, self.matches2) # self.matches = sorted(self.matches, key=lambda x: x.distance) def filter_distance(self, matches): # Filter the matches based on a distance threshold # @param matches: List of matches (matcher objects) # pdb.set_trace() # Clear matches for wich NearestNeighbor (NN) ratio is > than threshold dist = [] sel_matches = [] thres_dist = 0 temp_matches = [] # print "Entering in the filter_distance function..." for i in range(0, len(matches) - 1): # We keep only those match objects with two matches: if (len(matches[i])) == 2: # print "There is at least one match with 2 candidate points" # If there are two matches: for j in range(0, 2): dist.append(matches[i][j].distance) temp_matches.append(matches[i]) # Now, calculate the threshold: if dist: thres_dist = (sum(dist) / len(dist)) * self.ratio else: return # Keep only reasonable matches based on the threshold distance: for i in range(0, len(temp_matches)): if (temp_matches[i][0].distance / temp_matches[i][1].distance) < thres_dist: sel_matches.append(temp_matches[i]) print "matches after distance", len(sel_matches) return sel_matches def filter_asymmetric(self, matches1, matches2): # Filter the matches with the symetrical test. # @param matches1: First list of matches # @param matches2: Second list of matches # @return sel_matches: filtered matches # Keep only symmetric matches sel_matches = [] # For every match in the forward direction, we remove those that aren't # found in the other direction for match1 in matches1: for match2 in matches2: # If matches are symmetrical: if (match1[0].queryIdx) == (match2[0].trainIdx) and (match2[0].queryIdx) == (match1[0].trainIdx): # We keep only symmetric matches and store the keypoints # of this matches sel_matches.append(match1) self.good_kp2.append(self.kp2[match1[0].trainIdx].pt) self.good_kp1.append(self.kp1[match1[0].queryIdx].pt) # Store also the keypoints in original form self.curr_kp.append(self.kp1[match1[0].queryIdx]) self.prev_kp.append(self.kp2[match1[0].trainIdx]) self.curr_dsc.append(self.desc1[match1[0].queryIdx]) self.prev_dsc.append(self.desc2[match1[0].trainIdx]) break # We have stored twice every keypoint. Remove them self.good_kp1 = self.good_kp1[::2] self.good_kp2 = self.good_kp2[::2] sel_matches = sel_matches[::2] print "matches after simmetric filter", len(sel_matches) return sel_matches def filter_matches(self, matches1, matches2): # This function filter two list of matches based on the distance # threshold and the symmetric test # @param matches1: First list of matches # @param matches2: Second list of matches # @return good_matches: List of filtered matches matches1 = self.filter_distance(matches1) if matches1: print ( "Matches1 after distance filter:\ {}".format( len(matches1) ) ) matches2 = self.filter_distance(matches2) if matches2: print ( "Matches2 after distance filter:\ {}".format( len(matches2) ) ) print "Ended distance filtering" if not matches1 or not matches2: print "Not matches1 or not matches2" self.is_minmatches = False else: self.is_minmatches = True print self.is_minmatches if self.is_minmatches: print "Go to filter asymmetric matches..." good_matches = self.filter_asymmetric(matches1, matches2) return good_matches else: return None def match_flann(self, img_new, img_prev): # Compute matches for the two images based on Flann. # @param img_new: Current frame # @param img_prev: Reference frame # First, keypoints and descriptors for both images self.kp1, self.desc1 = self.orb.detectAndCompute(img_new, None) self.kp2, self.desc2 = self.orb.detectAndCompute(img_prev, None) # Next, match: print "kp1", len(self.kp1) print "kp2", len(self.kp2) if self.kp1 and self.kp2: matches1 = self.flann_matcher.knnMatch(self.desc1, self.desc2, k=2) print "matches1", len(matches1) matches2 = self.flann_matcher.knnMatch(self.desc2, self.desc1, k=2) print "matches2", len(matches2) self.good_matches = self.filter_matches(matches1, matches2) else: print "No matches found" def draw_matches(self, img, matches): # Draw matches in the last image # @param img: image # @param matches: a matcher object (opencv) # @param kp1: keypoints of the old frame # @param kp2: keypoints of the new frame # @return img: image with lines between correlated points for i in range(len(matches)): idtrain = matches[i][0].trainIdx idquery = matches[i][0].queryIdx point_train = self.kp2[idtrain].pt point_query = self.kp1[idquery].pt point_train = self.transform_float_int_tuple(point_train) point_query = self.transform_float_int_tuple(point_query) cv2.line(img, ((point_train[0]), (point_train[1])), ((point_query[0]), (point_query[1])), (255, 0, 0)) return img def draw_matches_np(self, img, kpts_c, kpts_p): # Draw the matches in the image img, taking as input a numpy array # @param img: image # @param kpts_c: keypoints in the current image (numpy ndarray) # @param kpts_p: keypoints in the previous image (numpy ndarray) # @return img: image with lines between correlated points for i in range(len(kpts_c)): cv2.line(img, ((kpts_c[i][0]), (kpts_c[i][1])), ((kpts_p[i][0]), (kpts_p[i][1])), (255, 0, 0)) return img def draw_outliers_np(self, img, kpts_c, kpts_p): # Draw the matches in the image img, taking as input a numpy array # @param img: image # @param kpts_c: keypoints in the current image (numpy ndarray) # @param kpts_p: keypoints in the previous image (numpy ndarray) # @return img: image with lines between correlated points for i in range(len(kpts_c)): cv2.line(img, ((kpts_c[i][0]), (kpts_c[i][1])), ((kpts_p[i][0]), (kpts_p[i][1])), (0, 0, 255)) return img def transform_float_int_tuple(self, input_tuple): output_tuple = [0, 0] if not input_tuple is None: for i in range(0, len(input_tuple)): output_tuple[i] = int(input_tuple[i]) else: return input_tuple return output_tuple def append_matches(self): # Store all matches in one list for i in range(len(self.good_matches)): self.global_matches.append(self.good_matches[i]) def append_keypoints1(self): # Store keypoints from the current image in one list for i in range(len(self.good_kp1)): self.global_kpts1.append(self.good_kp1[i]) def append_keypoints2(self): # Store keypoints from the current image in one list for i in range(len(self.good_kp2)): self.global_kpts2.append(self.good_kp2[i]) def append_global(self): self.append_keypoints1() self.append_keypoints2() self.append_matches() def sum_coord(self, x_ini, y_ini): # In order to get the true coordinates of the keypoits distributed along # over the entire image, not the coordinates refered to the roi's, we # have to sum the start vector to every keypoint for i in range(len(self.good_kp1)): self.good_kp1[i] = list(self.good_kp1[i]) self.good_kp2[i] = list(self.good_kp2[i]) self.good_kp1[i][0] += x_ini self.good_kp1[i][1] += y_ini self.good_kp2[i][0] += x_ini self.good_kp2[i][1] += y_ini self.good_kp1[i] = np.array([self.good_kp1[i][0], self.good_kp1[i][1]]).reshape(2) self.good_kp2[i] = np.array([self.good_kp2[i][0], self.good_kp2[i][1]]).reshape(2) def lktracker(self, img_prev, img_curr, prev_points): # Tracks the prev_points in the current image. # @param img_prev: image, the previous image # @param img_curr: the current image # @param prev_points: vector of points in the previous image list_tracks = [] curr_points, st, err = cv2.calcOpticalFlowPyrLK(img_prev, img_curr, prev_points, None, **self.lk_params) print "LK current points", len(curr_points) prev_points2, st, err = cv2.calcOpticalFlowPyrLK(img_curr, img_prev, curr_points, None, **self.lk_params) print "LK prev poiints", len(prev_points) d = abs(prev_points - prev_points2).reshape(-1, 2).max(-1) print "d", d good = d < 2 # print "good", good for (x, y), good_flag in zip(curr_points.reshape(-1, 2), good): if not good_flag: continue list_tracks.append((x, y)) return good, prev_points2, list_tracks