def generate_affine_transform(video='12'): if video == '12': gps, pixels = cam.video_12() elif video == '13': gps, pixels = cam.video_13() else: gps, pixels = cam.vid_both() gps_to_vid = cv2.estimateAffine2D(gps, pixels) vid_to_gps = cv2.estimateAffine2D(pixels, gps) return gps_to_vid[0], vid_to_gps[0]
def compute_homography(self, physical_object_image, cropped_image): # If the image of the physical object is too small, # scale it up to improve feature detection physical_object_image, cropped_image = self.checkAndScaleImages( physical_object_image, cropped_image) # I tried SIFT, SURF, and ORB too. AKAZE gives the best and fastest result. physicalObjectKeyPoints, physicalObjectDescriptors, croppedImageKeyPoints, croppedImageDescriptors \ = self.extract_feature_points(physical_object_image, cropped_image, algorithm='AKAZE') # I also tried BRUTE_FORCE_L1, BRUTE_FORCE_HAMMING with ratio test, and FLANN. matches = self.find_matches(physicalObjectDescriptors, croppedImageDescriptors, algorithm='BRUTE_FORCE_HAMMING', ratio_test=False) if debug: # Draw top matches im_matches = cv2.drawMatches(physical_object_image, physicalObjectKeyPoints, cropped_image, croppedImageKeyPoints, matches, None) cv2.imwrite(str(os.path.join(temp_folder_path, "matches.jpg")), im_matches) points1, points2 = self.find_points_from_matches( physicalObjectKeyPoints, croppedImageKeyPoints, matches) try: h1, mask = cv2.estimateAffine2D( points1, points2, method=cv2.RANSAC, ransacReprojThreshold=self.ransac_reprojection_threshold) except: h1 = DEFAULT_HOMOGRAPHY homography_result = PoseEstimationOutput(None, h1, None) if self.recompute_homography_using_only_inliers: # get the inlier matches (list of tuples of points) # run find homography again unsing only the inliers this time instead of LEMDS or RHO instead of RANSAC boolean_inliers_mask = (mask > 0) inlier_points1 = points1[boolean_inliers_mask.repeat( 2, axis=1)].reshape((-1, 2)) inlier_points2 = points2[boolean_inliers_mask.repeat( 2, axis=1)].reshape((-1, 2)) # h2, mask = cv2.findHomography(inlier_points1, inlier_points2, cv2.LMEDS) h2, mask = cv2.estimateAffine2D(inlier_points1, inlier_points2, cv2.LMEDS) # prosac_reprojection_error = 2.5 # h2, mask = cv2.findHomography(inlier_points1, inlier_points2, cv2.RHO, prosac_reprojection_error) if h2 is not None: homography_result.homography = h2 logger.debug("Homography: %s", homography_result.homography) return homography_result
def transferExpression(self, lmarkSeq, meanShape): exptransSeq = copy.deepcopy(lmarkSeq) firstFlmark = exptransSeq[0, :, :] indexes = np.array([60, 64, 62, 67]) tformMS, _ = cv2.estimateAffine2D(firstFlmark[:, :], np.float32(meanShape[:, :])) sx = np.sign(tformMS[0, 0]) * np.sqrt(tformMS[0, 0]**2 + tformMS[0, 1]**2) sy = np.sign(tformMS[1, 0]) * np.sqrt(tformMS[1, 0]**2 + tformMS[1, 1]**2) print(sx, sy) prevLmark = copy.deepcopy(firstFlmark) prevExpTransFlmark = copy.deepcopy(meanShape) zeroVecD = np.zeros((1, 68, 2)) diff = np.cumsum(np.insert(np.diff(exptransSeq, n=1, axis=0), 0, zeroVecD, axis=0), axis=0) msSeq = np.tile(np.reshape(meanShape, (1, 68, 2)), [lmarkSeq.shape[0], 1, 1]) diff[:, :, 0] = abs(sx) * diff[:, :, 0] diff[:, :, 1] = abs(sy) * diff[:, :, 1] exptransSeq = diff + msSeq return exptransSeq
def sample_rotated_patch(img, pos, angle, patch_size): rotate = np.asarray([ [np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)], ]) _dst = np.asarray([[-patch_size // 2, -patch_size // 2], [-patch_size // 2, patch_size // 2], [patch_size // 2, patch_size // 2], [patch_size // 2, -patch_size // 2]], dtype=np.float32) src = np.asarray([[pos[0], pos[1]], [pos[0], pos[1] + patch_size], [pos[0] + patch_size, pos[1] + patch_size], [pos[0] + patch_size, pos[1]]]) tran = src[0] - _dst[0] src = np.asarray([rotate.dot(_dst[i]) + tran for i in range(src.shape[0])], dtype=np.float32) dst = np.asarray([[0, 0], [0, patch_size - 1], [patch_size - 1, patch_size - 1], [patch_size - 1, 0]], dtype=np.float32) theta, _ = cv2.estimateAffine2D(src, dst) patch = cv2.warpAffine(img, theta, (patch_size, patch_size)) return patch
def findAffine(image_1_kp, image_2_kp, matches): image_1_points = np.zeros((len(matches), 1, 2), dtype=np.float32) image_2_points = np.zeros((len(matches), 1, 2), dtype=np.float32) # WRITE YOUR CODE HERE. # initiate two emtpy lists image_1_pts, image_2_pts = [], [] for match in matches: # Get the matching keypoints for each of the images pt1 = image_1_kp[match.queryIdx].pt pt2 = image_2_kp[match.trainIdx].pt image_1_pts.append(pt1) image_2_pts.append(pt2) image_1_points = np.float64(image_1_pts).reshape(-1, 1, 2) image_2_points = np.float64(image_2_pts).reshape(-1, 1, 2) # Compute Affine M, _ = cv2.estimateAffine2D(image_1_points, image_2_points, cv2.RANSAC) M_aff = np.eye(3) M_aff[:2,:] = M return M_aff
def similarityTransform(inPoints, outPoints): s60 = math.sin(60 * math.pi / 180) c60 = math.cos(60 * math.pi / 180) inPts = np.copy(inPoints).tolist() outPts = np.copy(outPoints).tolist() # The third point is calculated so that the three points make an equilateral triangle xin = c60 * (inPts[0][0] - inPts[1][0]) - s60 * (inPts[0][1] - inPts[1][1]) + inPts[1][0] yin = s60 * (inPts[0][0] - inPts[1][0]) + c60 * (inPts[0][1] - inPts[1][1]) + inPts[1][1] inPts.append([np.int(xin), np.int(yin)]) xout = c60 * (outPts[0][0] - outPts[1][0]) - s60 * ( outPts[0][1] - outPts[1][1]) + outPts[1][0] yout = s60 * (outPts[0][0] - outPts[1][0]) + c60 * ( outPts[0][1] - outPts[1][1]) + outPts[1][1] outPts.append([np.int(xout), np.int(yout)]) # Now we can use estimateRigidTransform for calculating the similarity transform. #tform = cv2.estimateRigidTransform(np.array([inPts]), np.array([outPts]), False) # This fxn is available in cv2 version 3.4 not in version 4 onward #return tform # This is for cv2 version 4 onward tform = cv2.estimateAffine2D(np.array([inPts]), np.array([outPts]), False) return tform[0]
def find_affine_w2s_random(dataset_name): with open("./results/%s_with_spline_clusters.vd" % (dataset_name), 'rb') as f: vd = pkl.load(f) plane = vd.plane rt_c2w = find_rt_c2w(vd) landmark_sate_path = "../visual_opengl/%s/bird_view.txt" % (dataset_name) with open(landmark_sate_path, 'r') as f: lines = f.readlines() landmark_sate = [] for line in lines: line_split = line.split(",") landmark_2d = [float(line_split[0]), float(line_split[1])] landmark_sate.append(landmark_2d) landmark_sate = np.array(landmark_sate) # landmark in cam coordinate landmark_perspective_path = "../visual_opengl/%s/cam_view.pklcor" % ( dataset_name) with open(landmark_perspective_path, 'rb') as f: landmark_perspective = pkl.load(f) offset = np.random.randint(-40, 40, tuple( landmark_perspective.shape)) # best for IMG_0121 landmark_perspective += offset landmark_cam = find_landmark_cam(plane, landmark_perspective, focal_dict[dataset_name]) landmark_world = cart((rt_c2w @ ((h**o(landmark_cam)).T)).T) affine_w2s, _ = cv2.estimateAffine2D(landmark_world[:, 0:2], landmark_sate) print(affine_w2s) return affine_w2s
def compute_timewise_homographies(frames, features, outputMatches=False): print("finding matches between frames...") bf = cv.BFMatcher(cv.NORM_HAMMING, crossCheck=True) timewise_homographies = [] for i in range(len(frames) - 1): matches = bf.match(features[i][1], features[i + 1][1]) if outputMatches: img3 = cv.drawMatches(frames[i], features[i][0], frames[i + 1], features[i + 1][0], matches, None, flags=2) cv.imwrite( 'data/matches/matches_' + str(i) + "-" + str(i + 1) + ".jpg", img3) src_pts = np.float32([features[i][0][m.queryIdx].pt for m in matches]).reshape(-1, 1, 2) dst_pts = np.float32([ features[i + 1][0][m.trainIdx].pt for m in matches ]).reshape(-1, 1, 2) M, _ = cv.estimateAffine2D(src_pts, dst_pts, method=cv.RANSAC) if M is None: return timewise_homographies, i H = np.append(M, np.array([0, 0, 1]).reshape((1, 3)), axis=0) timewise_homographies.append(H) return timewise_homographies, len(frames) - 1
def solveQ3Part1(): src = cv.imread("Q3/Dylan.jpg") target = cv.imread("Q3/frames.jpg") srcTri = np.array([[0, 0], [640, 0], [640, 480]]).astype(np.float32) ######################Affine transform##################################### dstTriAffin = np.array([[551, 220], [844, 66], [901, 299]]).astype(np.float32) warp_mat = cv.estimateAffine2D(srcTri, dstTriAffin)[0] warp_dst_affin = cv.warpAffine(src, warp_mat, (target.shape[1], target.shape[0])) ########################################################################### ######################Proj transform##################################### srcTri = np.array([[0, 0], [640, 0], [640, 480], [0, 480]]).astype(np.float32) dstTriProj = np.array([[195, 55], [495, 159], [431, 498], [37, 182]]).astype(np.float32) warp_mat_proj = cv.getPerspectiveTransform(srcTri, dstTriProj) warp_dst_affin_proj = cv.warpPerspective( src, warp_mat_proj, (target.shape[1], target.shape[0])) cv.imwrite('warpedImageOverFrames.jpg', target + warp_dst_affin + warp_dst_affin_proj) cv.imwrite('warpedImageOverBlackBackground.jpg', warp_dst_affin + warp_dst_affin_proj) plt.imshow(warp_dst_affin + warp_dst_affin_proj) plt.show() plt.imshow(target + warp_dst_affin + warp_dst_affin_proj) plt.show() print( "Results saved in warpedImageOverBlackBackground.jpg and warpedImageOverFrames.jpg\n" )
def to_refine(self, image, pts, scale=3.0): """ refine the image by input points. :param image_rgb: input image :param pts: points """ x1, y1, x2, y2, x3, y3, x4, y4 = pts.ravel() cx, cy = int(128 // 2), int(48 // 2) cw = 64 ch = 24 tx1 = cx - cw // 2 ty1 = cy - ch // 2 tx2 = cx + cw // 2 ty2 = cy - ch // 2 tx3 = cx + cw // 2 ty3 = cy + ch // 2 tx4 = cx - cw // 2 ty4 = cy + ch // 2 target_pts = np.array([[tx1, ty1], [tx2, ty2], [tx3, ty3], [tx4, ty4] ]).astype(np.float32) * scale org_pts = np.array([[x1, y1], [x2, y2], [x3, y3], [x4, y4]]).astype(np.float32) # mat_ = cv2.estimateRigidTransform(org_pts, target_pts, True) mat_ = cv2.estimateAffine2D(org_pts, target_pts)[0] dsize = (int(120 * scale), int(48 * scale)) warped = cv2.warpAffine(image, mat_, dsize) return warped
def getTransform(src, dst, method='affine'): pts1, pts2 = feature_matching(src, dst) # x, y = zip(*pts1) # plt.subplot(121) # plt.scatter(x, y, 1, c='r', marker='x', lw=4) # plt.imshow(img1, cmap='gray') # x, y = zip(*pts2) # plt.subplot(122) # plt.scatter(x,y, 1, c='r', marker='x', lw=4) # plt.imshow(img2, cmap='gray') # plt.show() src_pts = np.float32(pts1).reshape(-1, 1, 2) dst_pts = np.float32(pts2).reshape(-1, 1, 2) if method == 'affine': M, mask = cv2.estimateAffine2D(src_pts, dst_pts, cv2.RANSAC, ransacReprojThreshold=5.0) #M = np.append(M, [[0,0,1]], axis=0) if method == 'homography': M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0) matchesMask = mask.ravel().tolist() return (M, pts1, pts2, mask)
def CorrectImage(img, result_dict, boundboxes, width, height): # 获取对应基准点坐标对,计算最优仿射变换矩阵 points_from = [] points_to = [] for k, v in boundboxes.items(): if k[:3] != 'key': continue a = process.extractOne(k[3:], result_dict.keys(), scorer=fuzz.ratio, score_cutoff=90) if a: points_from.append([result_dict[a[0]][0], result_dict[a[0]][1]]) points_to.append([v[0], v[1]]) # print(points_from) # print(points_to) affine_transform_matrix = cv2.estimateAffine2D(np.array(points_from), np.array(points_to)) # print(affine_transform_matrix[0]) dstImg = cv2.warpAffine(img, affine_transform_matrix[0], (width, height), borderMode=cv2.INTER_LINEAR, borderValue=cv2.BORDER_REPLICATE) return dstImg
def __get_homography(self, matches, dst_kps, src_kps, reproj_thresh, is_affine=False): """Find homography from one photo to another :param matches: matches between both photos :param dst_kps: Destination frame of reference key-points :param src_kps: Source frame of reference key-points :param reproj_thresh: RANSAC threshold :param is_affine: Boolean to find an affine transforamtion instead of homography :return: (Homography/Affine matrix, status) """ kps_new = np.float32([kp.pt for kp in src_kps]) kps_panorama = np.float32([kp.pt for kp in dst_kps]) if len(matches) > 4: # construct the two sets of points pts_new = np.float32([kps_new[m.queryIdx] for m in matches]) pts_panorama = np.float32( [kps_panorama[m.trainIdx] for m in matches]) # estimate the homography between the sets of points if is_affine: return cv2.estimateAffine2D(pts_new, pts_panorama, cv2.RANSAC, reproj_thresh) else: return cv2.findHomography(pts_new, pts_panorama, cv2.RANSAC, reproj_thresh) else: return None
def translateImage(im1,im2): try: im1Gray = cv2.cvtColor(im1, cv2.COLOR_BGR2GRAY) im2Gray = cv2.cvtColor(im2, cv2.COLOR_BGR2GRAY) orb = cv2.ORB_create(MAX_FEATURES) keypoints1, descriptors1 = orb.detectAndCompute(im1Gray, None) keypoints2, descriptors2 = orb.detectAndCompute(im2Gray, None) matcher = cv2.DescriptorMatcher_create(cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING) matches = matcher.match(descriptors1, descriptors2, None) matches.sort(key=lambda x: x.distance, reverse=False) numGoodMatches = int(len(matches) * GOOD_MATCH_PERCENT) matches = matches[:numGoodMatches] points1 = np.zeros((len(matches), 2), dtype=np.float32) points2 = np.zeros((len(matches), 2), dtype=np.float32) for i, match in enumerate(matches): points1[i, :] = keypoints1[match.queryIdx].pt points2[i, :] = keypoints2[match.trainIdx].pt h,_ = cv2.estimateAffine2D(points1,points2) h[0][0]=1 h[0][1]=0 h[1][0]=0 h[1][1]=1 height, width, _ = im2.shape im1Reg = cv2.warpAffine(im1,h,(width,height),borderValue=(255,255,255)) return im1Reg, h except: h="error" return im1 , None
def get_rating_of_affine_transform(kp1, kp2): a = np.asarray([p.pt for p in kp1]) b = np.asarray([p.pt for p in kp2]) retval, inliers = cv2.estimateAffine2D(a, b, None) assert len(inliers) == len(kp1) inlier_ratio = np.count_nonzero(inliers) / len(inliers) return inlier_ratio
def compute_transform_matrix(self, im_prev, im_next): ''' Computes the transformation between two images Arguments: - im_prev: np array of shape self.img_shape and no color channels - im_next: np array of shape self.img_shape and no color channels Returns: - The affine transformation matrix, np array of shape (2,3) ''' prev_pts = cv2.goodFeaturesToTrack(im_prev, maxCorners=200, qualityLevel=0.01, minDistance=30, blockSize=3, mask=self.features_mask) next_pts, status, err = cv2.calcOpticalFlowPyrLK( im_prev, im_next, prev_pts, None) idx = np.where(status == 1)[0] prev_pts = prev_pts[idx] next_pts = next_pts[idx] m = cv2.estimateAffine2D(prev_pts, next_pts) return m[0]
def warp_image(image, points_in_image, points_in_baseline, transformation_type='affine'): if transformation_type == 'homographic': h, _ = cv2.findHomography(np.float32(points_in_image), np.float32(points_in_baseline)) warpped_image = cv2.warpPerspective(image, h, (image.shape[1], image.shape[0])) return warpped_image, h elif transformation_type == 'affine': M, _ = cv2.estimateAffine2D(np.float32(points_in_image), np.float32(points_in_baseline)) warpped_image = cv2.warpAffine(image, M, (image.shape[1], image.shape[0])) return warpped_image, M elif transformation_type == 'rigid': M = cv2.estimateRigidTransform(np.float32(points_in_image), np.float32(points_in_baseline), fullAffine=False) warpped_image = cv2.warpAffine(image, M, (image.shape[1], image.shape[0])) return warpped_image, M else: raise Exception('Unsupported method')
def get_dominant_motion(mdata): pt0, pt1, m01 = mdata i0, i1 = np.stack([(m.queryIdx, m.trainIdx) for m in m01], axis=1) #least_squares(cost_fn, #cv2.estimateRigidTransform(pt0[i0], pt1[i1], True) M, _ = cv2.estimateAffine2D(pt0[i0], pt1[i1]) A, b = M[:, :2], M[:, 2] return A, b
def KLTmain(im, im0, im0_small, p0): # Parameters for KLT EPS = cv2.TERM_CRITERIA_EPS COUNT = cv2.TERM_CRITERIA_COUNT lk_coarse = dict(winSize=(15, 15), maxLevel=4, criteria=(EPS | COUNT, 10, 0.1)) lk_fine = dict(winSize=(51, 51), maxLevel=0, criteria=(EPS | COUNT, 30, 0.001)) # 1. Coarse tracking on 1/8 scale full image scale = 1 / 4 im_small = cv2.resize(im, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_NEAREST) if im0_small is None: im0_small = cv2.resize(im0, (0, 0), fx=scale, fy=scale, interpolation=cv2.INTER_NEAREST) p, v, _ = cv2calcOpticalFlowPyrLK(im0_small, im_small, p0 * scale, None, **lk_coarse) p /= scale T23, inliers = cv2.estimateAffine2D( p0[v], p[v], method=cv2.RANSAC) # 2x3, better results v[v] = inliers.ravel().astype(bool) # import plots; plots.imshow(im0_small//2+im_small//2, p1=p0[v]*scale,p2=p[v]*scale) # 2. Coarse tracking on full resolution roi https://www.mathworks.com/discovery/affine-transformation.html translation = p[v] - p0[v] T = np.eye(3, 2) T[2] = translation.mean(0) # translation-only transform p, v = KLTregional(im0, im, p0, T, lk_coarse, fbt=1, translateFlag=True) if v.sum() > 10: # good fit T23, inliers = cv2.estimateAffine2D( p0[v], p[v], method=cv2.RANSAC) # 2x3, better results else: print('KLT coarse-affine failure, running SURF matches full scale.') T23, inliers = estimateAffine2D_SURF(im0, im, p0, scale=1) # 3. Fine tracking on affine-transformed regions p, v = KLTregional(im0, im, p0, T23.T, lk_fine, fbt=0.3) return p[v], v, im_small
def test_optical_flow(self): frames, fps = stabilization.extract_frames_from_video( self.motion_video_file) homographies = [] # params for ShiTomasi corner detection feature_params = dict(maxCorners=100, qualityLevel=0.1, minDistance=4, blockSize=4) # Parameters for lucas kanade optical flow lk_params = dict(winSize=(5, 5), maxLevel=2, criteria=(cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) for i in range(1, len(frames)): prev_frame = frames[i - 1] prev_gray = cv2.cvtColor(prev_frame, cv2.COLOR_BGR2GRAY) p0 = cv2.goodFeaturesToTrack(prev_gray, mask=None, **feature_params) curr_frame = frames[i] curr_gray = cv2.cvtColor(curr_frame, cv2.COLOR_BGR2GRAY) # calculate optical flow p1, st, err = cv2.calcOpticalFlowPyrLK(prev_gray, curr_gray, p0, None, **lk_params) # Select good points good_new = p1[st == 1] good_old = p0[st == 1] circled = np.copy(prev_frame) for j in good_old: x, y = j.ravel() cv2.circle(circled, (x, y), 1, (0, 0, 255), -1) frame_filename = path.join(self.motion_video_output_dir, "frame_{}_1.jpg".format(i)) cv2.imwrite(frame_filename, circled) circled = np.copy(curr_frame) for j in good_new: x, y = j.ravel() cv2.circle(circled, (x, y), 1, (0, 0, 255), -1) frame_filename = path.join(self.motion_video_output_dir, "frame_{}_2.jpg".format(i)) cv2.imwrite(frame_filename, circled) M, _ = cv2.estimateAffine2D(good_new, good_old, method=cv2.RANSAC) H = np.append(M, np.array([0, 0, 1]).reshape((1, 3)), axis=0) homographies.append(H) homographies = np.array(homographies) print(homographies) stabilization.plot_homographies(homographies, self.motion_video_output_dir)
def main(): files_directory = r'C:\Scratch\IPA_Data\Sampled\15_Ambient_Combined\06_red_green_blue' image_names = [r'a0_amb.tif', r'b0_amb.tif'] image_paths = [] for filename in image_names: image_paths.append(os.path.join(files_directory, filename)) img1 = cv.imread(image_paths[0], 1) img2 = cv.imread(image_paths[1], 1) surf = cv.xfeatures2d.SURF_create() surf.setHessianThreshold(400) # find the keypoints and descriptors with SIFT kp1, des1 = surf.detectAndCompute(img1, None) kp2, des2 = surf.detectAndCompute(img2, None) print(len(kp1)) print(len(kp2)) img3 = cv.drawKeypoints(img1, kp1, None, (255, 0, 0), 4) img4 = cv.drawKeypoints(img2, kp2, None, (255, 0, 0), 4) bf = cv.BFMatcher() matches = bf.knnMatch(des1, des2, k=2) # Apply ratio test good = [] pts2 = [] pts1 = [] for i, (m, n) in enumerate(matches): if m.distance < 0.8 * n.distance: good.append(m) pts2.append(kp2[m.trainIdx].pt) pts1.append(kp1[m.queryIdx].pt) pts1 = np.int32(pts1) pts2 = np.int32(pts2) f_matrix, mask = cv.findFundamentalMat(pts1, pts2, cv.FM_LMEDS) homog = cv.findHomography(pts1, pts2) aff = cv.estimateAffine2D(pts1, pts2, np.ones([len(pts1)]), cv.LMEDS, 3, 2000, 0.99, 10) warp_dst = cv.warpAffine(img1, aff[0], (img2.shape[1], img2.shape[0])) center = (warp_dst.shape[1] // 2, warp_dst.shape[0] // 2) cv.imshow('Source image', img2) cv.imshow('Warp', warp_dst) cv.waitKey()
def fit_affine(X_source, X_target, do_debug=False): A, _ = cv2.estimateAffine2D(numpy.array(X_source), numpy.array(X_target), confidence=0.95) result = cv2.transform(X_source.reshape((-1, 1, 2)), A).reshape((-1, 2)) loss = ((result - X_target)**2).mean() if do_debug: image = debug_projection(X_source, X_target, result) return A, result
def affine(from_data, to_data, target_data): """Compute the 2D affine transformation between the data sets via opencv""" # calculate an approximate affine affine_matrix, inliers = cv2.estimateAffine2D(from_data, to_data, ransacReprojThreshold=3, maxIters=20000, refineIters=0, method=cv2.LMEDS) print('Percentage inliers used:' + str(np.sum(inliers)*100/from_data.shape[0])) # make the transformed data homogeneous for multiplication with the affine transformed_data = np.squeeze(cv2.convertPointsToHomogeneous(target_data)) # apply the affine matrix return np.matmul(transformed_data, affine_matrix.T)
def alignImages(im1, im2): # Convert images to grayscale im1Gray = cv2.cvtColor(im1, cv2.COLOR_BGR2GRAY) im2Gray = cv2.cvtColor(im2, cv2.COLOR_BGR2GRAY) # Detect ORB features and compute descriptors. orb = cv2.ORB_create(MAX_FEATURES) keypoints1, descriptors1 = orb.detectAndCompute(im1Gray, None) keypoints2, descriptors2 = orb.detectAndCompute(im2Gray, None) print("number of keypoints " + str(len(keypoints1)) + " " + str(len(keypoints2))) # Match features. matcher = cv2.DescriptorMatcher_create( cv2.DESCRIPTOR_MATCHER_BRUTEFORCE_HAMMING) matches = matcher.match(descriptors1, descriptors2, None) print("number of matches " + str(len(matches))) # Sort matches by score matches.sort(key=lambda x: x.distance, reverse=False) # Remove not so good matches numGoodMatches = int(len(matches) * GOOD_MATCH_PERCENT) matches = matches[:numGoodMatches] print("number of goodmatches " + str(numGoodMatches)) # Draw top matches imMatches = cv2.drawMatches(im1, keypoints1, im2, keypoints2, matches, None) cv2.imwrite("matches.jpg", imMatches) # Extract location of good matches points1 = np.zeros((len(matches), 2), dtype=np.float32) points2 = np.zeros((len(matches), 2), dtype=np.float32) for i, match in enumerate(matches): points1[i, :] = keypoints1[match.queryIdx].pt points2[i, :] = keypoints2[match.trainIdx].pt # Find homography # h, mask = cv2.findHomography(points1, points2, cv2.RANSAC) # Use homography height, width, channels = im2.shape #im1Reg = cv2.warpPerspective(im1, h, (width, height)) #estimate rigid SetTransform affTransf, inliers = cv2.estimateAffine2D(points1, points2) im1Reg = cv2.warpAffine(im1, affTransf, (width, height)) return im1Reg, affTransf
def match_key_points(self, key_points_a, key_points_b, descriptors_a, descriptors_b, ratio, threshold, method="homography"): """ match key points given by SIFT/SURF :param method: homography/affine :param key_points_a: key points of first image :param key_points_b: key points of second image :param descriptors_a: descriptors of first image :param descriptors_b: descriptors of second image :param ratio: :param threshold: :return: matches and homography/affine matrix """ # Find matches raw_matches = self.desc_matcher.knnMatch(descriptors_a, descriptors_b, 2) matches = [] # Save matches for m in raw_matches: if len(m) == 2 and m[0].distance < m[1].distance * ratio: matches.append((m[0].trainIdx, m[0].queryIdx)) logger_instance.log(LogLevel.DEBUG, "matches: " + str(len(matches))) # Estimate geometrical transformation if len(matches) > self.matches_required: points_a = np.float32([key_points_a[i] for (_, i) in matches]) points_b = np.float32([key_points_b[i] for (i, _) in matches]) if method == "homography": (H, status) = cv2.findHomography(points_a, points_b, cv2.RANSAC, threshold) elif method == "affine": (H, status) = cv2.estimateAffine2D( points_a, points_b, cv2.RANSAC, ransacReprojThreshold=threshold) else: raise Exception( "Invalid call, unsupported transformation type: " + method) return matches, H, status return None
def affine(img1, img2): img1_pts, img2_pts = feature_matching(img1, img2) pts = img1_pts - img2_pts std = np.std(pts) #img1_pts = np.float32(pts1).reshape(-1,1,2) #img2_pts = np.float32(pts2).reshape(-1,1,2) M, mask = cv2.estimateAffine2D(img1_pts, img2_pts, cv2.RANSAC, ransacReprojThreshold=0.4) M = np.array([[1, 0, M[0, 2]], [0, 1, M[1, 2]]]) return M
def estimate_rigid_transform(self, image1_pts, image2_pts, use_full): if int(self.OPENCV_MAJOR) < 4: # new in opencv4 affine = cv2.estimateRigidTransform(image1_pts, image2_pts, fullAffine=use_full) else: if use_full: # noinspection PyUnresolvedReferences # new implementation returns also a vector which indicated which points are inliers affine = cv2.estimateAffine2D(image1_pts, image2_pts) else: # noinspection PyUnresolvedReferences affine = cv2.estimateAffinePartial2D(image1_pts, image2_pts) return affine
def feature_matching(self, img1, img2, transform='Affine'): # Initiate SIFT detector sift = cv.SIFT_create() # find the keypoints and descriptors with SIFT kp1, des1 = sift.detectAndCompute(img1, None) kp2, des2 = sift.detectAndCompute(img2, None) # FLANN parameters FLANN_INDEX_KDTREE = 1 index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) search_params = dict(checks=50) # or pass empty dictionary flann = cv.FlannBasedMatcher(index_params, search_params) matches2to1 = flann.knnMatch(des2, des1, k=2) matchesMask_ratio = [[0, 0] for i in range(len(matches2to1))] match_dict = {} for i, (m, n) in enumerate(matches2to1): if m.distance < 0.7 * n.distance: matchesMask_ratio[i] = [1, 0] match_dict[m.trainIdx] = m.queryIdx # perform reciprocal matching to ensure better matches good = [] recip_matches = flann.knnMatch(des1, des2, k=2) matchesMask_ratio_recip = [[0, 0] for i in range(len(recip_matches))] for i, (m, n) in enumerate(recip_matches): if m.distance < 0.7 * n.distance: # ratio if m.queryIdx in match_dict and match_dict[ m.queryIdx] == m.trainIdx: good.append(m) matchesMask_ratio_recip[i] = [1, 0] # draw_params = dict(matchColor = (0,255,0), singlePointColor = (255,0,0), matchesMask = matchesMask_ratio_recip, flags = 0) # img3 = cv.drawMatchesKnn(img1,kp1,img2,kp2,recip_matches,None,**draw_params) # cv.imshow("warped", img3) # cv.waitKey() pts1, pts2 = ([kp1[m.queryIdx].pt for m in good], [kp2[m.trainIdx].pt for m in good]) src_pts = np.float32(pts1).reshape(-1, 1, 2) dst_pts = np.float32(pts2).reshape(-1, 1, 2) if transform == 'Affine': M, mask = cv.estimateAffine2D(src_pts, dst_pts, cv.RANSAC, ransacReprojThreshold=5.0) elif transform == 'Homography': M, mask = cv.findHomography(src_pts, dst_pts, cv.RANSAC, 5.0) return M
def main(): if not (os.path.exists(CROPPED_RECTIFIED_PATH)): os.makedirs(CROPPED_RECTIFIED_PATH) fnames = os.listdir(CROPPED_LABEL_PATH) seg_zero = plt.imread(os.path.join(ZERO_FULL_PATH, ZERO_FULL + ".png")) seg_label = np.uint8(seg_zero[:, :, :3] * 255) #get keypoints from zero_full images dst = [] for c in colors["keypoints"]: dst.append(getColorSqrCenter(seg_label, c)) dst = np.array(dst) torch.cuda.set_device(CUDA_DEVICE) model_pred = Keypoints(NUM_CLASSES) model_pred = model_pred.cuda() model_pred.load_state_dict(torch.load(os.path.join(PATH, "model_100.pkl"))) model_pred.eval() num_of_correction = 0 for fname in fnames: #print(fname) image = cv2.imread(os.path.join(CROPPED_TEST_PATH, fname)) for i in range(0, 9, 3): #get keypoints from label images cropped_im = image[i:IMG_WIDTH - i, i:IMG_HEIGHT - i] cropped_im = cv2.resize(cropped_im, (IMG_WIDTH, IMG_HEIGHT), interpolation=cv2.INTER_NEAREST) im = transform1(np.array(cropped_im)) im = im.cuda() prediction = Prediction(model_pred, NUM_CLASSES, IMG_HEIGHT, IMG_WIDTH, IMG_SMALL_HEIGHT, IMG_SMALL_WIDTH) result, keypoints = prediction.predict(im) keypoints = keypoints.cpu().numpy() keypoints = np.array(keypoints) #transpose and save T = cv2.estimateAffine2D(keypoints, dst, ransacReprojThreshold=5) im_rect = cv2.warpAffine(image, T[0], (seg_zero.shape[1], seg_zero.shape[0])) if (rectified_judge(im_rect, seg_zero.shape[1], seg_zero.shape[0]) == 1): image = im_rect num_of_correction = num_of_correction + 1 break image = image[:, :, (2, 1, 0)] image = Image.fromarray(image.astype(np.uint8)) image.save(os.path.join(CROPPED_RECTIFIED_PATH, fname)) print("%d dial %d correction images" % (num_of_dial, num_of_correction))
def update_transformation_matrix(self): sr = np.array([ [175, 417], [313, 171], [469, 331] ]) ds = np.array([ [-40, -424], [30, -354], [-20, -290] ]) retval, inliers = cv.estimateAffine2D(sr, ds) with open('storage.pkl', 'wb') as f: pickle.dump(retval, f)
def getTransform(src, dst, method='affine'): pts1, pts2 = feature_matching(src, dst) src_pts = np.float32(pts1).reshape(-1, 1, 2) dst_pts = np.float32(pts2).reshape(-1, 1, 2) if method == 'affine': M, mask = cv2.estimateAffine2D( src_pts, dst_pts, cv2.RANSAC, ransacReprojThreshold=5.0) # M = np.append(M, [[0, 0, 1]], axis=0) if method == 'homography': M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC, 5.0) matchesMask = mask.ravel().tolist() return (M, pts1, pts2, mask)