def partial_data_registration_local(source, target, params): echo = params['echo'] o_y_size, o_x_size = source.shape source, target = initial_resample(source, target, params['local_max_size'], params) y_size, x_size = source.shape image_min_size = min(y_size, x_size) min_size = params['local_min_size'] min_ratio = image_min_size / min_size levels = int(np.log2(np.floor(min_ratio))) + 1 sources, targets = build_pyramids(source, target, levels) if echo: print("Local registration started.") for i in range(levels): if echo: print() print("Current level: %f/%f" % (i + 1, levels)) print() current_source = sources[i] current_target = targets[i] if i == 0: u_x, u_y = np.zeros(current_source.shape), np.zeros( current_source.shape) if i != 0: current_source = utils.warp_image(current_source, u_x, u_y) t_u_x, t_u_y = single_resolution_local(current_source, current_target, i, params) u_x, u_y = utils.compose_vector_fields(u_x, u_y, t_u_x, t_u_y) if i != levels - 1: ys, xs = sources[i + 1].shape u_x, u_y = utils.resample_displacement_field(u_x, u_y, xs, ys) u_x, u_y = utils.resample_displacement_field(u_x, u_y, o_x_size, o_y_size) return u_x, u_y
def single_resolution_local(source, target, level, params): echo = params['echo'] y_size, x_size = source.shape u_x = np.zeros(source.shape) u_y = np.zeros(target.shape) grid_x, grid_y = np.meshgrid(np.arange(x_size), np.arange(y_size)) t_grid_x = grid_x - np.max(grid_x) / 2 t_grid_y = grid_y - np.max(grid_y) / 2 t_grid_y = -t_grid_y outer_iterations = params['outer_iterations'] transformed_source = source.copy() for outer_iteration in range(outer_iterations): if echo: print("Current outer iteration: ", outer_iteration) grad_x, grad_y = gradient_both(transformed_source, target) diff = transformed_source - target transform, offrange = local_transform(transformed_source, target, diff, grad_x, grad_y, t_grid_x, t_grid_y, params) if echo: print("Initial transform calculated.") transform = transform_smoothing(transformed_source, target, transform, diff, grad_x, grad_y, t_grid_x, t_grid_y, offrange, params) if echo: print("Smoothing completed..") t_u_x, t_u_y = transform_to_displacement_field(transform, t_grid_x, t_grid_y) u_x, u_y = utils.compose_vector_fields(u_x, u_y, t_u_x, t_u_y) transformed_source = utils.warp_image(source, u_x, u_y) return u_x, u_y
def callback(self, msg): try: np_arr = np.fromstring(msg.data, np.uint8) img_bgr = cv2.imdecode(np_arr, cv2.IMREAD_COLOR) except CvBridgeError as e: print(e) self.img_hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV) lower_wlane = np.array([25, 5, 240]) upper_wlane = np.array([40, 15, 255]) self.img_wlane = cv2.inRange(self.img_hsv, lower_wlane, upper_wlane) self.img_wlane = cv2.cvtColor(self.img_wlane, cv2.COLOR_GRAY2BGR) img_concat = np.concatenate([img_bgr, self.img_hsv, self.img_wlane], axis=1) img_warp = warp_image(self.img_wlane, self.source_prop) cv2.imshow('mouseRGB', img_warp) cv2.namedWindow('mouseRGB') cv2.setMouseCallback('mouseRGB', self.mouseRGB) cv2.imshow('display', img_concat) cv2.namedWindow('display') cv2.waitKey(1)
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) transforms = [np.eye(3)] for i in range(len(matches)): transform_to_prev, robust_matches = ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1) transforms.append(transform_to_prev @ transforms[-1]) output_shape, offset = get_output_space(imgs[0], imgs[1:], transforms[1:]) imgs_warped = [] for i in range(len(transforms)): img_warped = warp_image(imgs[i], transforms[i], output_shape, offset) img_warped[img_warped == -1] = 0 imgs_warped.append(img_warped) panorama = imgs_warped[0] for img in imgs_warped[1:]: panorama = linear_blend(panorama, img) return panorama
def single_resolution_global(source, target, level, params): echo = params['echo'] y_size, x_size = source.shape u_x = np.zeros(source.shape) u_y = np.zeros(target.shape) grid_x, grid_y = np.meshgrid(np.arange(x_size), np.arange(y_size)) t_grid_x = grid_x - np.max(grid_x) / 2 t_grid_y = grid_y - np.max(grid_y) / 2 t_grid_y = -t_grid_y iterations = params['global_iterations'] transformed_source = source.copy() for iteration in range(int(iterations / 2**(level))): if echo: print("Current global iteration: ", iteration) transform = global_transform(transformed_source, target, t_grid_x, t_grid_y, params) t_u_x, t_u_y = transform_to_displacement_field(transform, t_grid_x, t_grid_y) u_x, u_y = utils.compose_vector_fields(u_x, u_y, t_u_x, t_u_y) transformed_source = utils.warp_image(source, u_x, u_y) return u_x, u_y
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE transforms = [] for i in range(len(imgs) - 1): H = ransac(keypoints[i], keypoints[i + 1], matches[i])[0] transforms.append(H) ''' mid_index = len(imgs)//2 new_transforms_left = [] for i in range(mid_index-1, -1, -1): new_transforms_left.append(np.linalg.inv(transforms[i])) for i in range(1, len(new_transforms_left)): new_transforms_left[i] = new_transforms_left[i-1]@new_transforms_left[i] ''' for i in range(1, len(transforms)): transforms[i] = transforms[i - 1] @ transforms[i] output_shape, offset = get_output_space(imgs[0], imgs[1:], transforms) panorama = imgs[0] panorama = warp_image(panorama, np.eye(3), output_shape, offset) panorama_mask = (panorama != -1) panorama[~panorama_mask] = 0 for i in range(1, len(imgs)): img2_warped = warp_image(imgs[i], transforms[i - 1], output_shape, offset) img2_mask = (img2_warped != -1) img2_warped[~img2_mask] = 0 panorama = linear_blend(panorama, img2_warped) panorama[panorama > 1] = 1 panorama[panorama < 0] = 0 ### END YOUR CODE return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs)-1): mtchs = match_descriptors(descriptors[i], descriptors[i+1], 0.7) matches.append(mtchs) ### YOUR CODE HERE Hs = [] imgs_init = imgs.copy() for i, mtchs in enumerate(matches): H, _ = ransac(keypoints[i], keypoints[i+1], mtchs, threshold=1) Hs.append(H) ref_num = (len(imgs) // 2) ref_img = imgs[ref_num] for i in range(ref_num+1, len(imgs)-1): Hs[i] = np.dot(Hs[i], Hs[i-1]) Hs[ref_num-1] = np.linalg.inv(Hs[ref_num-1]) for i in range(ref_num-2,-1,-1): Hs[i] = np.dot(np.linalg.inv(Hs[i]), Hs[i+1]) del imgs[ref_num] output_shape, offset = get_output_space(ref_img, imgs, Hs) Hs.insert(ref_num, np.eye(3)) imgs_warped = [] for i, img in enumerate(imgs_init): warped = warp_image(img, Hs[i], output_shape, offset) imgs_warped.append(warped) panorama = imgs_warped[0] for war in imgs_warped[1:]: panorama = linear_blend(panorama, war) ### END YOUR CODE return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE print("imgs num:", len(imgs)) Hs = [] for i in range(len(imgs) - 1): H, robust_matches = ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1) Hs.append(H) print("H num:", len(Hs)) print("H shape:", Hs[0].shape) Res = [] images = [] #without middle img mid = len(imgs) // 2 for i in range(len(imgs) - 1): tmp = np.eye(Hs[0].shape[0], Hs[0].shape[1]) if i < mid: for j in range(i, mid): tmp = np.matmul(tmp, np.linalg.inv(Hs[j])) elif i >= mid: for j in range(mid, i + 1): tmp = np.matmul(tmp, Hs[j]) Res.append(tmp) # print(tmp) if i != mid: images.append(imgs[i]) images.append(imgs[-1]) print("Res shape", Res[0].shape) output_shape, offset = get_output_space(imgs[mid], images, Res) img_warpeds = [] img_masks = [] for i in range(len(imgs)): # print(i) if i == mid: img_warp = warp_image(imgs[i], np.eye(3), output_shape, offset) img_mask = (img_warp != -1) img_warp[~img_mask] = 0 img_warpeds.append(img_warp) img_masks.append(img_mask) else: if i > mid: img_warp = warp_image(imgs[i], Res[i - 1], output_shape, offset) else: img_warp = warp_image(imgs[i], Res[i], output_shape, offset) img_mask = (img_warp != -1) img_warp[~img_mask] = 0 img_warpeds.append(img_warp) img_masks.append(img_mask) merged = img_warpeds[0] overlap = img_masks[0] * 1.0 for i in range(1, len(img_warpeds)): merged += img_warpeds[i] overlap += img_masks[i] panorama = merged / np.maximum(overlap, 1) ### END YOUR CODE return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE H = [] for i in range(len(imgs) - 1): H.append(ransac(keypoints[i], keypoints[i + 1], matches[i])[0]) # take left ref_ind = (len(imgs) - 1) // 2 output_shape, offset = get_output_space( imgs[ref_ind], [imgs[0], imgs[2], imgs[3]], [np.linalg.inv(H[0]), H[1], H[1].dot(H[2])]) img1_warped = warp_image(imgs[0], np.linalg.inv(H[0]), output_shape, offset) img1_mask = (img1_warped != -1) img1_warped[~img1_mask] = 0 img2_warped = warp_image(imgs[1], np.eye(3), output_shape, offset) img2_mask = (img2_warped != -1) img2_warped[~img2_mask] = 0 img3_warped = warp_image(imgs[2], H[1], output_shape, offset) img3_mask = (img3_warped != -1) img3_warped[~img3_mask] = 0 img4_warped = warp_image(imgs[3], H[1].dot(H[2]), output_shape, offset) img4_mask = (img4_warped != -1) img4_warped[~img4_mask] = 0 plt.subplot(2, 2, 1) plt.imshow(img1_warped) plt.title('Image 1 warped') plt.axis('off') plt.subplot(2, 2, 2) plt.imshow(img2_warped) plt.title('Image 2 warped') plt.axis('off') plt.subplot(2, 2, 3) plt.imshow(img3_warped) plt.title('Image 3 warped') plt.axis('off') plt.subplot(2, 2, 4) plt.imshow(img4_warped) plt.title('Image 4 warped') plt.axis('off') plt.show() merged2 = img1_warped + img2_warped + img3_warped + img4_warped # Track the overlap by adding the masks together #overlap = (img2_mask * 1.0 + # Multiply by 1.0 for bool -> float conversion # img1_mask + img3_mask + img4_mask) #normalized = merged / np.maximum(overlap, 1) ### END YOUR CODE return merged2
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs)-1): mtchs = match_descriptors(descriptors[i], descriptors[i+1], 0.7) matches.append(mtchs) ### YOUR CODE HERE Hs = [] Hprev = np.eye(3) for i in range(len(imgs)-1): H, robust_matches = ransac(keypoints[i], keypoints[i+1], matches[i], threshold=40) H = Hprev @ np.linalg.inv(H) Hs.append(H) output_shape, offset = get_output_space(imgs[0], imgs[1:], Hs) img_warped = warp_image(imgs[0], np.eye(3), output_shape, offset) img_mask = (img_warped != -1) img_warped[~img_mask] = 0 panorama = img_warped overlap = img_mask for i in range(len(imgs)-1): img_warped = warp_image(imgs[i+1], Hs[i], output_shape, offset) img_mask = (img_warped != -1) img_warped[~img_mask] = 0 panorama += img_warped overlap += img_mask panorama = panorama / np.maximum(overlap, 1) ### END YOUR CODE return panorama
import cv2 import utils as utils IMAGE_REPLACE = "../images/dog.jpeg" cap = cv2.VideoCapture(0) gabarito = cv2.imread("../images/board_aruco.png") foto = utils.read_img(IMAGE_REPLACE, gabarito) corners_or, ids_or = utils.detect_markers(gabarito) while (True): ret, frame = cap.read() corners_dest, ids_dest = utils.detect_markers(frame) if len(corners_dest) > 0: origin, dest = utils.get_points(ids_or, ids_dest, corners_or, corners_dest) warped_img = utils.warp_image(origin, dest, (frame.shape[0], frame.shape[1]), foto) frame = utils.merge_images(warped_img, frame) cv2.namedWindow('frame', cv2.WINDOW_NORMAL) cv2.resizeWindow('frame', 1920, 1080) cv2.imshow('frame', frame) if cv2.waitKey(1) & 0xFF == ord('q'): break cap.release() cv2.destroyAllWindows()
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # # Describe keypoints # descriptors = [] # descriptors[i] corresponds to keypoints[i] # for i, kypnts in enumerate(keypoints): # desc = describe_keypoints(imgs[i], kypnts, # desc_func=desc_func, # patch_size=patch_size) # descriptors.append(desc) # # Match keypoints in neighboring images # matches = [] # matches[i] corresponds to matches between # # descriptors[i] and descriptors[i+1] # for i in range(len(imgs)-1): # mtchs = match_descriptors(descriptors[i], descriptors[i+1], 0.7) # matches.append(mtchs) ### YOUR CODE HERE for i in range(len(imgs)-1): keypoints = [] # keypoints[i] corresponds to imgs[i] descriptors = [] # descriptors[i] corresponds to keypoints[i] for j in range(2): kypnts = corner_peaks(harris_corners(imgs[i+j], window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) desc = describe_keypoints(imgs[i+j], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) mtchs = match_descriptors(descriptors[0], descriptors[1], 0.7) H, robust_matches = ransac(keypoints[0], keypoints[1], mtchs, threshold=1) output_shape, offset = get_output_space(imgs[i], [imgs[i+1]], [H]) img1_warped = warp_image(imgs[i], np.eye(3), output_shape, offset) img1_mask = (img1_warped != -1) # Mask == 1 inside the image img1_warped[~img1_mask] = 0 # Return background values to 0 img2_warped = warp_image(imgs[i+1], H, output_shape, offset) img2_mask = (img2_warped != -1) # Mask == 1 inside the image img2_warped[~img2_mask] = 0 # Return background values to 0 # Merge the warped images using linear blending scheme imgs[i+1] = linear_blend(img1_warped, img2_warped) panorama = imgs[-1] ### END YOUR CODE return panorama
def ct_initial_alignment(source, target, echo=True): y_size, x_size = source.shape source = (source * 255).astype(np.uint8) target = (target * 255).astype(np.uint8) max_size = max(y_size, x_size) smoothing_size = utils.round_up_to_odd(max_size / 2048 * 31) source = cv2.GaussianBlur(source, (smoothing_size, smoothing_size), 0) target = cv2.GaussianBlur(target, (smoothing_size, smoothing_size), 0) ret_source, thresholded_source = fd.threshold_calculation_with_rotation(source) ret_target, thresholded_target = fd.threshold_calculation_with_rotation(target) xs_m = utils.round_up_to_odd(x_size * 20 / 2048) ys_m = utils.round_up_to_odd(y_size * 20 / 2048) struct = min([xs_m, ys_m]) thresholded_source = nd.binary_erosion(thresholded_source, structure=np.ones((struct, struct))).astype(np.uint8) thresholded_source = nd.binary_dilation(thresholded_source, structure=np.ones((struct, struct))).astype(np.uint8) thresholded_target = nd.binary_erosion(thresholded_target, structure=np.ones((struct, struct))).astype(np.uint8) thresholded_target = nd.binary_dilation(thresholded_target, structure=np.ones((struct, struct))).astype(np.uint8) Ms = cv2.moments(thresholded_source) Mt = cv2.moments(thresholded_target) cXs = Ms["m10"] / Ms["m00"] cYs = Ms["m01"] / Ms["m00"] cXt = Mt["m10"] / Mt["m00"] cYt = Mt["m01"] / Mt["m00"] transform_centroid = np.array([ [1, 0, (cXt-cXs)], [0, 1, (cYt-cYs)], [0, 0, 1]]) u_x_t, u_y_t = utils.rigid_dot(source, np.linalg.inv(transform_centroid)) failed = True angle_step = 2 initial_dice = utils.dice(thresholded_source, thresholded_target) if echo: print("Initial dice: ", initial_dice) best_dice = initial_dice for i in range(0, 360, angle_step): if echo: print("Current angle: ", i) rads = i * np.pi/180 matrix_1 = np.array([ [1, 0, cXt], [0, 1, cYt], [0, 0, 1], ]) matrix_i = np.array([ [np.cos(rads), -np.sin(rads), 0], [np.sin(rads), np.cos(rads), 0], [0, 0, 1], ]) matrix_2 = np.array([ [1, 0, -cXt], [0, 1, -cYt], [0, 0, 1], ]) matrix = matrix_1 @ matrix_i @ matrix_2 u_x, u_y = utils.rigid_dot(source, np.linalg.inv(matrix)) transformed_source = utils.warp_image(source, u_x + u_x_t, u_y + u_y_t) ret_transformed_source, thresholded_transformed_source = fd.threshold_calculation_with_threshold_with_rotation(transformed_source, ret_source) thresholded_transformed_source = nd.binary_erosion(thresholded_transformed_source, structure=np.ones((struct, struct))).astype(np.uint8) thresholded_transformed_source = nd.binary_dilation(thresholded_transformed_source, structure=np.ones((struct, struct))).astype(np.uint8) current_dice = utils.dice(thresholded_transformed_source, thresholded_target) if echo: print("Current dice: ", current_dice) if (current_dice > best_dice and current_dice > initial_dice + 0.10 and current_dice > 0.85) or (current_dice > 0.95 and current_dice > best_dice): failed = False best_dice = current_dice transform = matrix.copy() if echo: print("Current best dice: ", best_dice) if failed: transform = np.eye(3) final_transform = transform @ transform_centroid if echo: print("Calculated transform: ", final_transform) if failed: final_transform = np.eye(3) u_x, u_y = utils.rigid_dot(source, np.linalg.inv(final_transform)) return u_x, u_y, final_transform, failed
def example(): source_path = r"/home/mw/MW_Learning/ANHIR_Results/ia_test/6/source.png" target_path = r"/home/mw/MW_Learning/ANHIR_Results/ia_test/6/target_ia.png" source = utils.load_image(source_path) target = utils.load_image(target_path) source, target = utils.resample_both(source, target, 8) source = utils.normalize(source).astype(np.float32) target = utils.normalize(target).astype(np.float32) params = dict() params['echo'] = True params['global_min_size'] = 64 params['global_max_size'] = 512 params['local_min_size'] = 64 params['local_max_size'] = 1024 params['global_iterations'] = 100 params['inner_iterations'] = 15 params['outer_iterations'] = 5 params['L_smooth'] = 1e4 params['L_sigma'] = 1 params['R_sigma'] = 1 params['M_sigma'] = 3 params['x_box'] = 15 params['y_box'] = 15 b_t = time.time() # u_x_g, u_y_g, u_x, u_y = partial_data_registration(target, source, params) u_x_g, u_y_g, u_x, u_y = dm(target, source, params) e_t = time.time() print("Total time: ", e_t - b_t, " seconds.") transformed_target_global = utils.warp_image(target, u_x_g, u_y_g) transformed_target = utils.warp_image(target, u_x, u_y) num_cols = 4 num_rows = 2 plt.figure() plt.subplot(num_rows, num_cols, 1) plt.imshow(source, cmap='gray') plt.title("Source") plt.axis('off') plt.subplot(num_rows, num_cols, 2) plt.imshow(target, cmap='gray') plt.title("Target") plt.axis('off') plt.subplot(num_rows, num_cols, 3) plt.imshow(transformed_target_global, cmap='gray') plt.title("Transformed Target Global") plt.axis('off') plt.subplot(num_rows, num_cols, 4) plt.imshow(transformed_target, cmap='gray') plt.title("Transformed Target") plt.axis('off') plt.subplot(num_rows, num_cols, 5) plt.imshow(np.abs(transformed_target_global - source), cmap='gray', vmin=0, vmax=1) plt.title("Global diff") plt.axis('off') plt.subplot(num_rows, num_cols, 6) plt.imshow(np.abs(transformed_target - source), cmap='gray', vmin=0, vmax=1) plt.title("Global + Local diff") plt.axis('off') plt.subplot(num_rows, num_cols, 7) plt.imshow(np.sqrt(np.square(u_x_g - u_x) + np.square(u_y_g - u_y)), cmap='gray') plt.title("DF diff") plt.axis('off') plt.show()
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] H = [] robust_matches = [] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) # 计算仿射矩阵 h, robust_m = ransac(keypoints[i], keypoints[i + 1], matches[i]) H.append(h) robust_matches.append(robust_m) # 变换到一张图片上,使用第二章图片为参考图片 output_shape, offset = get_output_space( imgs[1], [imgs[0], imgs[2], imgs[3]], [np.linalg.inv(H[0]), H[1], np.dot(H[1], H[2])]) img1_warped = warp_image(imgs[0], np.linalg.inv(H[0]), output_shape, offset) img1_mask = (img1_warped != -1) # Mask == 1 inside the image img1_warped[~img1_mask] = 0 # Return background values to 0 img2_warped = warp_image(img[1], np.eye(3), output_shape, offset) img2_mask = (img2_warped != -1) # Mask == 1 inside the image img2_warped[~img2_mask] = 0 # Return background values to 0 img3_warped = warp_image(img[2], H[1], output_shape, offset) img3_mask = (img3_warped != -1) # Mask == 1 inside the image img3_warped[~img3_mask] = 0 # Return background values to 0 img4_warped = warp_image(imgs[3], np.dot(H[1], H[2]), output_shape, offset) img4_mask = (img4_warped != -1) # Mask == 1 inside the image img4_warped[~img4_mask] = 0 # Return background values to 0 merged = linear_blend(img1_warped, img2_warped) merged = linear_blend(merged, img3_warped) merged = linear_blend(merged, img4_warped) return merged
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE Hs = [np.identity(3)] for i in range(len(imgs) - 1): # calculate transformation matrices betw images except for the last image Hs.append( ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)[0]) for i in range(1, len(imgs)): # combine transformation matrices to go from 1st image to ith image Hs[i] = Hs[i].dot(Hs[i - 1]) output_shape, offset = get_output_space(imgs[0], imgs[1:], Hs[1:]) imgs_warped = [] for i in range(len(imgs)): imgs_warped.append(warp_image(imgs[i], Hs[i], output_shape, offset)) img_mask = (imgs_warped[-1] != -1) imgs_warped[-1][~img_mask] = 0 panorama = imgs_warped[0] for i in range(1, len(imgs)): # build panorama by blending images panorama = linear_blend(panorama, imgs_warped[i]) ### END YOUR CODE return panorama
def detect(fname): image = mpimg.imread(fname + '.jpeg') height, width = image.shape[:2] image = cv2.resize(image, (1280, 720))[:, :, :3] image_original = image kernel_size = 5 img_size = np.shape(image) ht_window = np.uint(img_size[0] / 1.5) hb_window = np.uint(img_size[0]) c_window = np.uint(img_size[1] / 2) ctl_window = c_window - .36 * np.uint(img_size[1] / 2) ctr_window = c_window + .36 * np.uint(img_size[1] / 2) cbl_window = c_window - 0.9 * np.uint(img_size[1] / 2) cbr_window = c_window + 0.9 * np.uint(img_size[1] / 2) src = np.float32([[cbl_window, hb_window], [cbr_window, hb_window], [ctr_window, ht_window], [ctl_window, ht_window]]) dst = np.float32([[0, img_size[0]], [img_size[1], img_size[0]], [img_size[1], 0], [0, 0]]) warped, M_warp, Minv_warp = utils.warp_image(image, src, dst, (img_size[1], img_size[0])) image_HSV = cv2.cvtColor(warped, cv2.COLOR_RGB2HSV) yellow_hsv_low = np.array([0, 100, 100]) yellow_hsv_high = np.array([80, 255, 255]) res_mask = utils.color_mask(image_HSV, yellow_hsv_low, yellow_hsv_high) res = utils.apply_color_mask(image_HSV, warped, yellow_hsv_low, yellow_hsv_high) image_HSV = cv2.cvtColor(warped, cv2.COLOR_RGB2HSV) white_hsv_low = np.array([0, 0, 160]) white_hsv_high = np.array([255, 80, 255]) res1 = utils.apply_color_mask(image_HSV, warped, white_hsv_low, white_hsv_high) mask_yellow = utils.color_mask(image_HSV, yellow_hsv_low, yellow_hsv_high) mask_white = utils.color_mask(image_HSV, white_hsv_low, white_hsv_high) mask_lane = cv2.bitwise_or(mask_yellow, mask_white) image = utils.gaussian_blur(warped, kernel=5) image_HLS = cv2.cvtColor(warped, cv2.COLOR_RGB2HLS) img_gs = image_HLS[:, :, 1] sobel_c = utils.sobel_combined(img_gs) img_abs_x = utils.abs_sobel_thresh(img_gs, 'x', 5, (50, 225)) img_abs_y = utils.abs_sobel_thresh(img_gs, 'y', 5, (50, 225)) wraped2 = np.copy(cv2.bitwise_or(img_abs_x, img_abs_y)) img_gs = image_HLS[:, :, 2] sobel_c = utils.sobel_combined(img_gs) img_abs_x = utils.abs_sobel_thresh(img_gs, 'x', 5, (50, 255)) img_abs_y = utils.abs_sobel_thresh(img_gs, 'y', 5, (50, 255)) wraped3 = np.copy(cv2.bitwise_or(img_abs_x, img_abs_y)) image_cmb = cv2.bitwise_or(wraped2, wraped3) image_cmb = utils.gaussian_blur(image_cmb, 3) image_cmb = cv2.bitwise_or(wraped2, wraped3) image_cmb1 = np.zeros_like(image_cmb) image_cmb1[(mask_lane >= .5) | (image_cmb >= .5)] = 1 mov_filtsize = img_size[1] / 50. mean_lane = np.mean(image_cmb1[img_size[0] / 2:, :], axis=0) indexes = find_peaks_cwt(mean_lane, [100], max_distances=[800]) window_size = 50 val_ind = np.array([mean_lane[indexes[i]] for i in range(len(indexes))]) ind_sorted = np.argsort(-val_ind) ind_peakR = indexes[ind_sorted[0]] ind_peakL = indexes[ind_sorted[1]] if ind_peakR < ind_peakL: ind_temp = ind_peakR ind_peakR = ind_peakL ind_peakL = ind_temp n_vals = 8 ind_min_L = ind_peakL - 50 ind_max_L = ind_peakL + 50 ind_min_R = ind_peakR - 50 ind_max_R = ind_peakR + 50 mask_L_poly = np.zeros_like(image_cmb1) mask_R_poly = np.zeros_like(image_cmb1) ind_peakR_prev = ind_peakR ind_peakL_prev = ind_peakL for i in range(8): img_y1 = img_size[0] - img_size[0] * i / 8 img_y2 = img_size[0] - img_size[0] * (i + 1) / 8 mean_lane_y = np.mean(image_cmb1[img_y2:img_y1, :], axis=0) indexes = find_peaks_cwt(mean_lane_y, [100], max_distances=[800]) if len(indexes) > 1.5: val_ind = np.array( [mean_lane[indexes[i]] for i in range(len(indexes))]) ind_sorted = np.argsort(-val_ind) ind_peakR = indexes[ind_sorted[0]] ind_peakL = indexes[ind_sorted[1]] if ind_peakR < ind_peakL: ind_temp = ind_peakR ind_peakR = ind_peakL ind_peakL = ind_temp else: if len(indexes) == 1: if np.abs(indexes[0] - ind_peakR_prev) < np.abs(indexes[0] - ind_peakL_prev): ind_peakR = indexes[0] ind_peakL = ind_peakL_prev else: ind_peakL = indexes[0] ind_peakR = ind_peakR_prev else: ind_peakL = ind_peakL_prev ind_peakR = ind_peakR_prev if np.abs(ind_peakL - ind_peakL_prev) >= 100: ind_peakL = ind_peakL_prev if np.abs(ind_peakR - ind_peakR_prev) >= 100: ind_peakR = ind_peakR_prev mask_L_poly[img_y2:img_y1, ind_peakL - window_size:ind_peakL + window_size] = 1. mask_R_poly[img_y2:img_y1, ind_peakR - window_size:ind_peakR + window_size] = 1. ind_peakL_prev = ind_peakL ind_peakR_prev = ind_peakR mask_L_poly, mask_R_poly = utils.get_initial_mask(image_cmb1, 50, mean_lane) mask_L = mask_L_poly img_L = np.copy(image_cmb1) img_L = cv2.bitwise_and(img_L, img_L, mask=mask_L_poly) mask_R = mask_R_poly img_R = np.copy(image_cmb1) img_R = cv2.bitwise_and(img_R, img_R, mask=mask_R_poly) vals = np.argwhere(img_L > .5) all_x = vals.T[0] all_y = vals.T[1] left_fit = np.polyfit(all_x, all_y, 2) left_y = np.arange(11) * img_size[0] / 10 left_fitx = left_fit[0] * left_y**2 + left_fit[1] * left_y + left_fit[2] vals = np.argwhere(img_R > .5) all_x = vals.T[0] all_y = vals.T[1] right_fit = np.polyfit(all_x, all_y, 2) right_y = np.arange(11) * img_size[0] / 10 right_fitx = right_fit[0] * right_y**2 + right_fit[ 1] * right_y + right_fit[2] window_sz = 20 mask_L_poly = np.zeros_like(image_cmb1) mask_R_poly = np.zeros_like(image_cmb1) left_pts = [] right_pts = [] pt_y_all = [] for i in range(8): img_y1 = img_size[0] - img_size[0] * i / 8 img_y2 = img_size[0] - img_size[0] * (i + 1) / 8 pt_y = (img_y1 + img_y2) / 2 pt_y_all.append(pt_y) left_pt = np.round(left_fit[0] * pt_y**2 + left_fit[1] * pt_y + left_fit[2]) right_pt = np.round(right_fit[0] * pt_y**2 + right_fit[1] * pt_y + right_fit[2]) right_pts.append(right_fit[0] * pt_y**2 + right_fit[1] * pt_y + right_fit[2]) left_pts.append(left_fit[0] * pt_y**2 + left_fit[1] * pt_y + left_fit[2]) warp_zero = np.zeros_like(image_cmb1).astype(np.uint8) color_warp = np.dstack((warp_zero, warp_zero, warp_zero)) pts_left = np.array([np.transpose(np.vstack([left_fitx, left_y]))]) pts_right = np.array( [np.flipud(np.transpose(np.vstack([right_fitx, right_y])))]) pts = np.hstack((pts_left, pts_right)) cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 255)) col_L = (255, 255, 0) col_R = (255, 255, 255) utils.draw_pw_lines(color_warp, np.int_(pts_left), col_L) utils.draw_pw_lines(color_warp, np.int_(pts_right), col_R) newwarp = cv2.warpPerspective(color_warp, Minv_warp, (image.shape[1], image.shape[0])) result = cv2.addWeighted(image_original, 1, newwarp, 0.5, 0) grid = [] coordinates = [] a = [[left_fitx[i], i * 72] for i in range(0, 11)] b = [[right_fitx[i], i * 72] for i in range(0, 11)] c = np.concatenate([a, b]) c = np.array([c], dtype='float32') coordinates = cv2.perspectiveTransform(c, Minv_warp)[0] return coordinates, result
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE Hs = [] for i in range(len(imgs) - 1): H, _ = ransac(keypoints[i], keypoints[i + 1], matches[i]) if (i == 0): Hs.append(H) else: Hs.append(Hs[i - 1] @ H) # Take image2 as reference image output_shape, offset = get_output_space(imgs[0], imgs[1:], Hs) imgs_warped = [] imgs_mask = [] for i in range(len(imgs)): img_warped = [] if (i == 0): img_warped = warp_image(imgs[i], np.eye(3), output_shape, offset) else: img_warped = warp_image(imgs[i], Hs[i - 1], output_shape, offset) img_mask = (img_warped != -1) img_warped[~img_mask] = 0 imgs_warped.append(img_warped) imgs_mask.append(img_mask) imgs_warped = np.array(imgs_warped) imgs_mask = np.array(imgs_mask) merged = np.sum(imgs_warped, axis=0) overlap = np.sum(imgs_mask * 1.0, axis=0) panorama = merged / np.maximum(overlap, 1) print(panorama.shape) ### END YOUR CODE return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE #transformations H_trans = [np.eye(3)] for i in range(len(imgs) - 1): #get affine matrix and add H_trans.append( ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)[0]) #calc for each pic for i in range(1, len(imgs)): H_trans[i] = H_trans[i].dot(H_trans[i - 1]) output_shape, offset = get_output_space(imgs[0], imgs[1:], H_trans[1:]) warpedImgs = [] for i in range(len(imgs)): warpedImgs.append(warp_image(imgs[i], H_trans[i], output_shape, offset)) img_mask = (warpedImgs[-1] != -1) warpedImgs[-1][~img_mask] = 0 #WIthout linear blend, last image gets messed up #pano_mask = (panorama != -1) #panorama[~pano_mask] = 0 #panorama += warpedImgs[-1] # Track the overlap by adding the masks together #overlap = (pano_mask * 1.0 + img_mask) # Normalize through division by `overlap` - but ensure the minimum is 1 #panorama /= np.maximum(overlap, 1) panorama = warpedImgs[0] for i in range(1, len(imgs)): panorama = linear_blend(panorama, warpedImgs[i]) return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE Hs = [np.eye(3)] for i in range(len(imgs) - 1): Hs.append( ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)[0]) for i in range(1, len(imgs)): Hs[i] = Hs[i].dot( Hs[i - 1] ) # combine multiple transformation matrices by accumulation previous homogeneous matrix output_shape, offset = get_output_space( imgs[0], imgs[1:], Hs[1:]) # get panorama image output shape and offset for i in range(len(imgs)): img_warped = warp_image(imgs[i], Hs[i], output_shape, offset) img_mask = (img_warped != -1) # Mask == 1 inside the image img_warped[~img_mask] = 0 # Return background values to 0 if i == 0: panorama = img_warped else: panorama = linear_blend(panorama, img_warped) pass ### END YOUR CODE return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE HList = [] for i in range(len(imgs) - 1): H, robust_matches = ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1) HList.append(H) ''' p1 = keypoints[i][matches[i][:,0]] p2 = keypoints[i+1][matches[i][:,1]] H = fit_affine_matrix(p1, p2) print(H) HList.append(H) ''' imgWarpedList = [] idxRef = len(imgs) // 2 - 1 imgRef = imgs[idxRef] HMerged = [] for i in range(len(imgs)): if i < idxRef: HRev = np.eye(3) for j in range(idxRef - i): HRev = np.dot(HRev, np.linalg.inv(HList[i + j])) HMerged.append(HRev) output_shape, offset = get_output_space(imgRef, [imgs[i]], [HRev]) imgRef = warp_image(imgRef, np.eye(3), output_shape, offset) #imgWarpedRev = warp_image(img[i], HRev, output_shape, offset) #imgWarpedList.append(imgWarpedRev) elif i > idxRef: H = np.eye(3) for j in range(i - idxRef): H = np.dot(H, HList[i - j - 1]) HMerged.append(H) output_shape, offset = get_output_space(imgRef, [imgs[i]], [H]) imgRef = warp_image(imgRef, np.eye(3), output_shape, offset) #imgWarpedRev = warp_image(img[i], HRev, output_shape, offset) #imgWarpedList.append(imgWarpedRev) imgMaskRef = (imgRef != -1) # Mask == 1 inside the image imgRef[~imgMaskRef] = 0 # Return background values to 0 imgs.pop(idxRef) for i in range(len(imgs)): imgWarped = warp_image(imgs[i], HMerged[i], output_shape, offset) imgMask = (imgWarped != -1) # Mask == 1 inside the image imgWarped[~imgMask] = 0 # Return background values to 0 imgWarpedList.append(imgWarped) #panorama = imgWarpedList[0]+imgWarpedList[1]+imgWarpedList[2]+imgRef imgWarpedList.insert(idxRef, imgRef) merged = imgWarpedList[0] for i in range(len(imgWarpedList) - 1): merged = linear_blend(merged, imgWarpedList[i + 1]) panorama = merged ### END YOUR CODE return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE # np.eye: Return a 2-D array with ones on the diagonal and zeros elsewhere imgs_warped = [] arr = [np.eye(3)] for i in range(len(imgs) - 1): ransacResult = ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1) arr.append(ransacResult[0]) for i in range(1, len(imgs)): arr[i] = arr[i].dot(arr[i - 1]) output_shape, offset = get_output_space(imgs[0], imgs[1:], arr[1:]) for i in range(len(imgs)): warpedImage = warp_image(imgs[i], arr[i], output_shape, offset) imgs_warped.append(warpedImage) # ~ : NOT - inverts all bits img_mask = ~(imgs_warped[-1] != -1) imgs_warped[-1][img_mask] = 0 panorama = imgs_warped[0] # linearly blending the images for i in range(1, len(imgs)): panorama = linear_blend(panorama, imgs_warped[i]) ### END YOUR CODE return panorama
pnt_list_src, pnt_list_dst = find_mtm_matches( anchor=img_anchor_gray, img=img_gray, patch_size=patch_size, alpha=alpha, n_points_to_match=n_points_match_registration, search_radius=search_radius) M, mask = utils.ransac_points(src_pnts=pnt_list_src, dst_pnts=pnt_list_dst, th=5) pnt_list_src_ransac = [m[0] for m in zip(pnt_list_src, mask) if m[1] == 1] pnt_list_dst_ransac = [m[0] for m in zip(pnt_list_dst, mask) if m[1] == 1] img_reg = utils.warp_image(img, np.linalg.inv(M)) registrated_images.append( np.array(img_reg).astype(np.uint8)[:, :, ::-1].copy()) utils.show_matches( src_img=img_anchor, dst_img=img, src_pnts=pnt_list_src_ransac, dst_pnts=pnt_list_dst_ransac, title=f'Match between anchor image {imgsFn[0]} and {imgsFn[i]}') utils.show_blend_images( img1=img_anchor, img2=img, title=f'{imgsFn[0]} and {imgsFn[i]} before registration') utils.show_blend_images( img1=img_anchor,
def anhir_method(source, target, echo=True): ##### Step 0 - Parameters, Smoothing and Initial Resampling ##### b_time_total = time.time() params = dict() params["echo"] = echo params["initial_alignment_size"] = 2048 params["centroid_rotation_size"] = 512 params["nonrigid_registration_size"] = 2048 params["gaussian_divider"] = 1.24 params['nr_method'] = "dm" # pd or dm nr_params = dict() nr_params['echo'] = echo nr_params['global_min_size'] = 64 nr_params['global_max_size'] = 512 nr_params['local_min_size'] = 64 nr_params['local_max_size'] = 768 nr_params['global_iterations'] = 100 nr_params['inner_iterations'] = 15 nr_params['outer_iterations'] = 5 nr_params['L_smooth'] = 1e7 nr_params['L_sigma'] = 1 nr_params['R_sigma'] = 1 nr_params['M_sigma'] = 2 nr_params['x_box'] = 19 nr_params['y_box'] = 19 nr_params['spacing'] = (1.0, 1.0) nr_params['update_mode'] = "composition" nr_params['gradient_mode'] = "symmetric" nr_params['diffusion_sigma'] = (2.0, 2.0) nr_params['fluid_sigma'] = (0.5, 0.5) nr_params['mind_sigma'] = (1.0, 1.0) nr_params['mind_radius'] = (2, 2) nr_params['early_stop'] = 10 return_dict = dict() initial_resample_ratio = utils.calculate_resample_size( source, target, max(params["initial_alignment_size"], params["nonrigid_registration_size"])) source = utils.gaussian_filter( source, initial_resample_ratio / params["gaussian_divider"]) target = utils.gaussian_filter( target, initial_resample_ratio / params["gaussian_divider"]) if echo: print() print("Registration start.") print() print("Source shape: ", source.shape) print("Target shape: ", target.shape) ##### Step 1 - Preprocessing ##### if echo: print() print("Preprocessing start.") print() b_time_r = time.time() p_source, p_target = utils.resample_both(source, target, initial_resample_ratio) e_time_r = time.time() tt_source = p_source.copy() if echo: print("Initially resampled source shape: ", p_source.shape) print("Initially resampled target shape: ", p_target.shape) print("Time for initial resampling: ", e_time_r - b_time_r, " seconds.") b_time_p = time.time() p_source, p_target, t_source, t_target, source_shift, target_shift = pp.preprocess( p_source, p_target, echo) e_time_p = time.time() return_dict["preprocessing_time"] = e_time_p - b_time_p if echo: print("Source shift: ", source_shift) print("Target shift: ", target_shift) print("Preprocessed source shape: ", p_source.shape) print("Preprocessed target shape: ", p_target.shape) print("Time for preprocessing: ", e_time_p - b_time_p, " seconds.") print() print("Preprocessing end.") print() ##### Step 2 - Initial Alignment ##### b_ia_time = time.time() if echo: print("Initial alignment start.") print() ia_resample_ratio = params["nonrigid_registration_size"] / params[ "initial_alignment_size"] to_cv_source, to_cv_target = utils.resample_both(p_source, p_target, ia_resample_ratio) cv_failed = False ct_failed = False ia_failed = False i_u_x, i_u_y, initial_transform, cv_failed = ia.cv_initial_alignment( to_cv_source, to_cv_target, echo) if cv_failed: if echo: print("CV failed.") print() print("CT start.") print() ia_resample_ratio = params["nonrigid_registration_size"] / params[ "centroid_rotation_size"] to_ct_source, to_ct_target = utils.resample_both( p_source, p_target, ia_resample_ratio) i_u_x, i_u_y, initial_transform, ct_failed = ia.ct_initial_alignment( to_ct_source, to_ct_target, echo) if ct_failed: if echo: print() print("CT failed.") print("Initial alignment failed.") ia_failed = True if ia_failed: i_u_x, i_u_y = np.zeros(p_source.shape), np.zeros(p_target.shape) else: y_size, x_size = np.shape(p_source) i_u_x, i_u_y = utils.resample_displacement_field( i_u_x, i_u_y, x_size, y_size) e_ia_time = time.time() return_dict["cv_failed"] = cv_failed return_dict["ct_failed"] = ct_failed return_dict["ia_failed"] = ia_failed return_dict["initial_alignment_time"] = e_ia_time - b_ia_time if echo: print() print("Elapsed time for initial alignment: ", e_ia_time - b_ia_time, " seconds.") print("Initial alignment end.") print() ia_source = utils.warp_image(p_source, i_u_x, i_u_y) u_x_g, u_y_g = nr.partial_data_registration_global(ia_source, p_target, nr_params) u_x_g, u_y_g = utils.compose_vector_fields(i_u_x, i_u_y, u_x_g, u_y_g) ng_source = utils.warp_image(p_source, u_x_g, u_y_g) success = fd.detect_mind_failure(ia_source, p_target, ng_source, echo) if not success: u_x_g, u_y_g = i_u_x, i_u_y ng_source = ia_source return_dict["ng_failed"] = not success ##### Step 3 - Nonrigid Registration ##### b_nr_time = time.time() if echo: print("Nonrigid registration start.") print() if params['nr_method'] == "dm": u_x_nr, u_y_nr = nr.dm(ng_source, p_target, nr_params) u_x_nr, u_y_nr = utils.compose_vector_fields(u_x_g, u_y_g, u_x_nr, u_y_nr) nr_source = utils.warp_image(p_source, u_x_nr, u_y_nr) elif params['nr_method'] == "pd": u_x_nr, u_y_nr = nr.partial_data_registration_local( ng_source, p_target, nr_params) u_x_nr, u_y_nr = utils.compose_vector_fields(u_x_g, u_y_g, u_x_nr, u_y_nr) nr_source = utils.warp_image(p_source, u_x_nr, u_y_nr) e_nr_time = time.time() return_dict["nonrigid_registration_time"] = e_nr_time - b_nr_time if echo: print() print("Elapsed time for nonrigid registration: ", e_nr_time - b_nr_time, " seconds.") print("Nonrigid registration end.") print() ##### Step 4 - Warping function creation ##### def warp_original_landmarks(source_landmarks): source_landmarks = source_landmarks / initial_resample_ratio source_landmarks = utils.pad_landmarks(source_landmarks, target_shift[0], target_shift[2]) source_landmarks = utils.transform_landmarks(source_landmarks, u_x_nr, u_y_nr) source_l_x, source_r_x, source_l_y, source_r_y = source_shift target_l_x, target_r_x, target_l_y, target_r_y = target_shift source_landmarks[:, 0] = source_landmarks[:, 0] - source_l_x source_landmarks[:, 1] = source_landmarks[:, 1] - source_l_y out_y_size, out_x_size = np.shape(source) in_y_size, in_x_size = np.shape(tt_source) source_landmarks[:, 0] = source_landmarks[:, 0] * out_x_size / in_x_size source_landmarks[:, 1] = source_landmarks[:, 1] * out_y_size / in_y_size return source_landmarks def warp_resampled_landmarks(source_landmarks, target_landmarks): source_landmarks = source_landmarks / initial_resample_ratio target_landmarks = target_landmarks / initial_resample_ratio source_landmarks = utils.pad_landmarks(source_landmarks, target_shift[0], target_shift[2]) target_landmarks = utils.pad_landmarks(target_landmarks, source_shift[0], source_shift[2]) transformed_source_landmarks = utils.transform_landmarks( source_landmarks, u_x_nr, u_y_nr) return source_landmarks, transformed_source_landmarks, target_landmarks e_time_total = time.time() return_dict["total_time"] = e_time_total - b_time_total if echo: print("Total registration time: ", e_time_total - b_time_total, " seconds.") print("End of registration.") print() return p_source, p_target, ia_source, ng_source, nr_source, i_u_x, i_u_y, u_x_nr, u_y_nr, warp_resampled_landmarks, warp_original_landmarks, return_dict
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5): """ Stitch an ordered chain of images together. Args: imgs: List of length m containing the ordered chain of m images desc_func: Function that takes in an image patch and outputs a 1D feature vector describing the patch patch_size: Size of square patch at each keypoint Returns: panorama: Final panorma image in coordinate frame of reference image """ # Detect keypoints in each image keypoints = [] # keypoints[i] corresponds to imgs[i] for img in imgs: kypnts = corner_peaks(harris_corners(img, window_size=3), threshold_rel=0.05, exclude_border=8) keypoints.append(kypnts) # Describe keypoints descriptors = [] # descriptors[i] corresponds to keypoints[i] for i, kypnts in enumerate(keypoints): desc = describe_keypoints(imgs[i], kypnts, desc_func=desc_func, patch_size=patch_size) descriptors.append(desc) # Match keypoints in neighboring images matches = [] # matches[i] corresponds to matches between # descriptors[i] and descriptors[i+1] for i in range(len(imgs) - 1): mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7) matches.append(mtchs) ### YOUR CODE HERE Hs = [] robust_matches = [] for i in range(len(imgs) - 1): H, rm = ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1) Hs.append(H) robust_matches.append(rm) idx_ref = len(imgs) // 2 img_ref = imgs[idx_ref] for i in reversed(range(idx_ref)): Hs[i] = np.linalg.inv(Hs[i]) for j in range(i + 1, idx_ref): Hs[i] = Hs[i].dot(np.linalg.inv(Hs[j])) for i in range(idx_ref + 1, len(imgs) - 1): Hs[i] = Hs[i].dot(Hs[i - 1]) others_imgs = imgs[:idx_ref] + imgs[idx_ref + 1:] output_shape, offset = get_output_space(img_ref, others_imgs, Hs) iref_warped = warp_image(img_ref, np.eye(3), output_shape, offset) iref_mask = (iref_warped != -1) # Mask == 1 inside the image iref_warped[~iref_mask] = 0 # Return background values to 0 merged = iref_warped overlap = iref_mask * 1.0 # Multiply by 1.0 for bool -> float conversion for img, H in zip(others_imgs, Hs): img_warped = warp_image(img, H, output_shape, offset) img_mask = (img_warped != -1) # Mask == 1 inside the image img_warped[~img_mask] = 0 # Return background values to 0 merged += img_warped overlap += img_mask panorama = merged / np.maximum(overlap, 1) ### END YOUR CODE return panorama
from utils import get_output_space, warp_image # Extract matched keypoints p1 = keypoints1[matches[:, 0]] p2 = keypoints2[matches[:, 1]] # Find affine transformation matrix H that maps p2 to p1 H = fit_affine_matrix(p1, p2) # 这里的实现还没看懂!!! output_shape, offset = get_output_space(img1, [img2], [H]) # print("Output shape:", output_shape) # print("Offset:", offset) # Warp images into output sapce img1_warped = warp_image(img1, np.eye(3), output_shape, offset) img1_mask = (img1_warped != -1) # Mask == 1 inside the image img1_warped[~img1_mask] = 0 # Return background values to 0 img2_warped = warp_image(img2, H, output_shape, offset) img2_mask = (img2_warped != -1) # Mask == 1 inside the image img2_warped[~img2_mask] = 0 # Return background values to 0 # Plot warped images plt.subplot(1, 2, 1) plt.imshow(img1_warped) plt.title('Image 1 warped') plt.axis('off') plt.subplot(1, 2, 2) plt.imshow(img2_warped)
def partial_data_registration(source, target, params): u_x_g, u_y_g = partial_data_registration_global(source, target, params) source = utils.warp_image(source, u_x_g, u_y_g) u_x_l, u_y_l = partial_data_registration_local(source, target, params) u_x_t, u_y_t = utils.compose_vector_fields(u_x_g, u_y_g, u_x_l, u_y_l) return u_x_g, u_y_g, u_x_t, u_y_t