def partial_data_registration_local(source, target, params):
echo = params['echo']
o_y_size, o_x_size = source.shape
source, target = initial_resample(source, target, params['local_max_size'],
params)
y_size, x_size = source.shape
image_min_size = min(y_size, x_size)
min_size = params['local_min_size']
min_ratio = image_min_size / min_size
levels = int(np.log2(np.floor(min_ratio))) + 1
sources, targets = build_pyramids(source, target, levels)
if echo:
print("Local registration started.")
for i in range(levels):
if echo:
print()
print("Current level: %f/%f" % (i + 1, levels))
print()
current_source = sources[i]
current_target = targets[i]
if i == 0:
u_x, u_y = np.zeros(current_source.shape), np.zeros(
current_source.shape)
if i != 0:
current_source = utils.warp_image(current_source, u_x, u_y)
t_u_x, t_u_y = single_resolution_local(current_source, current_target,
i, params)
u_x, u_y = utils.compose_vector_fields(u_x, u_y, t_u_x, t_u_y)
if i != levels - 1:
ys, xs = sources[i + 1].shape
u_x, u_y = utils.resample_displacement_field(u_x, u_y, xs, ys)
u_x, u_y = utils.resample_displacement_field(u_x, u_y, o_x_size, o_y_size)
return u_x, u_y
def single_resolution_local(source, target, level, params):
echo = params['echo']
y_size, x_size = source.shape
u_x = np.zeros(source.shape)
u_y = np.zeros(target.shape)
grid_x, grid_y = np.meshgrid(np.arange(x_size), np.arange(y_size))
t_grid_x = grid_x - np.max(grid_x) / 2
t_grid_y = grid_y - np.max(grid_y) / 2
t_grid_y = -t_grid_y
outer_iterations = params['outer_iterations']
transformed_source = source.copy()
for outer_iteration in range(outer_iterations):
if echo:
print("Current outer iteration: ", outer_iteration)
grad_x, grad_y = gradient_both(transformed_source, target)
diff = transformed_source - target
transform, offrange = local_transform(transformed_source, target, diff,
grad_x, grad_y, t_grid_x,
t_grid_y, params)
if echo:
print("Initial transform calculated.")
transform = transform_smoothing(transformed_source, target, transform,
diff, grad_x, grad_y, t_grid_x,
t_grid_y, offrange, params)
if echo:
print("Smoothing completed..")
t_u_x, t_u_y = transform_to_displacement_field(transform, t_grid_x,
t_grid_y)
u_x, u_y = utils.compose_vector_fields(u_x, u_y, t_u_x, t_u_y)
transformed_source = utils.warp_image(source, u_x, u_y)
return u_x, u_y
def callback(self, msg):
try:
np_arr = np.fromstring(msg.data, np.uint8)
img_bgr = cv2.imdecode(np_arr, cv2.IMREAD_COLOR)
except CvBridgeError as e:
print(e)
self.img_hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV)
lower_wlane = np.array([25, 5, 240])
upper_wlane = np.array([40, 15, 255])
self.img_wlane = cv2.inRange(self.img_hsv, lower_wlane, upper_wlane)
self.img_wlane = cv2.cvtColor(self.img_wlane, cv2.COLOR_GRAY2BGR)
img_concat = np.concatenate([img_bgr, self.img_hsv, self.img_wlane],
axis=1)
img_warp = warp_image(self.img_wlane, self.source_prop)
cv2.imshow('mouseRGB', img_warp)
cv2.namedWindow('mouseRGB')
cv2.setMouseCallback('mouseRGB', self.mouseRGB)
cv2.imshow('display', img_concat)
cv2.namedWindow('display')
cv2.waitKey(1)
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
transforms = [np.eye(3)]
for i in range(len(matches)):
transform_to_prev, robust_matches = ransac(keypoints[i],
keypoints[i + 1],
matches[i],
threshold=1)
transforms.append(transform_to_prev @ transforms[-1])
output_shape, offset = get_output_space(imgs[0], imgs[1:], transforms[1:])
imgs_warped = []
for i in range(len(transforms)):
img_warped = warp_image(imgs[i], transforms[i], output_shape, offset)
img_warped[img_warped == -1] = 0
imgs_warped.append(img_warped)
panorama = imgs_warped[0]
for img in imgs_warped[1:]:
panorama = linear_blend(panorama, img)
return panorama
def single_resolution_global(source, target, level, params):
echo = params['echo']
y_size, x_size = source.shape
u_x = np.zeros(source.shape)
u_y = np.zeros(target.shape)
grid_x, grid_y = np.meshgrid(np.arange(x_size), np.arange(y_size))
t_grid_x = grid_x - np.max(grid_x) / 2
t_grid_y = grid_y - np.max(grid_y) / 2
t_grid_y = -t_grid_y
iterations = params['global_iterations']
transformed_source = source.copy()
for iteration in range(int(iterations / 2**(level))):
if echo:
print("Current global iteration: ", iteration)
transform = global_transform(transformed_source, target, t_grid_x,
t_grid_y, params)
t_u_x, t_u_y = transform_to_displacement_field(transform, t_grid_x,
t_grid_y)
u_x, u_y = utils.compose_vector_fields(u_x, u_y, t_u_x, t_u_y)
transformed_source = utils.warp_image(source, u_x, u_y)
return u_x, u_y
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
transforms = []
for i in range(len(imgs) - 1):
H = ransac(keypoints[i], keypoints[i + 1], matches[i])[0]
transforms.append(H)
'''
mid_index = len(imgs)//2
new_transforms_left = []
for i in range(mid_index-1, -1, -1):
new_transforms_left.append(np.linalg.inv(transforms[i]))
for i in range(1, len(new_transforms_left)):
new_transforms_left[i] = new_transforms_left[i-1]@new_transforms_left[i]
'''
for i in range(1, len(transforms)):
transforms[i] = transforms[i - 1] @ transforms[i]
output_shape, offset = get_output_space(imgs[0], imgs[1:], transforms)
panorama = imgs[0]
panorama = warp_image(panorama, np.eye(3), output_shape, offset)
panorama_mask = (panorama != -1)
panorama[~panorama_mask] = 0
for i in range(1, len(imgs)):
img2_warped = warp_image(imgs[i], transforms[i - 1], output_shape,
offset)
img2_mask = (img2_warped != -1)
img2_warped[~img2_mask] = 0
panorama = linear_blend(panorama, img2_warped)
panorama[panorama > 1] = 1
panorama[panorama < 0] = 0
### END YOUR CODE
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i], kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs)-1):
mtchs = match_descriptors(descriptors[i], descriptors[i+1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
Hs = []
imgs_init = imgs.copy()
for i, mtchs in enumerate(matches):
H, _ = ransac(keypoints[i], keypoints[i+1], mtchs, threshold=1)
Hs.append(H)
ref_num = (len(imgs) // 2)
ref_img = imgs[ref_num]
for i in range(ref_num+1, len(imgs)-1):
Hs[i] = np.dot(Hs[i], Hs[i-1])
Hs[ref_num-1] = np.linalg.inv(Hs[ref_num-1])
for i in range(ref_num-2,-1,-1):
Hs[i] = np.dot(np.linalg.inv(Hs[i]), Hs[i+1])
del imgs[ref_num]
output_shape, offset = get_output_space(ref_img, imgs, Hs)
Hs.insert(ref_num, np.eye(3))
imgs_warped = []
for i, img in enumerate(imgs_init):
warped = warp_image(img, Hs[i], output_shape, offset)
imgs_warped.append(warped)
panorama = imgs_warped[0]
for war in imgs_warped[1:]:
panorama = linear_blend(panorama, war)
### END YOUR CODE
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
print("imgs num:", len(imgs))
Hs = []
for i in range(len(imgs) - 1):
H, robust_matches = ransac(keypoints[i],
keypoints[i + 1],
matches[i],
threshold=1)
Hs.append(H)
print("H num:", len(Hs))
print("H shape:", Hs[0].shape)
Res = []
images = [] #without middle img
mid = len(imgs) // 2
for i in range(len(imgs) - 1):
tmp = np.eye(Hs[0].shape[0], Hs[0].shape[1])
if i < mid:
for j in range(i, mid):
tmp = np.matmul(tmp, np.linalg.inv(Hs[j]))
elif i >= mid:
for j in range(mid, i + 1):
tmp = np.matmul(tmp, Hs[j])
Res.append(tmp)
# print(tmp)
if i != mid:
images.append(imgs[i])
images.append(imgs[-1])
print("Res shape", Res[0].shape)
output_shape, offset = get_output_space(imgs[mid], images, Res)
img_warpeds = []
img_masks = []
for i in range(len(imgs)):
# print(i)
if i == mid:
img_warp = warp_image(imgs[i], np.eye(3), output_shape, offset)
img_mask = (img_warp != -1)
img_warp[~img_mask] = 0
img_warpeds.append(img_warp)
img_masks.append(img_mask)
else:
if i > mid:
img_warp = warp_image(imgs[i], Res[i - 1], output_shape,
offset)
else:
img_warp = warp_image(imgs[i], Res[i], output_shape, offset)
img_mask = (img_warp != -1)
img_warp[~img_mask] = 0
img_warpeds.append(img_warp)
img_masks.append(img_mask)
merged = img_warpeds[0]
overlap = img_masks[0] * 1.0
for i in range(1, len(img_warpeds)):
merged += img_warpeds[i]
overlap += img_masks[i]
panorama = merged / np.maximum(overlap, 1)
### END YOUR CODE
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
H = []
for i in range(len(imgs) - 1):
H.append(ransac(keypoints[i], keypoints[i + 1], matches[i])[0])
# take left
ref_ind = (len(imgs) - 1) // 2
output_shape, offset = get_output_space(
imgs[ref_ind], [imgs[0], imgs[2], imgs[3]],
[np.linalg.inv(H[0]), H[1], H[1].dot(H[2])])
img1_warped = warp_image(imgs[0], np.linalg.inv(H[0]), output_shape,
offset)
img1_mask = (img1_warped != -1)
img1_warped[~img1_mask] = 0
img2_warped = warp_image(imgs[1], np.eye(3), output_shape, offset)
img2_mask = (img2_warped != -1)
img2_warped[~img2_mask] = 0
img3_warped = warp_image(imgs[2], H[1], output_shape, offset)
img3_mask = (img3_warped != -1)
img3_warped[~img3_mask] = 0
img4_warped = warp_image(imgs[3], H[1].dot(H[2]), output_shape, offset)
img4_mask = (img4_warped != -1)
img4_warped[~img4_mask] = 0
plt.subplot(2, 2, 1)
plt.imshow(img1_warped)
plt.title('Image 1 warped')
plt.axis('off')
plt.subplot(2, 2, 2)
plt.imshow(img2_warped)
plt.title('Image 2 warped')
plt.axis('off')
plt.subplot(2, 2, 3)
plt.imshow(img3_warped)
plt.title('Image 3 warped')
plt.axis('off')
plt.subplot(2, 2, 4)
plt.imshow(img4_warped)
plt.title('Image 4 warped')
plt.axis('off')
plt.show()
merged2 = img1_warped + img2_warped + img3_warped + img4_warped
# Track the overlap by adding the masks together
#overlap = (img2_mask * 1.0 + # Multiply by 1.0 for bool -> float conversion
# img1_mask + img3_mask + img4_mask)
#normalized = merged / np.maximum(overlap, 1)
### END YOUR CODE
return merged2
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i], kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs)-1):
mtchs = match_descriptors(descriptors[i], descriptors[i+1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
Hs = []
Hprev = np.eye(3)
for i in range(len(imgs)-1):
H, robust_matches = ransac(keypoints[i], keypoints[i+1], matches[i], threshold=40)
H = Hprev @ np.linalg.inv(H)
Hs.append(H)
output_shape, offset = get_output_space(imgs[0], imgs[1:], Hs)
img_warped = warp_image(imgs[0], np.eye(3), output_shape, offset)
img_mask = (img_warped != -1)
img_warped[~img_mask] = 0
panorama = img_warped
overlap = img_mask
for i in range(len(imgs)-1):
img_warped = warp_image(imgs[i+1], Hs[i], output_shape, offset)
img_mask = (img_warped != -1)
img_warped[~img_mask] = 0
panorama += img_warped
overlap += img_mask
panorama = panorama / np.maximum(overlap, 1)
### END YOUR CODE
return panorama
import cv2
import utils as utils
IMAGE_REPLACE = "../images/dog.jpeg"
cap = cv2.VideoCapture(0)
gabarito = cv2.imread("../images/board_aruco.png")
foto = utils.read_img(IMAGE_REPLACE, gabarito)
corners_or, ids_or = utils.detect_markers(gabarito)
while (True):
ret, frame = cap.read()
corners_dest, ids_dest = utils.detect_markers(frame)
if len(corners_dest) > 0:
origin, dest = utils.get_points(ids_or, ids_dest, corners_or,
corners_dest)
warped_img = utils.warp_image(origin, dest,
(frame.shape[0], frame.shape[1]), foto)
frame = utils.merge_images(warped_img, frame)
cv2.namedWindow('frame', cv2.WINDOW_NORMAL)
cv2.resizeWindow('frame', 1920, 1080)
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cap.release()
cv2.destroyAllWindows()
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# # Describe keypoints
# descriptors = [] # descriptors[i] corresponds to keypoints[i]
# for i, kypnts in enumerate(keypoints):
# desc = describe_keypoints(imgs[i], kypnts,
# desc_func=desc_func,
# patch_size=patch_size)
# descriptors.append(desc)
# # Match keypoints in neighboring images
# matches = [] # matches[i] corresponds to matches between
# # descriptors[i] and descriptors[i+1]
# for i in range(len(imgs)-1):
# mtchs = match_descriptors(descriptors[i], descriptors[i+1], 0.7)
# matches.append(mtchs)
### YOUR CODE HERE
for i in range(len(imgs)-1):
keypoints = [] # keypoints[i] corresponds to imgs[i]
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for j in range(2):
kypnts = corner_peaks(harris_corners(imgs[i+j], window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
desc = describe_keypoints(imgs[i+j], kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
mtchs = match_descriptors(descriptors[0], descriptors[1], 0.7)
H, robust_matches = ransac(keypoints[0], keypoints[1], mtchs, threshold=1)
output_shape, offset = get_output_space(imgs[i], [imgs[i+1]], [H])
img1_warped = warp_image(imgs[i], np.eye(3), output_shape, offset)
img1_mask = (img1_warped != -1) # Mask == 1 inside the image
img1_warped[~img1_mask] = 0 # Return background values to 0
img2_warped = warp_image(imgs[i+1], H, output_shape, offset)
img2_mask = (img2_warped != -1) # Mask == 1 inside the image
img2_warped[~img2_mask] = 0 # Return background values to 0
# Merge the warped images using linear blending scheme
imgs[i+1] = linear_blend(img1_warped, img2_warped)
panorama = imgs[-1]
### END YOUR CODE
return panorama
def ct_initial_alignment(source, target, echo=True):
y_size, x_size = source.shape
source = (source * 255).astype(np.uint8)
target = (target * 255).astype(np.uint8)
max_size = max(y_size, x_size)
smoothing_size = utils.round_up_to_odd(max_size / 2048 * 31)
source = cv2.GaussianBlur(source, (smoothing_size, smoothing_size), 0)
target = cv2.GaussianBlur(target, (smoothing_size, smoothing_size), 0)
ret_source, thresholded_source = fd.threshold_calculation_with_rotation(source)
ret_target, thresholded_target = fd.threshold_calculation_with_rotation(target)
xs_m = utils.round_up_to_odd(x_size * 20 / 2048)
ys_m = utils.round_up_to_odd(y_size * 20 / 2048)
struct = min([xs_m, ys_m])
thresholded_source = nd.binary_erosion(thresholded_source, structure=np.ones((struct, struct))).astype(np.uint8)
thresholded_source = nd.binary_dilation(thresholded_source, structure=np.ones((struct, struct))).astype(np.uint8)
thresholded_target = nd.binary_erosion(thresholded_target, structure=np.ones((struct, struct))).astype(np.uint8)
thresholded_target = nd.binary_dilation(thresholded_target, structure=np.ones((struct, struct))).astype(np.uint8)
Ms = cv2.moments(thresholded_source)
Mt = cv2.moments(thresholded_target)
cXs = Ms["m10"] / Ms["m00"]
cYs = Ms["m01"] / Ms["m00"]
cXt = Mt["m10"] / Mt["m00"]
cYt = Mt["m01"] / Mt["m00"]
transform_centroid = np.array([
[1, 0, (cXt-cXs)],
[0, 1, (cYt-cYs)],
[0, 0, 1]])
u_x_t, u_y_t = utils.rigid_dot(source, np.linalg.inv(transform_centroid))
failed = True
angle_step = 2
initial_dice = utils.dice(thresholded_source, thresholded_target)
if echo:
print("Initial dice: ", initial_dice)
best_dice = initial_dice
for i in range(0, 360, angle_step):
if echo:
print("Current angle: ", i)
rads = i * np.pi/180
matrix_1 = np.array([
[1, 0, cXt],
[0, 1, cYt],
[0, 0, 1],
])
matrix_i = np.array([
[np.cos(rads), -np.sin(rads), 0],
[np.sin(rads), np.cos(rads), 0],
[0, 0, 1],
])
matrix_2 = np.array([
[1, 0, -cXt],
[0, 1, -cYt],
[0, 0, 1],
])
matrix = matrix_1 @ matrix_i @ matrix_2
u_x, u_y = utils.rigid_dot(source, np.linalg.inv(matrix))
transformed_source = utils.warp_image(source, u_x + u_x_t, u_y + u_y_t)
ret_transformed_source, thresholded_transformed_source = fd.threshold_calculation_with_threshold_with_rotation(transformed_source, ret_source)
thresholded_transformed_source = nd.binary_erosion(thresholded_transformed_source, structure=np.ones((struct, struct))).astype(np.uint8)
thresholded_transformed_source = nd.binary_dilation(thresholded_transformed_source, structure=np.ones((struct, struct))).astype(np.uint8)
current_dice = utils.dice(thresholded_transformed_source, thresholded_target)
if echo:
print("Current dice: ", current_dice)
if (current_dice > best_dice and current_dice > initial_dice + 0.10 and current_dice > 0.85) or (current_dice > 0.95 and current_dice > best_dice):
failed = False
best_dice = current_dice
transform = matrix.copy()
if echo:
print("Current best dice: ", best_dice)
if failed:
transform = np.eye(3)
final_transform = transform @ transform_centroid
if echo:
print("Calculated transform: ", final_transform)
if failed:
final_transform = np.eye(3)
u_x, u_y = utils.rigid_dot(source, np.linalg.inv(final_transform))
return u_x, u_y, final_transform, failed
def example():
source_path = r"/home/mw/MW_Learning/ANHIR_Results/ia_test/6/source.png"
target_path = r"/home/mw/MW_Learning/ANHIR_Results/ia_test/6/target_ia.png"
source = utils.load_image(source_path)
target = utils.load_image(target_path)
source, target = utils.resample_both(source, target, 8)
source = utils.normalize(source).astype(np.float32)
target = utils.normalize(target).astype(np.float32)
params = dict()
params['echo'] = True
params['global_min_size'] = 64
params['global_max_size'] = 512
params['local_min_size'] = 64
params['local_max_size'] = 1024
params['global_iterations'] = 100
params['inner_iterations'] = 15
params['outer_iterations'] = 5
params['L_smooth'] = 1e4
params['L_sigma'] = 1
params['R_sigma'] = 1
params['M_sigma'] = 3
params['x_box'] = 15
params['y_box'] = 15
b_t = time.time()
# u_x_g, u_y_g, u_x, u_y = partial_data_registration(target, source, params)
u_x_g, u_y_g, u_x, u_y = dm(target, source, params)
e_t = time.time()
print("Total time: ", e_t - b_t, " seconds.")
transformed_target_global = utils.warp_image(target, u_x_g, u_y_g)
transformed_target = utils.warp_image(target, u_x, u_y)
num_cols = 4
num_rows = 2
plt.figure()
plt.subplot(num_rows, num_cols, 1)
plt.imshow(source, cmap='gray')
plt.title("Source")
plt.axis('off')
plt.subplot(num_rows, num_cols, 2)
plt.imshow(target, cmap='gray')
plt.title("Target")
plt.axis('off')
plt.subplot(num_rows, num_cols, 3)
plt.imshow(transformed_target_global, cmap='gray')
plt.title("Transformed Target Global")
plt.axis('off')
plt.subplot(num_rows, num_cols, 4)
plt.imshow(transformed_target, cmap='gray')
plt.title("Transformed Target")
plt.axis('off')
plt.subplot(num_rows, num_cols, 5)
plt.imshow(np.abs(transformed_target_global - source),
cmap='gray',
vmin=0,
vmax=1)
plt.title("Global diff")
plt.axis('off')
plt.subplot(num_rows, num_cols, 6)
plt.imshow(np.abs(transformed_target - source),
cmap='gray',
vmin=0,
vmax=1)
plt.title("Global + Local diff")
plt.axis('off')
plt.subplot(num_rows, num_cols, 7)
plt.imshow(np.sqrt(np.square(u_x_g - u_x) + np.square(u_y_g - u_y)),
cmap='gray')
plt.title("DF diff")
plt.axis('off')
plt.show()
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
H = []
robust_matches = []
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
# 计算仿射矩阵
h, robust_m = ransac(keypoints[i], keypoints[i + 1], matches[i])
H.append(h)
robust_matches.append(robust_m)
# 变换到一张图片上,使用第二章图片为参考图片
output_shape, offset = get_output_space(
imgs[1], [imgs[0], imgs[2], imgs[3]],
[np.linalg.inv(H[0]), H[1],
np.dot(H[1], H[2])])
img1_warped = warp_image(imgs[0], np.linalg.inv(H[0]), output_shape,
offset)
img1_mask = (img1_warped != -1) # Mask == 1 inside the image
img1_warped[~img1_mask] = 0 # Return background values to 0
img2_warped = warp_image(img[1], np.eye(3), output_shape, offset)
img2_mask = (img2_warped != -1) # Mask == 1 inside the image
img2_warped[~img2_mask] = 0 # Return background values to 0
img3_warped = warp_image(img[2], H[1], output_shape, offset)
img3_mask = (img3_warped != -1) # Mask == 1 inside the image
img3_warped[~img3_mask] = 0 # Return background values to 0
img4_warped = warp_image(imgs[3], np.dot(H[1], H[2]), output_shape, offset)
img4_mask = (img4_warped != -1) # Mask == 1 inside the image
img4_warped[~img4_mask] = 0 # Return background values to 0
merged = linear_blend(img1_warped, img2_warped)
merged = linear_blend(merged, img3_warped)
merged = linear_blend(merged, img4_warped)
return merged
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
Hs = [np.identity(3)]
for i in range(len(imgs) - 1):
# calculate transformation matrices betw images except for the last image
Hs.append(
ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)[0])
for i in range(1, len(imgs)):
# combine transformation matrices to go from 1st image to ith image
Hs[i] = Hs[i].dot(Hs[i - 1])
output_shape, offset = get_output_space(imgs[0], imgs[1:], Hs[1:])
imgs_warped = []
for i in range(len(imgs)):
imgs_warped.append(warp_image(imgs[i], Hs[i], output_shape, offset))
img_mask = (imgs_warped[-1] != -1)
imgs_warped[-1][~img_mask] = 0
panorama = imgs_warped[0]
for i in range(1, len(imgs)):
# build panorama by blending images
panorama = linear_blend(panorama, imgs_warped[i])
### END YOUR CODE
return panorama
def detect(fname):
image = mpimg.imread(fname + '.jpeg')
height, width = image.shape[:2]
image = cv2.resize(image, (1280, 720))[:, :, :3]
image_original = image
kernel_size = 5
img_size = np.shape(image)
ht_window = np.uint(img_size[0] / 1.5)
hb_window = np.uint(img_size[0])
c_window = np.uint(img_size[1] / 2)
ctl_window = c_window - .36 * np.uint(img_size[1] / 2)
ctr_window = c_window + .36 * np.uint(img_size[1] / 2)
cbl_window = c_window - 0.9 * np.uint(img_size[1] / 2)
cbr_window = c_window + 0.9 * np.uint(img_size[1] / 2)
src = np.float32([[cbl_window, hb_window], [cbr_window, hb_window],
[ctr_window, ht_window], [ctl_window, ht_window]])
dst = np.float32([[0, img_size[0]], [img_size[1], img_size[0]],
[img_size[1], 0], [0, 0]])
warped, M_warp, Minv_warp = utils.warp_image(image, src, dst,
(img_size[1], img_size[0]))
image_HSV = cv2.cvtColor(warped, cv2.COLOR_RGB2HSV)
yellow_hsv_low = np.array([0, 100, 100])
yellow_hsv_high = np.array([80, 255, 255])
res_mask = utils.color_mask(image_HSV, yellow_hsv_low, yellow_hsv_high)
res = utils.apply_color_mask(image_HSV, warped, yellow_hsv_low,
yellow_hsv_high)
image_HSV = cv2.cvtColor(warped, cv2.COLOR_RGB2HSV)
white_hsv_low = np.array([0, 0, 160])
white_hsv_high = np.array([255, 80, 255])
res1 = utils.apply_color_mask(image_HSV, warped, white_hsv_low,
white_hsv_high)
mask_yellow = utils.color_mask(image_HSV, yellow_hsv_low, yellow_hsv_high)
mask_white = utils.color_mask(image_HSV, white_hsv_low, white_hsv_high)
mask_lane = cv2.bitwise_or(mask_yellow, mask_white)
image = utils.gaussian_blur(warped, kernel=5)
image_HLS = cv2.cvtColor(warped, cv2.COLOR_RGB2HLS)
img_gs = image_HLS[:, :, 1]
sobel_c = utils.sobel_combined(img_gs)
img_abs_x = utils.abs_sobel_thresh(img_gs, 'x', 5, (50, 225))
img_abs_y = utils.abs_sobel_thresh(img_gs, 'y', 5, (50, 225))
wraped2 = np.copy(cv2.bitwise_or(img_abs_x, img_abs_y))
img_gs = image_HLS[:, :, 2]
sobel_c = utils.sobel_combined(img_gs)
img_abs_x = utils.abs_sobel_thresh(img_gs, 'x', 5, (50, 255))
img_abs_y = utils.abs_sobel_thresh(img_gs, 'y', 5, (50, 255))
wraped3 = np.copy(cv2.bitwise_or(img_abs_x, img_abs_y))
image_cmb = cv2.bitwise_or(wraped2, wraped3)
image_cmb = utils.gaussian_blur(image_cmb, 3)
image_cmb = cv2.bitwise_or(wraped2, wraped3)
image_cmb1 = np.zeros_like(image_cmb)
image_cmb1[(mask_lane >= .5) | (image_cmb >= .5)] = 1
mov_filtsize = img_size[1] / 50.
mean_lane = np.mean(image_cmb1[img_size[0] / 2:, :], axis=0)
indexes = find_peaks_cwt(mean_lane, [100], max_distances=[800])
window_size = 50
val_ind = np.array([mean_lane[indexes[i]] for i in range(len(indexes))])
ind_sorted = np.argsort(-val_ind)
ind_peakR = indexes[ind_sorted[0]]
ind_peakL = indexes[ind_sorted[1]]
if ind_peakR < ind_peakL:
ind_temp = ind_peakR
ind_peakR = ind_peakL
ind_peakL = ind_temp
n_vals = 8
ind_min_L = ind_peakL - 50
ind_max_L = ind_peakL + 50
ind_min_R = ind_peakR - 50
ind_max_R = ind_peakR + 50
mask_L_poly = np.zeros_like(image_cmb1)
mask_R_poly = np.zeros_like(image_cmb1)
ind_peakR_prev = ind_peakR
ind_peakL_prev = ind_peakL
for i in range(8):
img_y1 = img_size[0] - img_size[0] * i / 8
img_y2 = img_size[0] - img_size[0] * (i + 1) / 8
mean_lane_y = np.mean(image_cmb1[img_y2:img_y1, :], axis=0)
indexes = find_peaks_cwt(mean_lane_y, [100], max_distances=[800])
if len(indexes) > 1.5:
val_ind = np.array(
[mean_lane[indexes[i]] for i in range(len(indexes))])
ind_sorted = np.argsort(-val_ind)
ind_peakR = indexes[ind_sorted[0]]
ind_peakL = indexes[ind_sorted[1]]
if ind_peakR < ind_peakL:
ind_temp = ind_peakR
ind_peakR = ind_peakL
ind_peakL = ind_temp
else:
if len(indexes) == 1:
if np.abs(indexes[0] -
ind_peakR_prev) < np.abs(indexes[0] -
ind_peakL_prev):
ind_peakR = indexes[0]
ind_peakL = ind_peakL_prev
else:
ind_peakL = indexes[0]
ind_peakR = ind_peakR_prev
else:
ind_peakL = ind_peakL_prev
ind_peakR = ind_peakR_prev
if np.abs(ind_peakL - ind_peakL_prev) >= 100:
ind_peakL = ind_peakL_prev
if np.abs(ind_peakR - ind_peakR_prev) >= 100:
ind_peakR = ind_peakR_prev
mask_L_poly[img_y2:img_y1,
ind_peakL - window_size:ind_peakL + window_size] = 1.
mask_R_poly[img_y2:img_y1,
ind_peakR - window_size:ind_peakR + window_size] = 1.
ind_peakL_prev = ind_peakL
ind_peakR_prev = ind_peakR
mask_L_poly, mask_R_poly = utils.get_initial_mask(image_cmb1, 50,
mean_lane)
mask_L = mask_L_poly
img_L = np.copy(image_cmb1)
img_L = cv2.bitwise_and(img_L, img_L, mask=mask_L_poly)
mask_R = mask_R_poly
img_R = np.copy(image_cmb1)
img_R = cv2.bitwise_and(img_R, img_R, mask=mask_R_poly)
vals = np.argwhere(img_L > .5)
all_x = vals.T[0]
all_y = vals.T[1]
left_fit = np.polyfit(all_x, all_y, 2)
left_y = np.arange(11) * img_size[0] / 10
left_fitx = left_fit[0] * left_y**2 + left_fit[1] * left_y + left_fit[2]
vals = np.argwhere(img_R > .5)
all_x = vals.T[0]
all_y = vals.T[1]
right_fit = np.polyfit(all_x, all_y, 2)
right_y = np.arange(11) * img_size[0] / 10
right_fitx = right_fit[0] * right_y**2 + right_fit[
1] * right_y + right_fit[2]
window_sz = 20
mask_L_poly = np.zeros_like(image_cmb1)
mask_R_poly = np.zeros_like(image_cmb1)
left_pts = []
right_pts = []
pt_y_all = []
for i in range(8):
img_y1 = img_size[0] - img_size[0] * i / 8
img_y2 = img_size[0] - img_size[0] * (i + 1) / 8
pt_y = (img_y1 + img_y2) / 2
pt_y_all.append(pt_y)
left_pt = np.round(left_fit[0] * pt_y**2 + left_fit[1] * pt_y +
left_fit[2])
right_pt = np.round(right_fit[0] * pt_y**2 + right_fit[1] * pt_y +
right_fit[2])
right_pts.append(right_fit[0] * pt_y**2 + right_fit[1] * pt_y +
right_fit[2])
left_pts.append(left_fit[0] * pt_y**2 + left_fit[1] * pt_y +
left_fit[2])
warp_zero = np.zeros_like(image_cmb1).astype(np.uint8)
color_warp = np.dstack((warp_zero, warp_zero, warp_zero))
pts_left = np.array([np.transpose(np.vstack([left_fitx, left_y]))])
pts_right = np.array(
[np.flipud(np.transpose(np.vstack([right_fitx, right_y])))])
pts = np.hstack((pts_left, pts_right))
cv2.fillPoly(color_warp, np.int_([pts]), (0, 255, 255))
col_L = (255, 255, 0)
col_R = (255, 255, 255)
utils.draw_pw_lines(color_warp, np.int_(pts_left), col_L)
utils.draw_pw_lines(color_warp, np.int_(pts_right), col_R)
newwarp = cv2.warpPerspective(color_warp, Minv_warp,
(image.shape[1], image.shape[0]))
result = cv2.addWeighted(image_original, 1, newwarp, 0.5, 0)
grid = []
coordinates = []
a = [[left_fitx[i], i * 72] for i in range(0, 11)]
b = [[right_fitx[i], i * 72] for i in range(0, 11)]
c = np.concatenate([a, b])
c = np.array([c], dtype='float32')
coordinates = cv2.perspectiveTransform(c, Minv_warp)[0]
return coordinates, result
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
Hs = []
for i in range(len(imgs) - 1):
H, _ = ransac(keypoints[i], keypoints[i + 1], matches[i])
if (i == 0):
Hs.append(H)
else:
Hs.append(Hs[i - 1] @ H)
# Take image2 as reference image
output_shape, offset = get_output_space(imgs[0], imgs[1:], Hs)
imgs_warped = []
imgs_mask = []
for i in range(len(imgs)):
img_warped = []
if (i == 0):
img_warped = warp_image(imgs[i], np.eye(3), output_shape, offset)
else:
img_warped = warp_image(imgs[i], Hs[i - 1], output_shape, offset)
img_mask = (img_warped != -1)
img_warped[~img_mask] = 0
imgs_warped.append(img_warped)
imgs_mask.append(img_mask)
imgs_warped = np.array(imgs_warped)
imgs_mask = np.array(imgs_mask)
merged = np.sum(imgs_warped, axis=0)
overlap = np.sum(imgs_mask * 1.0, axis=0)
panorama = merged / np.maximum(overlap, 1)
print(panorama.shape)
### END YOUR CODE
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
#transformations
H_trans = [np.eye(3)]
for i in range(len(imgs) - 1):
#get affine matrix and add
H_trans.append(
ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)[0])
#calc for each pic
for i in range(1, len(imgs)):
H_trans[i] = H_trans[i].dot(H_trans[i - 1])
output_shape, offset = get_output_space(imgs[0], imgs[1:], H_trans[1:])
warpedImgs = []
for i in range(len(imgs)):
warpedImgs.append(warp_image(imgs[i], H_trans[i], output_shape,
offset))
img_mask = (warpedImgs[-1] != -1)
warpedImgs[-1][~img_mask] = 0
#WIthout linear blend, last image gets messed up
#pano_mask = (panorama != -1)
#panorama[~pano_mask] = 0
#panorama += warpedImgs[-1]
# Track the overlap by adding the masks together
#overlap = (pano_mask * 1.0 + img_mask)
# Normalize through division by `overlap` - but ensure the minimum is 1
#panorama /= np.maximum(overlap, 1)
panorama = warpedImgs[0]
for i in range(1, len(imgs)):
panorama = linear_blend(panorama, warpedImgs[i])
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
Hs = [np.eye(3)]
for i in range(len(imgs) - 1):
Hs.append(
ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)[0])
for i in range(1, len(imgs)):
Hs[i] = Hs[i].dot(
Hs[i - 1]
) # combine multiple transformation matrices by accumulation previous homogeneous matrix
output_shape, offset = get_output_space(
imgs[0], imgs[1:],
Hs[1:]) # get panorama image output shape and offset
for i in range(len(imgs)):
img_warped = warp_image(imgs[i], Hs[i], output_shape, offset)
img_mask = (img_warped != -1) # Mask == 1 inside the image
img_warped[~img_mask] = 0 # Return background values to 0
if i == 0:
panorama = img_warped
else:
panorama = linear_blend(panorama, img_warped)
pass
### END YOUR CODE
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
HList = []
for i in range(len(imgs) - 1):
H, robust_matches = ransac(keypoints[i],
keypoints[i + 1],
matches[i],
threshold=1)
HList.append(H)
'''
p1 = keypoints[i][matches[i][:,0]]
p2 = keypoints[i+1][matches[i][:,1]]
H = fit_affine_matrix(p1, p2)
print(H)
HList.append(H)
'''
imgWarpedList = []
idxRef = len(imgs) // 2 - 1
imgRef = imgs[idxRef]
HMerged = []
for i in range(len(imgs)):
if i < idxRef:
HRev = np.eye(3)
for j in range(idxRef - i):
HRev = np.dot(HRev, np.linalg.inv(HList[i + j]))
HMerged.append(HRev)
output_shape, offset = get_output_space(imgRef, [imgs[i]], [HRev])
imgRef = warp_image(imgRef, np.eye(3), output_shape, offset)
#imgWarpedRev = warp_image(img[i], HRev, output_shape, offset)
#imgWarpedList.append(imgWarpedRev)
elif i > idxRef:
H = np.eye(3)
for j in range(i - idxRef):
H = np.dot(H, HList[i - j - 1])
HMerged.append(H)
output_shape, offset = get_output_space(imgRef, [imgs[i]], [H])
imgRef = warp_image(imgRef, np.eye(3), output_shape, offset)
#imgWarpedRev = warp_image(img[i], HRev, output_shape, offset)
#imgWarpedList.append(imgWarpedRev)
imgMaskRef = (imgRef != -1) # Mask == 1 inside the image
imgRef[~imgMaskRef] = 0 # Return background values to 0
imgs.pop(idxRef)
for i in range(len(imgs)):
imgWarped = warp_image(imgs[i], HMerged[i], output_shape, offset)
imgMask = (imgWarped != -1) # Mask == 1 inside the image
imgWarped[~imgMask] = 0 # Return background values to 0
imgWarpedList.append(imgWarped)
#panorama = imgWarpedList[0]+imgWarpedList[1]+imgWarpedList[2]+imgRef
imgWarpedList.insert(idxRef, imgRef)
merged = imgWarpedList[0]
for i in range(len(imgWarpedList) - 1):
merged = linear_blend(merged, imgWarpedList[i + 1])
panorama = merged
### END YOUR CODE
return panorama
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
# np.eye: Return a 2-D array with ones on the diagonal and zeros elsewhere
imgs_warped = []
arr = [np.eye(3)]
for i in range(len(imgs) - 1):
ransacResult = ransac(keypoints[i],
keypoints[i + 1],
matches[i],
threshold=1)
arr.append(ransacResult[0])
for i in range(1, len(imgs)):
arr[i] = arr[i].dot(arr[i - 1])
output_shape, offset = get_output_space(imgs[0], imgs[1:], arr[1:])
for i in range(len(imgs)):
warpedImage = warp_image(imgs[i], arr[i], output_shape, offset)
imgs_warped.append(warpedImage)
# ~ : NOT - inverts all bits
img_mask = ~(imgs_warped[-1] != -1)
imgs_warped[-1][img_mask] = 0
panorama = imgs_warped[0]
# linearly blending the images
for i in range(1, len(imgs)):
panorama = linear_blend(panorama, imgs_warped[i])
### END YOUR CODE
return panorama
pnt_list_src, pnt_list_dst = find_mtm_matches(
anchor=img_anchor_gray,
img=img_gray,
patch_size=patch_size,
alpha=alpha,
n_points_to_match=n_points_match_registration,
search_radius=search_radius)
M, mask = utils.ransac_points(src_pnts=pnt_list_src,
dst_pnts=pnt_list_dst,
th=5)
pnt_list_src_ransac = [m[0] for m in zip(pnt_list_src, mask) if m[1] == 1]
pnt_list_dst_ransac = [m[0] for m in zip(pnt_list_dst, mask) if m[1] == 1]
img_reg = utils.warp_image(img, np.linalg.inv(M))
registrated_images.append(
np.array(img_reg).astype(np.uint8)[:, :, ::-1].copy())
utils.show_matches(
src_img=img_anchor,
dst_img=img,
src_pnts=pnt_list_src_ransac,
dst_pnts=pnt_list_dst_ransac,
title=f'Match between anchor image {imgsFn[0]} and {imgsFn[i]}')
utils.show_blend_images(
img1=img_anchor,
img2=img,
title=f'{imgsFn[0]} and {imgsFn[i]} before registration')
utils.show_blend_images(
img1=img_anchor,
def anhir_method(source, target, echo=True):
##### Step 0 - Parameters, Smoothing and Initial Resampling #####
b_time_total = time.time()
params = dict()
params["echo"] = echo
params["initial_alignment_size"] = 2048
params["centroid_rotation_size"] = 512
params["nonrigid_registration_size"] = 2048
params["gaussian_divider"] = 1.24
params['nr_method'] = "dm" # pd or dm
nr_params = dict()
nr_params['echo'] = echo
nr_params['global_min_size'] = 64
nr_params['global_max_size'] = 512
nr_params['local_min_size'] = 64
nr_params['local_max_size'] = 768
nr_params['global_iterations'] = 100
nr_params['inner_iterations'] = 15
nr_params['outer_iterations'] = 5
nr_params['L_smooth'] = 1e7
nr_params['L_sigma'] = 1
nr_params['R_sigma'] = 1
nr_params['M_sigma'] = 2
nr_params['x_box'] = 19
nr_params['y_box'] = 19
nr_params['spacing'] = (1.0, 1.0)
nr_params['update_mode'] = "composition"
nr_params['gradient_mode'] = "symmetric"
nr_params['diffusion_sigma'] = (2.0, 2.0)
nr_params['fluid_sigma'] = (0.5, 0.5)
nr_params['mind_sigma'] = (1.0, 1.0)
nr_params['mind_radius'] = (2, 2)
nr_params['early_stop'] = 10
return_dict = dict()
initial_resample_ratio = utils.calculate_resample_size(
source, target,
max(params["initial_alignment_size"],
params["nonrigid_registration_size"]))
source = utils.gaussian_filter(
source, initial_resample_ratio / params["gaussian_divider"])
target = utils.gaussian_filter(
target, initial_resample_ratio / params["gaussian_divider"])
if echo:
print()
print("Registration start.")
print()
print("Source shape: ", source.shape)
print("Target shape: ", target.shape)
##### Step 1 - Preprocessing #####
if echo:
print()
print("Preprocessing start.")
print()
b_time_r = time.time()
p_source, p_target = utils.resample_both(source, target,
initial_resample_ratio)
e_time_r = time.time()
tt_source = p_source.copy()
if echo:
print("Initially resampled source shape: ", p_source.shape)
print("Initially resampled target shape: ", p_target.shape)
print("Time for initial resampling: ", e_time_r - b_time_r,
" seconds.")
b_time_p = time.time()
p_source, p_target, t_source, t_target, source_shift, target_shift = pp.preprocess(
p_source, p_target, echo)
e_time_p = time.time()
return_dict["preprocessing_time"] = e_time_p - b_time_p
if echo:
print("Source shift: ", source_shift)
print("Target shift: ", target_shift)
print("Preprocessed source shape: ", p_source.shape)
print("Preprocessed target shape: ", p_target.shape)
print("Time for preprocessing: ", e_time_p - b_time_p, " seconds.")
print()
print("Preprocessing end.")
print()
##### Step 2 - Initial Alignment #####
b_ia_time = time.time()
if echo:
print("Initial alignment start.")
print()
ia_resample_ratio = params["nonrigid_registration_size"] / params[
"initial_alignment_size"]
to_cv_source, to_cv_target = utils.resample_both(p_source, p_target,
ia_resample_ratio)
cv_failed = False
ct_failed = False
ia_failed = False
i_u_x, i_u_y, initial_transform, cv_failed = ia.cv_initial_alignment(
to_cv_source, to_cv_target, echo)
if cv_failed:
if echo:
print("CV failed.")
print()
print("CT start.")
print()
ia_resample_ratio = params["nonrigid_registration_size"] / params[
"centroid_rotation_size"]
to_ct_source, to_ct_target = utils.resample_both(
p_source, p_target, ia_resample_ratio)
i_u_x, i_u_y, initial_transform, ct_failed = ia.ct_initial_alignment(
to_ct_source, to_ct_target, echo)
if ct_failed:
if echo:
print()
print("CT failed.")
print("Initial alignment failed.")
ia_failed = True
if ia_failed:
i_u_x, i_u_y = np.zeros(p_source.shape), np.zeros(p_target.shape)
else:
y_size, x_size = np.shape(p_source)
i_u_x, i_u_y = utils.resample_displacement_field(
i_u_x, i_u_y, x_size, y_size)
e_ia_time = time.time()
return_dict["cv_failed"] = cv_failed
return_dict["ct_failed"] = ct_failed
return_dict["ia_failed"] = ia_failed
return_dict["initial_alignment_time"] = e_ia_time - b_ia_time
if echo:
print()
print("Elapsed time for initial alignment: ", e_ia_time - b_ia_time,
" seconds.")
print("Initial alignment end.")
print()
ia_source = utils.warp_image(p_source, i_u_x, i_u_y)
u_x_g, u_y_g = nr.partial_data_registration_global(ia_source, p_target,
nr_params)
u_x_g, u_y_g = utils.compose_vector_fields(i_u_x, i_u_y, u_x_g, u_y_g)
ng_source = utils.warp_image(p_source, u_x_g, u_y_g)
success = fd.detect_mind_failure(ia_source, p_target, ng_source, echo)
if not success:
u_x_g, u_y_g = i_u_x, i_u_y
ng_source = ia_source
return_dict["ng_failed"] = not success
##### Step 3 - Nonrigid Registration #####
b_nr_time = time.time()
if echo:
print("Nonrigid registration start.")
print()
if params['nr_method'] == "dm":
u_x_nr, u_y_nr = nr.dm(ng_source, p_target, nr_params)
u_x_nr, u_y_nr = utils.compose_vector_fields(u_x_g, u_y_g, u_x_nr,
u_y_nr)
nr_source = utils.warp_image(p_source, u_x_nr, u_y_nr)
elif params['nr_method'] == "pd":
u_x_nr, u_y_nr = nr.partial_data_registration_local(
ng_source, p_target, nr_params)
u_x_nr, u_y_nr = utils.compose_vector_fields(u_x_g, u_y_g, u_x_nr,
u_y_nr)
nr_source = utils.warp_image(p_source, u_x_nr, u_y_nr)
e_nr_time = time.time()
return_dict["nonrigid_registration_time"] = e_nr_time - b_nr_time
if echo:
print()
print("Elapsed time for nonrigid registration: ",
e_nr_time - b_nr_time, " seconds.")
print("Nonrigid registration end.")
print()
##### Step 4 - Warping function creation #####
def warp_original_landmarks(source_landmarks):
source_landmarks = source_landmarks / initial_resample_ratio
source_landmarks = utils.pad_landmarks(source_landmarks,
target_shift[0],
target_shift[2])
source_landmarks = utils.transform_landmarks(source_landmarks, u_x_nr,
u_y_nr)
source_l_x, source_r_x, source_l_y, source_r_y = source_shift
target_l_x, target_r_x, target_l_y, target_r_y = target_shift
source_landmarks[:, 0] = source_landmarks[:, 0] - source_l_x
source_landmarks[:, 1] = source_landmarks[:, 1] - source_l_y
out_y_size, out_x_size = np.shape(source)
in_y_size, in_x_size = np.shape(tt_source)
source_landmarks[:,
0] = source_landmarks[:, 0] * out_x_size / in_x_size
source_landmarks[:,
1] = source_landmarks[:, 1] * out_y_size / in_y_size
return source_landmarks
def warp_resampled_landmarks(source_landmarks, target_landmarks):
source_landmarks = source_landmarks / initial_resample_ratio
target_landmarks = target_landmarks / initial_resample_ratio
source_landmarks = utils.pad_landmarks(source_landmarks,
target_shift[0],
target_shift[2])
target_landmarks = utils.pad_landmarks(target_landmarks,
source_shift[0],
source_shift[2])
transformed_source_landmarks = utils.transform_landmarks(
source_landmarks, u_x_nr, u_y_nr)
return source_landmarks, transformed_source_landmarks, target_landmarks
e_time_total = time.time()
return_dict["total_time"] = e_time_total - b_time_total
if echo:
print("Total registration time: ", e_time_total - b_time_total,
" seconds.")
print("End of registration.")
print()
return p_source, p_target, ia_source, ng_source, nr_source, i_u_x, i_u_y, u_x_nr, u_y_nr, warp_resampled_landmarks, warp_original_landmarks, return_dict
def stitch_multiple_images(imgs, desc_func=simple_descriptor, patch_size=5):
"""
Stitch an ordered chain of images together.
Args:
imgs: List of length m containing the ordered chain of m images
desc_func: Function that takes in an image patch and outputs
a 1D feature vector describing the patch
patch_size: Size of square patch at each keypoint
Returns:
panorama: Final panorma image in coordinate frame of reference image
"""
# Detect keypoints in each image
keypoints = [] # keypoints[i] corresponds to imgs[i]
for img in imgs:
kypnts = corner_peaks(harris_corners(img, window_size=3),
threshold_rel=0.05,
exclude_border=8)
keypoints.append(kypnts)
# Describe keypoints
descriptors = [] # descriptors[i] corresponds to keypoints[i]
for i, kypnts in enumerate(keypoints):
desc = describe_keypoints(imgs[i],
kypnts,
desc_func=desc_func,
patch_size=patch_size)
descriptors.append(desc)
# Match keypoints in neighboring images
matches = [] # matches[i] corresponds to matches between
# descriptors[i] and descriptors[i+1]
for i in range(len(imgs) - 1):
mtchs = match_descriptors(descriptors[i], descriptors[i + 1], 0.7)
matches.append(mtchs)
### YOUR CODE HERE
Hs = []
robust_matches = []
for i in range(len(imgs) - 1):
H, rm = ransac(keypoints[i], keypoints[i + 1], matches[i], threshold=1)
Hs.append(H)
robust_matches.append(rm)
idx_ref = len(imgs) // 2
img_ref = imgs[idx_ref]
for i in reversed(range(idx_ref)):
Hs[i] = np.linalg.inv(Hs[i])
for j in range(i + 1, idx_ref):
Hs[i] = Hs[i].dot(np.linalg.inv(Hs[j]))
for i in range(idx_ref + 1, len(imgs) - 1):
Hs[i] = Hs[i].dot(Hs[i - 1])
others_imgs = imgs[:idx_ref] + imgs[idx_ref + 1:]
output_shape, offset = get_output_space(img_ref, others_imgs, Hs)
iref_warped = warp_image(img_ref, np.eye(3), output_shape, offset)
iref_mask = (iref_warped != -1) # Mask == 1 inside the image
iref_warped[~iref_mask] = 0 # Return background values to 0
merged = iref_warped
overlap = iref_mask * 1.0 # Multiply by 1.0 for bool -> float conversion
for img, H in zip(others_imgs, Hs):
img_warped = warp_image(img, H, output_shape, offset)
img_mask = (img_warped != -1) # Mask == 1 inside the image
img_warped[~img_mask] = 0 # Return background values to 0
merged += img_warped
overlap += img_mask
panorama = merged / np.maximum(overlap, 1)
### END YOUR CODE
return panorama
from utils import get_output_space, warp_image
# Extract matched keypoints
p1 = keypoints1[matches[:, 0]]
p2 = keypoints2[matches[:, 1]]
# Find affine transformation matrix H that maps p2 to p1
H = fit_affine_matrix(p1, p2)
# 这里的实现还没看懂！！！
output_shape, offset = get_output_space(img1, [img2], [H])
# print("Output shape:", output_shape)
# print("Offset:", offset)
# Warp images into output sapce
img1_warped = warp_image(img1, np.eye(3), output_shape, offset)
img1_mask = (img1_warped != -1) # Mask == 1 inside the image
img1_warped[~img1_mask] = 0 # Return background values to 0
img2_warped = warp_image(img2, H, output_shape, offset)
img2_mask = (img2_warped != -1) # Mask == 1 inside the image
img2_warped[~img2_mask] = 0 # Return background values to 0
# Plot warped images
plt.subplot(1, 2, 1)
plt.imshow(img1_warped)
plt.title('Image 1 warped')
plt.axis('off')
plt.subplot(1, 2, 2)
plt.imshow(img2_warped)
def partial_data_registration(source, target, params):
u_x_g, u_y_g = partial_data_registration_global(source, target, params)
source = utils.warp_image(source, u_x_g, u_y_g)
u_x_l, u_y_l = partial_data_registration_local(source, target, params)
u_x_t, u_y_t = utils.compose_vector_fields(u_x_g, u_y_g, u_x_l, u_y_l)
return u_x_g, u_y_g, u_x_t, u_y_t
