def anms_outputs(self):
self.x_left, self.y_left, self.rmax_left = anms(\
self.cimg_left, 250)
self.x_center, self.y_center, self.rmax_center = anms(\
self.cimg_center, 250)
self.x_right, self.y_right, self.rmax_right = anms(\
self.cimg_right, 250)
def getFeatures(img, bbox):
#TODO: Your code here
faces_count = bbox.shape[0]
bbox = bbox.astype(int)
maxPoints = 50
x = np.zeros((maxPoints,faces_count))
y = np.zeros((maxPoints,faces_count))
for i in range(0,faces_count):
width = bbox[i,3,0] - bbox[i,0,0]
height = bbox[i,1,1] - bbox[i,0,1]
face_holder = np.zeros((height.astype(int),width.astype(int)))
face_holder = img[bbox[i,0,1]:bbox[i,1,1], bbox[i,0,0]:bbox[i,3,0]]
cface = corner_detector(face_holder)
x1, y1, rmax1 = anms.anms(cface, maxPoints)
#plt.imshow(face_holder, cmap='gray')
#plt.scatter(x1, y1)
#plt.show()
x1 = x1.reshape((x1.size, 1))
y1 = y1.reshape((y1.size, 1))
x[:,i] = y1[:,0]
y[:,i] = x1[:,0]
return x,y
#return x, y
#sys.path.append('/usr/local/lib/python2.7/site-packages')
#img = cv2.imread('/home/prateek/Documents/cis581/project4/PartA/Easy/video1/image1.jpg')
#gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#bbox = np.load('bbox.npy')
#getFeatures(gray_img, bbox)
def mymosaic(img_input):
import numpy as np
from helpers import rgb2gray
from helpers import warp_image
from corner_detector import corner_detector
from anms import anms
from feat_desc import feat_desc
from feat_match import feat_match
from ransac_est_homography import ransac_est_homography
import matplotlib.pyplot as plt
import math
# Set out constants
max_pts = 1000
thresh = 0.5
h, w, d = img_input[0].shape
# ---------- Part 1: Get descs for each image ---------- #
# Initialize all the cell arrays that we will be using
# For now, I'm only saving the variables that matter for later steps
x = np.zeros(3, dtype=object)
y = np.zeros(3, dtype=object)
descs = np.zeros(3, dtype=object)
# Get x, y, and descs for each image
for i in range(3):
print("---------- Processing Image %d ----------" % (i + 1))
gray = rgb2gray(img_input[i])
print("Detecting corners")
cimg = corner_detector(gray)
print("Suppressing non maxima")
x[i], y[i], rmax = anms(cimg, max_pts)
print("Finding descriptors")
descs[i] = feat_desc(gray, x[i], y[i])
# ---------- Part 2: Estimate homographies ---------- #
# Initialize all the cell arrays that we will be using
# For now, I'm only saving the variables that matter for later steps
H = np.zeros(3, dtype=object)
inlier_ind = np.zeros(3, dtype=object)
corners = np.zeros(3, dtype=object)
corners[1] = np.stack(
[np.array([0, w, w, 0]),
np.array([0, 0, h, h]),
np.ones(4)])
H[1] = np.identity(3)
for i in [0, 2]:
print("---------- Matching Images %d and %d ----------" % (i + 1, 2))
print("Finding matching descriptors")
match = feat_match(descs[1], descs[i])
matches = (np.where([match >= 0])[1], match[match >= 0])
print("Performing RANSAC")
H[i], inlier_ind[i] = ransac_est_homography(x[i][matches[1]],
y[i][matches[1]],
x[1][matches[0]],
y[1][matches[0]], thresh)
# Find the boundaries that the mosaic has to fit into
warped_corners = np.matmul(H[i], corners[1])
corners[i] = warped_corners / warped_corners[2]
# ---------- Part 3: Assemble the mosaic ---------- #
print("---------- Assembling the Mosaic ----------")
# Initialize the mosaic using corners
xmin = int(math.floor(np.amin(corners[0][0])))
xmax = int(math.ceil(np.amax(corners[2][0])))
ymin = int(
math.floor(np.amin([np.amin(corners[0][1]),
np.amin(corners[2][1])])))
ymax = int(
math.ceil(np.amax([np.amax(corners[0][1]),
np.amax(corners[2][1])])))
img_mosaic = np.zeros((ymax - ymin, xmax - xmin, 3)).astype(int)
# Need to find the mesh to interpolate with
print("Warping images")
left, yi0, w0, h0 = warp_image(img_input[0], H[0], corners[0])
center = img_input[1].astype(int)
right, yi2, w2, h2 = warp_image(img_input[2], H[2], corners[2])
# Need the offsets to correctly align images
xi1 = -xmin
yi1 = -ymin
w1 = w
h1 = h
# Assemble the mosaic
print("Putting it all together")
if yi0 < yi2:
img_mosaic[:h0, :w0][left > 0] = left[left > 0]
img_mosaic[yi2 - yi0:h2 + yi2 - yi0,
-w2 - 1:-1][right > 0] = right[right > 0]
else:
img_mosaic[yi0 - yi2:h0 + yi0 - yi2, :w0][left > 0] = left[left > 0]
img_mosaic[:h2, -w2 - 1:-1][right > 0] = right[right > 0]
img_mosaic[yi1:yi1 + h1, xi1:xi1 + w1] = center
return img_mosaic
from feat_match import feat_match
from ransac_est_homography import ransac_est_homography
print("---------- Processing First Image ----------")
path1 = "street-2.jpg"
img1 = Image.open(path1)
img1 = np.array(img1)[..., :3]
gray1 = rgb2gray(img1)
max_pts = 1000
print("Detecting corners")
cimg1 = corner_detector(gray1)
print("Suppressing non maxima")
x1, y1, rmax1 = anms(cimg1, max_pts)
print("Finding descriptors")
descs1, boxes1, oris1, ori1 = feat_desc(gray1, x1, y1)
# plt.imshow(img1)
# plt.scatter(x1, y1)
# for i in range(boxes1.shape[2]):
# plt.plot(boxes1[:,0,i], boxes1[:,1,i], color="red")
# plt.plot(oris1[0,:,i], oris1[1,:,i], color="green")
# plt.show()
print("---------- Processing Second Image ----------")
path2 = "street-3.jpg"
img2 = Image.open(path2)
img2 = np.array(img2)[..., :3]
def corner_detector(img):
cimg = np.zeros([img.shape[0], img.shape[1]])
img = img.astype(np.float32)
cimg = cv2.cornerHarris(img, 2, 3, 0.04)
#cimg = corner_harris(img)
cimg[cimg < 0.1 * cimg.max()] = 0
print('detected points:')
print(cimg[cimg != 0].size)
return cimg
if __name__ == '__main__':
I = np.array(Image.open('im1.jpg').convert('RGB'))
im_gray = utils.rgb2gray(I)
cimg = corner_detector(im_gray)
Icopy = I.copy()
Icopy[cimg > 0] = [0, 0, 255]
plt.subplot(1, 2, 1)
plt.imshow(Icopy)
features = cimg[cimg > 0].size
x, y, rmax = anms(cimg, round(features * 0.1))
print(x.size)
Icopyy = np.zeros([np.shape(I)[0], np.shape(I)[1]])
Icopyy[y, x] = 1
plt.subplot(1, 2, 2)
plt.imshow(Icopyy)
plt.show()
def Stitch(middle, left):
h, w, z = left.shape
center_img = utils.rgb2gray(middle)
cimg_c = corner_detector(center_img)
xc, yc, rmaxc = anms(cimg_c, pts_num) # col, row
output_c = feat_desc(center_img, yc, xc)
left_img = utils.rgb2gray(left)
cimg_left = corner_detector(left_img)
xleft, yleft, rmaxleft = anms(cimg_left, pts_num) # col, row
output_left = feat_desc(left_img, yleft, xleft)
match = feat_match(output_c, output_left)
a = (match != -1).sum()
if a < 20:
return 0
'''
fig = plt.figure()
axis1 = fig.add_subplot(221)
axis1.imshow(middle)
axis1.scatter(xc, yc)
axis2 = fig.add_subplot(222)
axis2.imshow(left)
axis2.scatter(xleft, yleft)
a = (match != -1).sum()
xl = np.zeros([a, 1])
xr = np.zeros([a, 1])
yl = np.zeros([a, 1])
yr = np.zeros([a, 1])
counter = 0
for i in range(0, len(match)):
if match[i] != -1:
xl[counter] = xc[i]
yl[counter] = yc[i]
xr[counter] = xleft[match[i]]
yr[counter] = yleft[match[i]]
counter += 1
ax1 = fig.add_subplot(223)
ax1.imshow(middle)
colors = cm.rainbow(np.linspace(0, 1, len(yl)))
for x, y, c in zip(xl, yl, colors):
ax1.scatter(x, y, color = c)
ax2 = fig.add_subplot(224)
ax2.imshow(left)
for x, y, c in zip(xr, yr, colors):
ax2.scatter(x, y, color = c)
plt.show()
'''
H, inlier_ind = MatchToH(match, xc, yc, xleft, yleft, middle, left)
new_h, new_w, offseth, offsetw = findCanvas(H, middle, left)
x = np.linspace(0, h - 1, h)
y = np.linspace(0, w - 1, w)
xv, yv = np.meshgrid(y, x)
coord_ori = np.ones([3, h * w])
coord_ori[0, :] = xv.flatten()
coord_ori[1, :] = yv.flatten()
coord_ori = coord_ori.astype(np.int)
coord_new = np.dot(H, coord_ori)
coord_new = coord_new / coord_new[2, :]
coord_new[0, :] = coord_new[0, :] + offsetw
coord_new[1, :] = coord_new[1, :] + offseth
coord_new = coord_new.astype(np.int)
# insert left color into new canvas
new = np.zeros([new_h + 1, new_w + 1, 3])
counter = 0
while coord_new.max(axis=1)[0] > new_w or coord_new.max(
axis=1)[1] > new_h or coord_new.max(
axis=1)[0] < 0 or coord_new.max(axis=1)[1] < 0:
print('ayamaya')
counter = counter + 1
num = 200
cimg_c = corner_detector(center_img)
xc, yc, rmaxc = anms(cimg_c, pts_num + counter * num) # col, row
output_c = feat_desc(center_img, yc, xc)
cimg_left = corner_detector(left_img)
xleft, yleft, rmaxleft = anms(cimg_left,
pts_num + counter * num) # col, row
output_left = feat_desc(left_img, yleft, xleft)
match = feat_match(output_c, output_left)
H, inlier_ind = MatchToH(match, xc, yc, xleft, yleft, middle, left)
fig2 = plt.figure()
ind = np.where(inlier_ind != 0)
xinl = xc[ind[0]]
yinl = yc[ind[0]]
xinr = xleft[ind[0]]
yinr = yleft[ind[0]]
ind_out = np.where(inlier_ind == 0)
xoutl = xc[ind_out[0]]
youtl = yc[ind_out[0]]
xoutr = xleft[ind_out[0]]
youtr = yleft[ind_out[0]]
ax1 = fig2.add_subplot(223)
ax1.imshow(center_img)
ax1.scatter(xinl, yinl, color='r')
ax1.scatter(xoutl, youtl, color='b')
ax2 = fig2.add_subplot(224)
ax2.imshow(left_img)
ax2.scatter(xinr, yinr, color='r')
ax2.scatter(xoutr, youtr, color='b')
plt.show()
new_h, new_w, offseth, offsetw = findCanvas(H, middle, left)
coord_new = np.dot(H, coord_ori)
coord_new = coord_new / coord_new[2, :]
coord_new[0, :] = coord_new[0, :] + offsetw
coord_new[1, :] = coord_new[1, :] + offseth
coord_new = coord_new.astype(np.int)
new = np.zeros([new_h + 1, new_w + 1, 3])
new[coord_new[1, :], coord_new[0, :], 0] = left[coord_ori[1, :],
coord_ori[0, :], 0]
new[coord_new[1, :], coord_new[0, :], 1] = left[coord_ori[1, :],
coord_ori[0, :], 1]
new[coord_new[1, :], coord_new[0, :], 2] = left[coord_ori[1, :],
coord_ori[0, :], 2]
kernel = np.ones((5, 5), np.uint8)
new = cv2.dilate(new, kernel, iterations=1)
#new[coord_new[0, :], coord_new[1, :], 0] = left[coord_ori[0, :], coord_ori[1, :], 0]
#new[coord_new[0, :], coord_new[1, :], 1] = left[coord_ori[0, :], coord_ori[1, :], 1]
#new[coord_new[0, :], coord_new[1, :], 2] = left[coord_ori[0, :], coord_ori[1, :], 2]
new_center = np.zeros([new_h + 1, new_w + 1, 3])
hm, wm, z = middle.shape
#plt.figure()
#plt.imshow(new)
#plt.show()
new_center[offseth:offseth + hm, offsetw:offsetw + wm, 0] = middle[:, :, 0]
new_center[offseth:offseth + hm, offsetw:offsetw + wm, 1] = middle[:, :, 1]
new_center[offseth:offseth + hm, offsetw:offsetw + wm, 2] = middle[:, :, 2]
#plt.figure()
#plt.imshow(new_center)
#plt.show()
# insert center color into new canvas
A = new[offseth:offseth + hm, offsetw:offsetw + wm, :]
Ac = new_center[offseth:offseth + hm, offsetw:offsetw + wm, :]
A[np.bitwise_and(A != 0, Ac != 0)] = A[np.bitwise_and(
A != 0, Ac != 0)] * 0.5 + Ac[np.bitwise_and(A != 0, Ac != 0)] * 0.5
#plt.figure()
#plt.imshow(new)
#plt.show()
A[A == 0] = Ac[A == 0]
#plt.figure()
#plt.imshow(new)
#plt.show()
new = new.astype('uint8')
return new
def stitch(im1,im2):
gray1 = cv2.cvtColor(im1,cv2.COLOR_BGR2GRAY)
gray1 = np.float32(gray1)
gray2 = cv2.cvtColor(im2,cv2.COLOR_BGR2GRAY)
gray2 = np.float32(gray2)
print("corners")
corners1=corner_detector(gray1)
corners2=corner_detector(gray2)
print("anms")
x1,y1,_=anms(corners1, 1000)
x2,y2,_=anms(corners2, 1000)
print("feat_desc")
descs1 =feat_desc(gray1, x1, y1)
descs2 =feat_desc(gray2, x2, y2)
print("match")
matches = feat_match(descs1,descs2)
matched_x2 = []
matched_y2 = []
matched_x1 = []
matched_y1 = []
for i in range(len(matches)):
if matches[i]!=-1:
matched_x1.append(x1[i])
matched_y1.append(y1[i])
matched_x2.append(x2[int(matches[i])])
matched_y2.append(y2[int(matches[i])])
print("RANSAC")
H,inliers = ransac_est_homography(matched_x1,matched_y1,matched_x2,matched_y2,5)
#Find outer bounds and displacements
x,y=[],[]
corners = [(0,0),(0,im1.shape[1]-1),(im1.shape[0]-1,0),(im1.shape[0]-1,im1.shape[1]-1)]
for i in corners:
transform = np.matmul(H,np.transpose([i[1],i[0],1]))
transform = transform*(1/transform[2])
x.append(transform[0])
y.append(transform[1])
if np.array(y).min()<0:
dispy= np.array(y).min()
else:
dispy=0
if np.array(x).min()<0:
dispx= np.array(x).min()
else:
dispx=0
warped_img = np.zeros((int(np.ceil(max(gray2.shape[0],np.array(y).max())+np.abs(dispy))),int(np.ceil(max(gray2.shape[1],np.array(x).max())+np.abs(dispx))),3))
inverse = np.linalg.inv(H)
print("stitching")
for i in range(warped_img.shape[0]):
for j in range (warped_img.shape[1]):
transform = np.matmul(inverse,np.transpose([j+dispx,i+dispy,1]))
transform = transform*(1/transform[2])
if i in range(gray2.shape[0]) and j in range(gray2.shape[1]):
for channel in range(3):
warped_img[i+int(np.abs(np.ceil(dispy))),j+int(np.abs(np.ceil(dispx))),channel]=im2[i,j,channel]
if int(transform[0]) in range(gray1.shape[1]) and int(transform[1]) in range(gray1.shape[0]):
i_wt=np.ceil(transform[1])-transform[1]
j_wt=np.ceil(transform[0])-transform[0]
try:
for channel in range(3):
warped_img[i,j,channel]=(i_wt*j_wt)*im1[int(transform[1]),int(transform[0]),channel]+(1-i_wt)*j_wt*im1[int(transform[1])+1,int(transform[0]),channel]+(1-j_wt)*i_wt*im1[int(transform[1]),int(transform[0])+1,channel]+(1-j_wt)*(1-i_wt)*im1[int(transform[1])+1,int(transform[0])+1,channel]
except:
for channel in range(3):
warped_img[i,j,channel]=im1[int((transform[1])),int((transform[0])),channel]
cv2.imwrite("intermediate.jpg",warped_img)
warped_img = cv2.imread("intermediate.jpg")
return warped_img
# left = cv2.imread("../test_img/small_1L.jpg")
# middle = cv2.imread("../test_img/small_1M.jpg")
left = cv2.imread("../test_img/1L.jpg")
middle = cv2.imread("../test_img/1M.jpg")
# Convert to grayscale
gray_left = cv2.cvtColor(left, cv2.COLOR_BGR2GRAY)
gray_middle = cv2.cvtColor(middle, cv2.COLOR_BGR2GRAY)
left_cmm = corner_detector(gray_left)
middle_cmm = corner_detector(gray_middle)
features = 1000
left_c, left_r, left_rmax = anms(left_cmm, features)
middle_c, middle_r, middle_rmax = anms(middle_cmm, features)
# Show the points for the left
# fig, ax = plt.subplots()
# ax.imshow(gray_left, origin='upper', cmap=plt.cm.gray)
# ax.plot(left_c, left_r, 'r.', markersize=5)
# plt.show()
# Show the points for the middle
fig, (ax1, ax2) = plt.subplots(1, 2)
ax1.imshow(gray_left, origin='upper', cmap=plt.cm.gray)
ax1.plot(left_c, left_r, 'r.', markersize=5)
ax2.imshow(gray_middle, origin='upper', cmap=plt.cm.gray)
ax2.plot(middle_c, middle_r, 'r.', markersize=5)
def mymosaic(img_input):
imgs = []
for img in img_input:
imgs.append(cv2.cvtColor(img, cv2.COLOR_BGR2GRAY))
cimg = []
aNMS = []
descs = []
fmatchsing = []
featMatchesDouble = []
hMat = []
for img in imgs:
cimg.append(corner_detector(img))
for i, c in enumerate(cimg):
ares = anms(c, 1000)
"""
fig, ax = plt.subplots()
ax.imshow(imgs[i], origin="upper", cmap=plt.cm.gray)
ax.plot(ares[0], ares[1], '.r', markersize=2, color="red")
plt.show()
"""
aNMS.append(ares)
for i, aN in enumerate(aNMS):
descs.append(feat_desc(imgs[i], aN[0], aN[1]))
for i in range(len(descs) - 1):
fDirect = feat_match(descs[i], descs[i + 1])
bDirect = feat_match(descs[i + 1], descs[i])
m1m = fDirect.T[0][(fDirect.T[0] != -1)].astype(int)
m2m = bDirect.T[0][(bDirect.T[0] != -1)].astype(int)
"""
fig, ax = plt.subplots(ncols=2)
ax[0].imshow(imgs[i], origin="upper", cmap=plt.cm.gray)
ax[0].plot(aNMS[i][0], aNMS[i][1], '.r', markersize=5, color='blue')
ax[0].plot(aNMS[i][0][m2m], aNMS[i][1][m2m], '.r', markersize=5, color='red')
ax[1].imshow(imgs[i+1], origin="upper", cmap=plt.cm.gray)
ax[1].plot(aNMS[i+1][0], aNMS[i+1][1], '.r', markersize=5, color='blue')
ax[1].plot(aNMS[i+1][0][m1m], aNMS[i+1][1][m1m], '.r', markersize=5, color='red')
plt.show()
"""
fmatchsing.append(fDirect)
fmatchsing.append(bDirect)
for i in range(int(len(fmatchsing) / 2)):
srcindexes, dstindexes = unityOfMatch(fmatchsing[2 * i], fmatchsing[2 * i + 1])
mX1 = aNMS[i][0][srcindexes]
mY1 = aNMS[i][1][srcindexes]
mX2 = aNMS[i+1][0][dstindexes]
mY2 = aNMS[i+1][1][dstindexes]
"""
if i < 2:
m = np.zeros((imgs[i].shape[0], 2*imgs[i].shape[1], 3))
m[0:imgs[i].shape[0], 0: imgs[i].shape[1],:] = cv2.cvtColor(imgs[i], cv2.COLOR_GRAY2RGB)
m[0:imgs[i].shape[0], imgs[i].shape[1]:, :] = cv2.cvtColor(imgs[i+1], cv2.COLOR_GRAY2RGB)
# color_m = cv2.cvtColor(m, cv2.COLOR_GRAY2RGB)
offset = imgs[i].shape[1]
for j in range(len(mX1)):
cv2.line(m, (mY1[j], mX1[j]), (mY2[j], offset + mX2[j]), (255,0,0), 2)
cv2.imwrite("matches_{0}.png".format(i), m)
"""
pts1 = np.vstack((mX1, mY1)).T
pts2 = np.vstack((mX2, mY2)).T
fig, ax = plt.subplots(ncols=2)
ax[0].imshow(imgs[i], origin="upper", cmap=plt.cm.gray)
ax[0].plot(aNMS[i][0], aNMS[i][1], '.r', markersize=2, color='blue')
ax[0].plot(mX1, mY1, '.r', markersize=2, color='red')
ax[1].imshow(imgs[i+1], origin="upper", cmap=plt.cm.gray)
ax[1].plot(aNMS[i+1][0], aNMS[i+1][1], '.r', markersize=2, color='blue')
ax[1].plot(mX2, mY2, '.r', markersize=2, color='red')
ax[1].set_zorder(-1)
for j in range(len(mX1)):
con = ConnectionPatch(pts1[j], pts2[j], "data", "data", axesA=ax[0], axesB=ax[1],zorder = 0.5)
ax[0].add_patch(con)
plt.show()
rsac_results = ransac_est_homography(mX1, mY1, mX2, mY2, RSAC_THRESH_VAL)
filtSrc = srcindexes[np.where(rsac_results[1])]
filtDst = dstindexes[np.where(rsac_results[1])]
hMat.append(rsac_results)
H = rsac_results[0]
mX1 = aNMS[i][0][filtSrc]
mY1 = aNMS[i][1][filtSrc]
mX2 = aNMS[i+1][0][filtDst]
mY2 = aNMS[i+1][1][filtDst]
pts1 = np.vstack((mX1, mY1)).T
pts2 = np.vstack((mX2, mY2)).T
fig, ax = plt.subplots(ncols=2)
ax[0].imshow(imgs[i], origin="upper", cmap=plt.cm.gray)
ax[0].plot(aNMS[i][0], aNMS[i][1], '.r', markersize=2, color='blue')
ax[0].plot(mX1, mY1, '.r', markersize=2, color='red')
ax[1].imshow(imgs[i+1], origin="upper", cmap=plt.cm.gray)
ax[1].plot(aNMS[i+1][0], aNMS[i+1][1], '.r', markersize=2, color='blue')
ax[1].plot(mX2, mY2, '.r', markersize=2, color='red')
ax[1].set_zorder(-1)
for j in range(len(mX1)):
con = ConnectionPatch(pts1[j], pts2[j], "data", "data", axesA=ax[0], axesB=ax[1], zorder=0.5)
ax[0].add_patch(con)
plt.show()
"""
im2 = cv2.warpPerspective(imgs[1], H, dsize=(200, 150))
fig, ax = plt.subplots(ncols=2)
ax[0].imshow(imgs[i], origin="upper", cmap=plt.cm.gray)
ax[1].imshow(im2, origin="upper", cmap=plt.cm.gray)
plt.show()
"""
#at this point hMat[i] is the left-to-right H-result for that pair
imIndexes = len(img_input) - 1
m = (int)(imIndexes / 2)#floor; 3 images->index 1, 2 images->index 0
partialImages = []
hXform = []
for result in hMat:
hXform.append(result[0])#all the H matrixes
numAbove = imIndexes - m
numBelow = m
#while(True):
#pdb.set_trace()
#reload(stitcher)
intermed1, offset = stitcher.stitch(img_input[0], img_input[1], hXform[0])
img_mosaic = intermed1
intermed2 = None
if len(hXform) > 1:
intermed2 = stitcher.stitchRR(img_input[2], img_input[1], np.linalg.inv(hXform[1]), offset, intermed1)
img_mosaic = intermed2
if(np.amax(img_mosaic) < 2):
return img_mosaic
else:
return img_mosaic.astype(int)
def mymosaic(img_input):
# Your Code Here
img_mosaic = np.zeros((img_input.shape[0], 1), dtype=object)
for i in range(img_input.shape[0]):
imgA = img_input[i, 0]
imgB = img_input[i, 1]
imgC = img_input[i, 2]
imgA_gray = rgb2gray(imgA)
imgB_gray = rgb2gray(imgB)
imgC_gray = rgb2gray(imgC)
cimgA = corner_detector(imgA_gray)
cimgB = corner_detector(imgB_gray)
cimgC = corner_detector(imgC_gray)
max_pts = 500
xA, yA, rmaxA = anms(cimgA, max_pts)
xB, yB, rmaxB = anms(cimgB, max_pts)
xC, yC, rmaxC = anms(cimgC, max_pts)
# =============================================================================
# # Demonstrate ANMS result
# IA = flipChannel(imgA)
# drawPoints(IA,xA,yA,(0,0,255))
# cv.imwrite('A'+str(i+1)+'.jpg',IA)
#
# IB = flipChannel(imgB)
# drawPoints(IB,xB,yB,(0,0,255))
# cv.imwrite('B'+str(i+1)+'.jpg',IB)
#
# IC = flipChannel(imgC)
# drawPoints(IC,xC,yC,(0,0,255))
# cv.imwrite('C'+str(i+1)+'.jpg',IC)
# =============================================================================
descsA = feat_desc(imgA_gray, xA, yA)
descsB = feat_desc(imgB_gray, xB, yB)
descsC = feat_desc(imgC_gray, xC, yC)
match1 = feat_match(descsA, descsB)
match2 = feat_match(descsC, descsB)
ransac_thresh = 10
xA1, yA1 = xA[match1 > 0].reshape(-1, 1), yA[match1 > 0].reshape(-1, 1)
xB1, yB1 = xB[match1[match1 > 0]].reshape(
-1, 1), yB[match1[match1 > 0]].reshape(-1, 1)
H1, inlier_ind1 = ransac_est_homography(xA1, yA1, xB1, yB1,
ransac_thresh)
xC2, yC2 = xC[match2 > 0].reshape(-1, 1), yC[match2 > 0].reshape(-1, 1)
xB2, yB2 = xB[match2[match2 > 0]].reshape(
-1, 1), yB[match2[match2 > 0]].reshape(-1, 1)
H2, inlier_ind2 = ransac_est_homography(xC2, yC2, xB2, yB2,
ransac_thresh)
# =============================================================================
# # Demonstrating RANSAC match result
# row,col,_ = imgA.shape
#
# outlier_ind1 = np.delete(np.arange(len(xA1)),inlier_ind1)
# IA1 = flipChannel(imgA)
# drawPoints(IA1,xA1[inlier_ind1],yA1[inlier_ind1],(0,0,255))
# drawPoints(IA1,xA1[outlier_ind1],yA1[outlier_ind1],(255,0,0))
# IB1 = flipChannel(imgB)
# drawPoints(IB1,xB1[inlier_ind1],yB1[inlier_ind1],(0,0,255))
# drawPoints(IB1,xB1[outlier_ind1],yB1[outlier_ind1],(255,0,0))
# imgAB = np.zeros((row,2*col,3))
# imgAB[:,0:col,:] = IA1
# imgAB[:,col:2*col,:] = IB1
# drawLines(imgAB,xA1[inlier_ind1],yA1[inlier_ind1]\
# ,xB1[inlier_ind1]+col,yB1[inlier_ind1],(0,255,0))
# cv.imwrite('left_match'+str(i+1)+'.jpg',imgAB)
#
# outlier_ind2 = np.delete(np.arange(len(xC2)),inlier_ind2)
# IC2 = flipChannel(imgC)
# drawPoints(IC2,xC2[inlier_ind2],yC2[inlier_ind2],(0,0,255))
# drawPoints(IC2,xC2[outlier_ind2],yC2[outlier_ind2],(255,0,0))
# IB2 = flipChannel(imgB)
# drawPoints(IB2,xB2[inlier_ind2],yB2[inlier_ind2],(0,0,255))
# drawPoints(IB2,xB2[outlier_ind2],yB2[outlier_ind2],(255,0,0))
# imgBC = np.zeros((row,2*col,3))
# imgBC[:,0:col,:] = IB2
# imgBC[:,col:2*col,:] = IC2
# drawLines(imgBC,xB2[inlier_ind2],yB2[inlier_ind2]\
# ,xC2[inlier_ind2]+col,yC2[inlier_ind2],(0,255,0))
# cv.imwrite('right_match'+str(i+1)+'.jpg',imgBC)
# =============================================================================
new_left, new_middle, new_right = getNewSize(H1, H2, imgA, imgB, imgC)
# Blend Images by Seam Carving or Alpha Blending
img_mosaic[i, 0] = seamBlend(new_left, new_middle, new_right)
# img_mosaic[i,0] = alphaBlend(alphaBlend(new_left,new_middle),new_right)
return img_mosaic
output[:, i] = (output[:, i] - output_mean) / output_var
else:
output[:, i] = 0
return output
if __name__ == '__main__':
I = np.array(Image.open('im1.jpg').convert('RGB'))
im_gray = utils.rgb2gray(I)
cimg = corner_detector(im_gray)
Icopy = I.copy()
Icopy[cimg > 0] = [0, 0, 255]
plt.subplot(2, 2, 1)
plt.imshow(Icopy)
x1, y1, rmax = anms(cimg, 1000)
Icopyy = np.zeros([np.shape(I)[0], np.shape(I)[1]])
Icopyy[y1, x1] = 1
plt.subplot(2, 2, 2)
plt.imshow(Icopyy)
# plt.show()
output1 = feat_desc(im_gray, y1, x1)
print(output1)
I2 = np.array(Image.open('im2.jpg').convert('RGB'))
im_gray2 = utils.rgb2gray(I2)
cimg2 = corner_detector(im_gray2)
Icopy2 = I2.copy()
Icopy2[cimg2 > 0] = [255, 0, 0]
plt.subplot(2, 2, 3)
plt.imshow(Icopy2)
img1_cyl, trackimg1 = cylindricalWarp(img)
img1_cyl, trackimg1 = removeBlackSides(img1_cyl, trackimg1)
img1_cyl = img1_cyl.astype(np.uint8)
gray = cv2.cvtColor(img1_cyl.astype(np.uint8), cv2.COLOR_BGR2GRAY)
img2 = cv2.imread('collegeGreen2.jpg')
img2_cyl, trackimg2 = cylindricalWarp(img2)
img2_cyl, trackimg2 = removeBlackSides(img2_cyl, trackimg2)
img2_cyl = img2_cyl.astype(np.uint8)
gray2 = cv2.cvtColor(img2_cyl.astype(np.uint8), cv2.COLOR_BGR2GRAY)
cimg = corner_detector(gray)
cimg[trackimg1 == 0] = 0
# img1_cyl[cimg > 0.01 * cimg.max()] = [0,0,255]
# cv2.imwrite('collegeGreen1_cyl_corner.jpg',img1_cyl)
x1, y1, r = anms(cimg, 1000)
print('x1.shape', x1.shape)
desc = feat_desc(gray, x1, y1)
cimg2 = corner_detector(gray2)
cimg2[trackimg2 == 0] = 0
# img2_cyl[cimg2 > 0.01 * cimg2.max()] = [0,0,255]
# cv2.imwrite('collegeGreen2_cyl_corner.jpg',img2_cyl)
x2, y2, r2 = anms(cimg2, 1000)
print('x2.shape', x2.shape)
desc2 = feat_desc(gray2, x2, y2)
good = feat_match(desc, desc2)
# print(good.shape)
npgood = np.array(good)
'''
post-RANSAC matching (outliers in blue dots) for at least five distinct frames.
'''
img1 = cv2.imread("1L.png")
img2 = cv2.imread("1M.png")
im1 = np.float32(cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY))
im2 = np.float32(cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY))
img1RGB = cv2.cvtColor(img1, cv2.COLOR_BGR2RGB)
img2RGB = cv2.cvtColor(img2, cv2.COLOR_BGR2RGB)
#Detecting Corners
corners1 = corner_detector(im1)
corners2 = corner_detector(im2)
x1, y1, z1 = anms(corners1, 1000)
x2, y2, z2 = anms(corners2, 1000)
descs1 = feat_desc(im1, x1, y1)
descs2 = feat_desc(im2, x2, y2)
matches = feat_match(descs1, descs2)
matched_x2 = []
matched_y2 = []
matched_x1 = []
matched_y1 = []
for i in range(len(matches)):
if matches[i] != -1:
matched_x1.append(x1[i])
matched_y1.append(y1[i])
matched_x2.append(x2[int(matches[i])])
from PIL import Image import matplotlib.pyplot as plt from helpers import rgb2gray from corner_detector import corner_detector from anms import anms from feat_desc import feat_desc path1 = "1L.jpg" img1 = Image.open(path1) img1 = np.array(img1)[..., :3] gray1 = rgb2gray(img1) max_pts = 50 cimg1 = corner_detector(gray1) x1, y1, rmax1 = anms(cimg1, max_pts) descs1, boxes1, oris1, ori1 = feat_desc(gray1, x1, y1) plt.imshow(img1) plt.imshow(cimg1) # plt.scatter(x1, y1) # for i in range(boxes1.shape[2]): # plt.plot(boxes1[:,0,i], boxes1[:,1,i], color="red") # plt.plot(oris1[0,:,i], oris1[1,:,i], color="green") # plt.show() path2 = "1M.jpg" img2 = Image.open(path2) img2 = np.array(img2)[..., :3]
def main():
max_pts = 1000
left_image_filename = 'building1.JPG'
middle_image_filename = 'building2.JPG'
right_image_filename = 'building3.JPG'
img_left, gray_left = open_image(left_image_filename)
img_middle, gray_middle = open_image(middle_image_filename)
img_right, gray_right = open_image(right_image_filename)
corners_left = corner_detector(gray_left)
corners_middle = corner_detector(gray_middle)
corners_right = corner_detector(gray_right)
plotting_img_left = np.copy(img_left)
plotting_img_middle = np.copy(img_middle)
plotting_img_right = np.copy(img_right)
print('co', corners_left, corners_left.shape)
plotting_img_left[corners_left >= 0.01 * np.max(corners_left)] = [
255, 0, 0
]
plotting_img_middle[corners_middle >= 0.01 * np.max(corners_left)] = [
255, 0, 0
]
plotting_img_right[corners_right >= 0.01 * np.max(corners_left)] = [
255, 0, 0
]
plot_corner_harris(plotting_img_left, 'corner_harris_left_2')
plot_corner_harris(plotting_img_middle, 'corner_harris_middle_2')
plot_corner_harris(plotting_img_right, 'corner_harris_right_2')
x_left, y_left = anms(corners_left, max_pts)
x_middle, y_middle = anms(corners_middle, max_pts)
x_right, y_right = anms(corners_right, max_pts)
plot_anms(img_left, x_left, y_left)
plot_anms(img_middle, x_middle, y_middle)
plot_anms(img_right, x_right, y_right)
desc_left = feat_desc(gray_left, x_left, y_left)
desc_middle = feat_desc(gray_middle, x_middle, y_middle)
desc_right = feat_desc(gray_right, x_right, y_right)
matched_index_left_middle = feat_match(desc_left, desc_middle)
x1 = x_left[matched_index_left_middle != -1]
y1 = y_left[matched_index_left_middle != -1]
x2 = x_middle[matched_index_left_middle[matched_index_left_middle != -1]]
y2 = y_middle[matched_index_left_middle[matched_index_left_middle != -1]]
thresh = 0.5
H12, inlier_ind = ransac_est_homography(x1, y1, x2, y2, thresh)
bx1 = x1[inlier_ind == 0]
by1 = y1[inlier_ind == 0]
bx2 = x2[inlier_ind == 0]
by2 = y2[inlier_ind == 0]
x1 = x1[inlier_ind != 0]
y1 = y1[inlier_ind != 0]
x2 = x2[inlier_ind != 0]
y2 = y2[inlier_ind != 0]
plot_image_matching(x1, y1, x2, y2, bx1, by1, bx2, by2, img_left,
img_middle)
matched_index_right_middle = feat_match(desc_right, desc_middle)
x1 = x_right[matched_index_right_middle != -1]
y1 = y_right[matched_index_right_middle != -1]
x2 = x_middle[matched_index_right_middle[matched_index_right_middle != -1]]
y2 = y_middle[matched_index_right_middle[matched_index_right_middle != -1]]
thresh = 1
H32, inlier_ind = ransac_est_homography(x1, y1, x2, y2, thresh)
bx1 = x1[inlier_ind == 0]
by1 = y1[inlier_ind == 0]
bx2 = x2[inlier_ind == 0]
by2 = y2[inlier_ind == 0]
x1 = x1[inlier_ind != 0]
y1 = y1[inlier_ind != 0]
x2 = x2[inlier_ind != 0]
y2 = y2[inlier_ind != 0]
plot_image_matching(x1, y1, x2, y2, bx1, by1, bx2, by2, img_right,
img_middle)
mymosaic(img_left, img_middle, img_right, H12, H32)
def mosaic(image1, image2):
''' Get the corners of the images '''
print("Running corner detection...")
corner_matrix1 = corner_detector(rgb2gray(image1))
corner_matrix2 = corner_detector(rgb2gray(image2))
''' Adaptive non-maximal suppression '''
print("Non-maximal suppression...")
max_pts = MAX_PTS
corners1 = anms(corner_matrix1, max_pts)[0:2]
corners2 = anms(corner_matrix2, max_pts)[0:2]
# FINAL: Display corners after ANMS
def display_after_anms():
# Display image
plt.imshow(image1)
plt.plot(corners1[0], corners1[1], 'ro')
plt.show()
plt.imshow(image2)
plt.plot(corners2[0], corners2[1], 'ro')
plt.show()
#display_after_anms()
''' Get descriptors '''
print("Get descriptor vectors...")
descriptors1 = feat_desc(rgb2gray(image1), corners1[0], corners1[1])
descriptors2 = feat_desc(rgb2gray(image2), corners2[0], corners2[1])
''' Finding feature matches '''
print("Finding matches...")
matches_idx = feat_match(descriptors1, descriptors2)
# Boolean vector corresponding to whether or not there was a match found
num_match = matches_idx.shape[0]
has_match = np.where(matches_idx != -1)
matches_idx[np.where(matches_idx == -1)] = 0
# Pair up the matches
x1, y1 = corners1[0], corners1[1]
x2, y2 = np.zeros(num_match).astype(int), np.zeros(num_match).astype(int)
idx = np.arange(0, num_match)
x2[idx] = corners2[0][matches_idx]
y2[idx] = corners2[1][matches_idx]
# Get rid of non-matches
x1 = x1[has_match]
y1 = y1[has_match]
x2 = x2[has_match]
y2 = y2[has_match]
''' RANSAC '''
print("Doing RANSAC...")
homography, inlier_idx = ransac_est_homography(x1, y1, x2, y2,
ERROR_THRESH)
# FINAL: Visualize matches after RANSAC
def show_ransac_12():
big_im = np.concatenate((image1, image2), axis=1)
plt.imshow(big_im)
x2_shift = x2 + image1.shape[1]
for i in range(x1.shape[0]):
if inlier_idx[i] == 1:
plt.plot([x1[i], x2_shift[i]], [y1[i], y2[i]], marker="o")
plt.show()
#show_ransac_12()
''' Mosaicing '''
# Apply homography to corner points, to get size of canvas
h, w = image2.shape[0:2]
x_corners = np.array([[0, 0, w - 1, w - 1], [0, h - 1, 0, h - 1],
[1, 1, 1, 1]])
homography_inv = np.linalg.inv(homography)
x_corners_trans = np.dot(homography_inv, x_corners)
x_corners_trans = x_corners_trans / x_corners_trans[2]
''' Create canvas '''
min_x = np.floor(np.min(x_corners_trans[0])).astype(int)
min_y = np.floor(np.min(x_corners_trans[1])).astype(int)
max_x = np.ceil(np.max(x_corners_trans[0])).astype(int)
max_y = np.ceil(np.max(x_corners_trans[1])).astype(int)
# Find required padding
neg_padding = -1 * min(np.array([0, min_y, min_x])).astype(int)
pos_padding = max(np.array([max_y - h, max_x - w])).astype(int)
# Compute required height+width
num_colors = image1.shape[2]
h1, w1 = image1.shape[0:2]
mosaic = np.zeros(
(neg_padding + h1 + pos_padding, neg_padding + w1 + pos_padding,
num_colors)).astype(np.uint8)
mosaic[neg_padding:h1 + neg_padding,
neg_padding:w1 + neg_padding, :] = image1
# Define function to warp with
def apply_homography(output_coords):
# Do homography
x_output = np.array([output_coords[1], output_coords[0], 1]).reshape(
(3, 1)).astype(int)
x_transform = np.dot(homography, x_output)
x_transform = x_transform / x_transform[2]
# Add constant shift to deal with padding
x_input = x_transform[1] - neg_padding, x_transform[0] - neg_padding
return x_input
# Add a translation to the homography, based on negative padding
translation = np.array([
[1, 0, neg_padding],
[0, 1, neg_padding],
[0, 0, 1],
])
homography_trans = np.dot(translation, homography_inv)
# Warp image
print("Warping image...")
warped_image = np.zeros(mosaic.shape).astype(np.uint8)
warped_image = cv2.warpPerspective(
image2, homography_trans,
(warped_image.shape[1], warped_image.shape[0]))
''' Combining images '''
print("Combining images...")
# Copy the warped image into the mosaic
where_nonzero_warped = (np.sum(warped_image, axis=2) > 0)
where_nonzero_original = (np.sum(mosaic, axis=2) > 0)
for r in range(mosaic.shape[0]):
for c in range(mosaic.shape[1]):
# Both: average the two
if where_nonzero_original[r, c] and where_nonzero_warped[r, c]:
mosaic[
r,
c, :] = 0.5 * mosaic[r, c, :] + 0.5 * warped_image[r, c, :]
# Warped image only
elif where_nonzero_warped[r, c]:
mosaic[r, c, :] = warped_image[r, c, :]
return mosaic