def estimateFeatureTranslation(startX, startY, Ix, Iy, img1, img2): X = startX Y = startY mesh_x, mesh_y = np.meshgrid(np.arange(WINDOW_SIZE), np.arange(WINDOW_SIZE)) img1_gray = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY) img2_gray = cv2.cvtColor(img2, cv2.COLOR_RGB2GRAY) mesh_x_flat_fix = mesh_x.flatten() + X - np.floor(WINDOW_SIZE / 2) mesh_y_flat_fix = mesh_y.flatten() + Y - np.floor(WINDOW_SIZE / 2) coor_fix = np.vstack((mesh_x_flat_fix, mesh_y_flat_fix)) I1_value = interp2(img1_gray, coor_fix[[0], :], coor_fix[[1], :]) Ix_value = interp2(Ix, coor_fix[[0], :], coor_fix[[1], :]) Iy_value = interp2(Iy, coor_fix[[0], :], coor_fix[[1], :]) I = np.vstack((Ix_value, Iy_value)) A = I.dot(I.T) for _ in range(15): mesh_x_flat = mesh_x.flatten() + X - np.floor(WINDOW_SIZE / 2) mesh_y_flat = mesh_y.flatten() + Y - np.floor(WINDOW_SIZE / 2) coor = np.vstack((mesh_x_flat, mesh_y_flat)) I2_value = interp2(img2_gray, coor[[0], :], coor[[1], :]) Ip = (I2_value - I1_value).reshape((-1, 1)) b = -I.dot(Ip) solution = inv(A).dot(b) X += solution[0, 0] Y += solution[1, 0] return X, Y
def nonMaxSup(Mag, Ori): (yLength, xLength) = Mag.shape coordinates = (yLength, xLength) coordinates_x = coordinates[1] coordinates_y = coordinates[0] # Initialize the meshgrid mGridY, mGridX = np.meshgrid(np.arange(0, coordinates_x), np.arange(0, coordinates_y)) # Modify the meshgrid with orientation interp_location = np.add(mGridY, np.sin(Ori)) interp_location_2 = np.add(mGridX, np.cos(Ori)) interp_location_3 = np.subtract(mGridY, np.sin(Ori)) interp_location_4 = np.subtract(mGridX, np.cos(Ori)) # Interpolate and get magnitude of new points interp_pos = interp2(mGridX, mGridY, Mag, interp_location, interp_location_2) interp_neg = interp2(mGridX, mGridY, Mag, interp_location_3, interp_location_4) # New matrix created by logical operation Compare = np.logical_and(Mag > interp_pos, Mag > interp_neg) # Change into binary matrix Compare_int = Compare.astype(np.int) return Compare_int
def edgeLink(M, Mag, Ori): h, w = Mag.shape M1 = M * Mag mean = M1.sum() / M.sum() #high=0.08 #low=0.035 high = 1 * mean low = 0.2 * mean High = high * np.ones([h, w]) Low = low * np.ones([h, w]) M = M * Mag hpoint = np.int64(M - High > 0) mpoint = np.int64(High - M > 0) * np.int64(M - Low > 0) temp = mpoint E = hpoint lx_p = np.sin(Ori) ly_p = np.cos(Ori) x, y = np.meshgrid(np.arange(w), np.arange(h)) xq_p = x + lx_p yq_p = y + ly_p xq_n = x - lx_p yq_n = y - ly_p while (temp.sum() > 0): Pos1 = interp2(E * Mag, xq_p, yq_p) * mpoint Neg1 = interp2(E * Mag, xq_n, yq_n) * mpoint temp = 1 * np.logical_or(np.int64(Pos1 - High > 0), np.int64(Neg1 - High > 0)) E = E + temp mpoint = mpoint - temp return E
def edgeLink(M, Mag, Ori): # TODO: your code here edgeMap = M * Mag updatedMap = edgeMap mean = np.mean(Mag) median = np.median(Mag) std_dev = np.std(Mag) # print("mean",mean) # print("median",median) # print("dev",std_dev) # low_threshold=int(max(0, (1.0 - sigma) * np.median(Mag))) # high_threshold=int(min(255, (1.0 + sigma) * np.median(Mag))) high_threshold = mean * 0.9 low_threshold = mean * 0.65 print("high", high_threshold) print("low", low_threshold) # high_threshold=np.mean(Mag)*0.8 # low_threshold=np.mean(Mag) strongEdgeMap = (edgeMap > high_threshold) * 1 weakEdgeMap = np.logical_and(edgeMap > low_threshold, edgeMap < high_threshold) * 1 X, Y = np.meshgrid(np.arange(M.shape[1]), np.arange(M.shape[0])) OriEdge1 = Ori + (np.pi / 2) x1 = X + np.cos(OriEdge1) y1 = Y + np.sin(OriEdge1) OriEdge2 = Ori - (np.pi / 2) x2 = X + np.cos(OriEdge2) y2 = Y + np.sin(OriEdge2) edge1x = weakEdgeMap * x1 edge1y = weakEdgeMap * y1 edge2x = weakEdgeMap * x2 edge2y = weakEdgeMap * y2 strongEdgeMap_old = np.empty(strongEdgeMap.shape) weakEdgeMap_new = weakEdgeMap while True: side1 = weakEdgeMap_new * (interp.interp2(updatedMap, edge1x, edge1y) > high_threshold) side2 = weakEdgeMap_new * (interp.interp2(updatedMap, edge2x, edge2y) > high_threshold) sides = np.asarray(side1 + side2, dtype=bool) updatedMap = edgeMap + sides * high_threshold weakEdgeMap_new = weakEdgeMap - sides strongEdgeMap = np.asarray(strongEdgeMap + sides, dtype=bool) edge1x = weakEdgeMap_new * x1 edge1y = weakEdgeMap_new * y1 edge2x = weakEdgeMap_new * x2 edge2y = weakEdgeMap_new * y2 # print(np.sum(strongEdgeMap)) if (strongEdgeMap_old == strongEdgeMap).all(): return strongEdgeMap else: strongEdgeMap_old = strongEdgeMap
def nonMaxSup(Mag, Ori): # TODO: your code here X,Y=np.meshgrid(np.arange(Mag.shape[1]),np.arange(Mag.shape[0])) cosTheta=np.cos(Ori) sinTheta=np.sin(Ori) xq=X+cosTheta yq=Y+sinTheta side1=interp.interp2(Mag,xq,yq) Ori1=Ori+np.pi cosTheta1=np.cos(Ori1) sinTheta1=np.sin(Ori1) xq1=X+cosTheta1 yq1=Y+sinTheta1 side2=interp.interp2(Mag,xq1,yq1) grad1=Mag>side2 grad2=Mag>side1 ans=(np.logical_and(grad1,grad2))*1 return ans
def nonMaxSup(Mag, Ori): h, w = Mag.shape lx = np.cos(Ori) ly = np.sin(Ori) x, y = np.meshgrid(np.arange(w), np.arange(h)) xq_p = x + lx yq_p = y - ly xq_n = x - lx yq_n = y + ly Pos = interp2(Mag, xq_p, yq_p) Neg = interp2(Mag, xq_n, yq_n) M = np.int64(Mag - Pos > 0) * np.int64(Mag - Neg > 0) return M
def nonMaxSup(Mag, Ori): # Construct interpolation for upper point and lower point x, y = np.meshgrid(np.arange(Mag.shape[1]), np.arange(Mag.shape[0])) x_pri = x + np.cos(Ori) y_pri = y + np.sin(Ori) Mag_point1 = interp.interp2(Mag, x_pri, y_pri) X_pri = x - np.cos(Ori) Y_pri = y - np.sin(Ori) Mag_point2 = interp.interp2(Mag, X_pri, Y_pri) # Compare three matrices: Mag, Mag_point1, Mag_point2 # Only when Mag > Mag_point1 and Mag > Mag_point2, M=1 # if replace 1 with Mag, we will get a picture looks like Mag, but thinner M = np.where((Mag > Mag_point1) & (Mag > Mag_point2), 1, 0) # plt.imshow(M, cmap='gray') # plt.show() return M
def im_transform_2d(img, t, x=None, y=None, center=(0.0, 0.0)): img = np.asarray(img, dtype=np.float) H, W = img.shape if x is None: x = np.arange(W, dtype=np.float) - center[0] if y is None: y = np.arange(H, dtype=np.float) - center[1] Y, X = np.meshgrid(y, x, indexing='ij') Xt, Yt = pt_transform_2d(t, X, Y, inverse=True) oimg = interp.interp2(Xt, Yt, x, y, img) return oimg
def edgeLink(M, Mag, Ori): # set the high and low threshold highThresholdRatio = 2.5 lowThresholdRatio = 1.1 Mag1 = np.copy(Mag) highThreshold = np.mean(M * Mag1) * highThresholdRatio lowThreshold = np.mean(M * Mag1) * lowThresholdRatio # construct the next edge points' coordinate x, y = np.meshgrid(np.arange(Mag.shape[1]), np.arange(Mag.shape[0])) new_ori = np.tan(-1 / (np.tan(Ori) + 0.0001)) x_pri = x + np.cos(new_ori) y_pri = y + np.sin(new_ori) X_pri = x - np.cos(new_ori) Y_pri = y - np.sin(new_ori) # iterating until no further high threshold edge is detected oldNumOne = 0 numOne = 0.1 while (oldNumOne != numOne): oldNumOne = numOne Mag_point1 = interp.interp2(Mag1, x_pri, y_pri) Mag_point2 = interp.interp2(Mag1, X_pri, Y_pri) Mag1 = np.where((M == 1) & (Mag1 >= highThreshold), highThreshold, np.where((M == 1) & (Mag1 >= lowThreshold) & ((Mag_point1 >= highThreshold) | (Mag_point2 >= highThreshold)), highThreshold, Mag1)) numOne = np.sum(Mag1 == highThreshold) E = np.where(Mag1 == highThreshold, 1, 0) # plt.imshow(E, cmap='gray') # plt.show() return E
def estimateFeatureTranslation(startX, startY, Ix, Iy, img1, img2): prev_diff = float("inf") su,sv = 0,0 meshy, meshx = np.meshgrid(np.arange(startY-5,startY+5), np.arange(startX-5, startX+5)) img2_window = img2[startY-5:startY+5,startX-5:startX+5] while 1: It = img2 - img1 pIx = interp.interp2(Ix, meshx, meshy) pIy = interp.interp2(Iy, meshx, meshy) pIt = interp.interp2(It, meshx, meshy) xx = np.sum(np.multiply(pIx,pIx)) xy = np.sum(np.multiply(pIx,pIy)) yy = np.sum(np.multiply(pIy,pIy)) xt = np.sum(np.multiply(pIx,pIt)) yt = np.sum(np.multiply(pIy,pIt)) b = - np.array([xt,yt]).transpose() A = np.array([[xx+10e-6,xy],[xy,yy+10e-6]]) uv = np.matmul(np.linalg.inv(A),b) meshy,meshx = meshy+(uv[1]),meshx+(uv[0]) newImg = interp.interp2(img1, meshx, meshy) diff = np.linalg.norm(img2_window - newImg) if (prev_diff-diff) < 10e-6: # return np.round(startX + su), np.round(startY + sv) return startX + su, startY + sv #return np.round(startX + uv[0]), np.round(startY + uv[1]) else: su += uv[0] sv += uv[1] prev_diff = diff
def nonMaxSup(Mag, Ori): # TODO: your code here ''' Ori[Ori<0]=np.pi+ Ori[Ori<0] Ori[Ori>7*(np.pi/8)]=np.pi-Ori[Ori>7*(np.pi/8)] Ori[np.logical_and(Ori>=0,Ori<np.pi/8)]=0.0 Ori[np.logical_and(Ori>=np.pi/8,Ori<3*np.pi/8)]=np.pi/4 Ori[np.logical_and(Ori>=3*np.pi/8,Ori<5*(np.pi/8))]=np.pi/2 Ori[np.logical_and(Ori>=5*np.pi/8,Ori<7*np.pi/8)]=3*np.pi/4 #plt.imshow(ang1,cmap='gray') #print(ang1) fx=np.array([[-1,2,-1]]) fy=np.array([[-1],[2],[-1]]) fxy=np.array([[0,0,-1],[0,2,0],[-1,0,0]]) fyx=np.array([[-1,0,0],[0,2,0],[0,0,-1]]) ang1x=(Ori==0) ang1y=(Ori==np.pi/2) ang1xy=(Ori==np.pi/4) ang1yx=(Ori==3*np.pi/4) #edgex1=np.logical_and(signal.convolve2d(M,np.asarray([[-1,1,0]]),'same')>0, signal.convolve2d(M,np.asarray([[0,1,-1]]),'same')>0).astype(int) edgex=np.logical_and(signal.convolve2d(Mag,np.asarray([[-1,1.1,0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0,1.1,-1]]),'same')>0).astype(int) #edgex[edgex>0]=1.0 #edgex[edgex<=0]=0.0 edgex=edgex*ang1x edgex=edgex*Mag #print(edgex1) #edgey=signal.convolve2d(M,fy,'same') edgey=np.logical_and(signal.convolve2d(Mag,np.asarray([[-1],[1.1],[0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0],[1.1],[-1]]),'same')>0).astype(int) #edgey[edgey>0]=1.0 #edgey[edgey<=0]=0.0 edgey=edgey*ang1y edgey=edgey*Mag #edgexy1=np.logical_and(signal.convolve2d(M,np.asarray([[0,0,-1],[0,1,0],[0,0,0]]),'same'), signal.convolve2d(M,np.asarray([[0,0,0],[0,1,0],[-1,0,0]]),'same')).astype(int) edgexy=np.logical_and(signal.convolve2d(Mag,np.asarray([[0,0,-1],[0,1.1,0],[0,0,0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0,0,0],[0,1.1,0],[-1,0,0]]),'same')>0).astype(int) #edgexy=signal.convolve2d(M,fxy,'same') #edgexy[edgexy>0]=1.0 #edgexy[edgexy<=0]=0.0 edgexy=edgexy*ang1xy edgexy=edgexy*Mag #print(edgexy1) edgeyx=np.logical_and(signal.convolve2d(Mag,np.asarray([[-1,0,0],[0,1.1,0],[0,0,0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0,0,0],[0,1.1,0],[0,0,-1]]),'same')>0).astype(int) #edgeyx=signal.convolve2d(M,fyx,'same') #edgeyx[edgeyx>0]=1.0 #edgeyx[edgeyx<0]=0.0 edgeyx=edgeyx*ang1yx edgeyx=edgeyx*Mag #plt.imshow(edgex,cmap='gray') edge=edgex+edgey+edgexy+edgeyx return edge ''' #X, Y = np.meshgrid(np.arange(0, Mag.shape[1], 1), np.arange(0, Mag.shape[0], 1)) X, Y = np.meshgrid(np.arange(Mag.shape[1]), np.arange(Mag.shape[0])) Ori = +1 * Ori Xc = X + np.cos(Ori) Ys = Y + np.sin(Ori) Xc_neg = X - np.cos(Ori) Ys_neg = Y - np.sin(Ori) M_interp_pos = interp.interp2(Mag, Xc, Ys) M_interp_neg = interp.interp2(Mag, Xc_neg, Ys_neg) Magx = Mag > M_interp_pos Magy = Mag > M_interp_neg Mag1 = np.logical_and(Magx, Magy) return Mag1
def morph_tri(im1, im2, im1_pts, im2_pts, warp_frac, dissolve_frac): h1 = im1.shape[0] w1 = im1.shape[1] h2 = im2.shape[0] w2 = im2.shape[1] morphed_im = [] for t in range(len(warp_frac)): warp_value = warp_frac[t] dissolve_value = dissolve_frac[t] # find triangles for intermediate image # print("========intermediate image=========") intermediate_pts = (1 - warp_value) * im1_pts + warp_value * im2_pts Tri = Delaunay(intermediate_pts) simplices = Tri.simplices # find the inverse of matrix A for each intermediate triangle, #inverse_matrix=#triangles # print("========inverse of matrix A for intermediate triangle=========") temp = intermediate_pts[simplices].transpose( (0, 2, 1)) #get coordinates of convex points temp_1 = np.zeros([simplices.shape[0], 3, 3]) for t in range(len(temp)): temp_1[t, :, :] = np.linalg.inv(np.vstack((temp[t], [1, 1, 1]))) # find matrix A for img 1, #matrix=#triangles # print("========find matrix A for img 1=========") temp_img1 = im1_pts[simplices].transpose((0, 2, 1)) temp_1_img1 = np.zeros([simplices.shape[0], 3, 3]) for t in range(len(temp_img1)): temp_1_img1[t, :, :] = np.vstack((temp_img1[t], [1, 1, 1])) # find matrix A for img 2, #matrix=#triangles # print("========find matrix A for img 2=========") temp_img2 = im2_pts[simplices].transpose((0, 2, 1)) temp_1_img2 = np.zeros([simplices.shape[0], 3, 3]) for t in range(len(temp_img2)): temp_1_img2[t, :, :] = np.vstack((temp_img2[t], [1, 1, 1])) # get alpha beta gamma # print("========homogeneous coordinators=========") x, y = np.meshgrid(np.arange(w1), np.arange(h1)) x = x.flatten() y = y.flatten() simplex = Tri.find_simplex(list(zip( x, y))) # the index of triangle each pixel is in pt_A = temp_1[simplex] #get interm_A_inverse for each point alpha = pt_A[:, 0, 0] * x + pt_A[:, 0, 1] * y + pt_A[:, 0, 2] beta = pt_A[:, 1, 0] * x + pt_A[:, 1, 1] * y + pt_A[:, 1, 2] gamma = pt_A[:, 2, 0] * x + pt_A[:, 2, 1] * y + pt_A[:, 2, 2] # get coordinate in im1 # print("========get coordinate in im1=========") pt_A_img1 = temp_1_img1[simplex] #get im1_A matrix for each point x_t_1 = pt_A_img1[:, 0, 0] * alpha + pt_A_img1[:, 0, 1] * beta + pt_A_img1[:, 0, 2] * gamma y_t_1 = pt_A_img1[:, 1, 0] * alpha + pt_A_img1[:, 1, 1] * beta + pt_A_img1[:, 1, 2] * gamma z_t_1 = alpha + beta + gamma x_t_1 = x_t_1 / z_t_1 y_t_1 = y_t_1 / z_t_1 im1_warped = im1.copy() im1_warped[:, :, 0] = np.reshape(interp2(im1[:, :, 0], x_t_1, y_t_1), [h1, w1]) im1_warped[:, :, 1] = np.reshape(interp2(im1[:, :, 1], x_t_1, y_t_1), [h1, w1]) im1_warped[:, :, 2] = np.reshape(interp2(im1[:, :, 2], x_t_1, y_t_1), [h1, w1]) # print(sum(im1_warped[:,:,0] == im1[:,:,0])) # print(sum(im1_warped[:,:,1] == im1[:,:,1])) # print(sum(im1_warped[:,:,2] == im1[:,:,2])) # get coordinate in im2 # print("========get coordinate in im2=========") pt_A_img2 = temp_1_img2[simplex] #get im2_A matrix for each point x_t_2 = pt_A_img2[:, 0, 0] * alpha + pt_A_img2[:, 0, 1] * beta + pt_A_img2[:, 0, 2] * gamma y_t_2 = pt_A_img2[:, 1, 0] * alpha + pt_A_img2[:, 1, 1] * beta + pt_A_img2[:, 1, 2] * gamma z_t_2 = alpha + beta + gamma x_t_2 = x_t_2 / z_t_2 y_t_2 = y_t_2 / z_t_2 # im2 = im1 im2_warped = im2.copy() im2_warped[:, :, 0] = np.reshape(interp2(im2[:, :, 0], x_t_2, y_t_2), [h2, w2]) im2_warped[:, :, 1] = np.reshape(interp2(im2[:, :, 1], x_t_2, y_t_2), [h2, w2]) im2_warped[:, :, 2] = np.reshape(interp2(im2[:, :, 2], x_t_2, y_t_2), [h2, w2]) # print(sum(im2_warped[:,:,0] == im2[:,:,0])) # print(sum(im2_warped[:,:,1] == im2[:,:,1])) # print(sum(im2_warped[:,:,2] == im2[:,:,2])) # cross-dissolve # print("========cross-dissolve=========") morphed = im1_warped.copy() morphed[:, :, 0] = (1 - dissolve_value ) * im1_warped[:, :, 0] + dissolve_value * im2_warped[:, :, 0] morphed[:, :, 1] = (1 - dissolve_value ) * im1_warped[:, :, 1] + dissolve_value * im2_warped[:, :, 1] morphed[:, :, 2] = (1 - dissolve_value ) * im1_warped[:, :, 2] + dissolve_value * im2_warped[:, :, 2] # print(sum(morphed[:,:,0] == im1[:,:,0])) # print(sum(morphed[:,:,1] == im1[:,:,1])) # print(sum(morphed[:,:,2] == im1[:,:,2])) # print(sum(morphed[:,:,0] == im2[:,:,0])) # print(sum(morphed[:,:,1] == im2[:,:,1])) # print(sum(morphed[:,:,2] == im2[:,:,2])) # print(morphed.max(), morphed.min()) # putting together the morphed images # print("========put together=========") morphed_im.append(morphed) morphed_im = np.array(morphed_im) return morphed_im.astype('uint8')
# find points in Delaunay triangles target_face = np.zeros_like(img2, dtype=np.uint8) for ind, triangle in enumerate(tri.simplices): cartesian_coor = points[np.where(belong_to_tri == ind)] A = np.vstack( (img2_reference_points[triangle, :].T, np.array([1, 1, 1]))) b = np.hstack( (cartesian_coor, np.ones([cartesian_coor.shape[0], 1]))) barycentric_coor = np.linalg.inv(A).dot(b.T) img1_reference_points = np.vstack((landmarks1_trans, four_corners)) interp_position = barycentric_coor.T.dot( img1_reference_points[triangle, :]) for k in range(3): interp_1 = interp2(img1_trans[:, :, k], np.array([interp_position[:, 0]]), np.array([interp_position[:, 1]])).T target_face[:, :, k][cartesian_coor[:, 1], cartesian_coor[:, 0]] = interp_1.reshape(-1, ) time2 = time.time() # img1_reference_points = np.vstack((landmarks1_trans,four_corners)) # plt.figure(1) # plt.imshow(cv2.cvtColor(img1_trans,cv2.COLOR_RGB2BGR)) # plt.triplot(img1_reference_points[:,0], img1_reference_points[:,1], tri.simplices.copy()) # plt.figure(2) # plt.imshow(cv2.cvtColor(target_face,cv2.COLOR_RGB2BGR)) # plt.triplot(img2_reference_points[:,0], img2_reference_points[:,1], tri.simplices.copy()) # plt.figure(3) # plt.imshow(cv2.cvtColor(img2,cv2.COLOR_RGB2BGR))