def estimateFeatureTranslation(startX, startY, Ix, Iy, img1, img2):
X = startX
Y = startY
mesh_x, mesh_y = np.meshgrid(np.arange(WINDOW_SIZE),
np.arange(WINDOW_SIZE))
img1_gray = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY)
img2_gray = cv2.cvtColor(img2, cv2.COLOR_RGB2GRAY)
mesh_x_flat_fix = mesh_x.flatten() + X - np.floor(WINDOW_SIZE / 2)
mesh_y_flat_fix = mesh_y.flatten() + Y - np.floor(WINDOW_SIZE / 2)
coor_fix = np.vstack((mesh_x_flat_fix, mesh_y_flat_fix))
I1_value = interp2(img1_gray, coor_fix[[0], :], coor_fix[[1], :])
Ix_value = interp2(Ix, coor_fix[[0], :], coor_fix[[1], :])
Iy_value = interp2(Iy, coor_fix[[0], :], coor_fix[[1], :])
I = np.vstack((Ix_value, Iy_value))
A = I.dot(I.T)
for _ in range(15):
mesh_x_flat = mesh_x.flatten() + X - np.floor(WINDOW_SIZE / 2)
mesh_y_flat = mesh_y.flatten() + Y - np.floor(WINDOW_SIZE / 2)
coor = np.vstack((mesh_x_flat, mesh_y_flat))
I2_value = interp2(img2_gray, coor[[0], :], coor[[1], :])
Ip = (I2_value - I1_value).reshape((-1, 1))
b = -I.dot(Ip)
solution = inv(A).dot(b)
X += solution[0, 0]
Y += solution[1, 0]
return X, Y
def nonMaxSup(Mag, Ori):
(yLength, xLength) = Mag.shape
coordinates = (yLength, xLength)
coordinates_x = coordinates[1]
coordinates_y = coordinates[0]
# Initialize the meshgrid
mGridY, mGridX = np.meshgrid(np.arange(0, coordinates_x),
np.arange(0, coordinates_y))
# Modify the meshgrid with orientation
interp_location = np.add(mGridY, np.sin(Ori))
interp_location_2 = np.add(mGridX, np.cos(Ori))
interp_location_3 = np.subtract(mGridY, np.sin(Ori))
interp_location_4 = np.subtract(mGridX, np.cos(Ori))
# Interpolate and get magnitude of new points
interp_pos = interp2(mGridX, mGridY, Mag, interp_location,
interp_location_2)
interp_neg = interp2(mGridX, mGridY, Mag, interp_location_3,
interp_location_4)
# New matrix created by logical operation
Compare = np.logical_and(Mag > interp_pos, Mag > interp_neg)
# Change into binary matrix
Compare_int = Compare.astype(np.int)
return Compare_int
def edgeLink(M, Mag, Ori):
h, w = Mag.shape
M1 = M * Mag
mean = M1.sum() / M.sum()
#high=0.08
#low=0.035
high = 1 * mean
low = 0.2 * mean
High = high * np.ones([h, w])
Low = low * np.ones([h, w])
M = M * Mag
hpoint = np.int64(M - High > 0)
mpoint = np.int64(High - M > 0) * np.int64(M - Low > 0)
temp = mpoint
E = hpoint
lx_p = np.sin(Ori)
ly_p = np.cos(Ori)
x, y = np.meshgrid(np.arange(w), np.arange(h))
xq_p = x + lx_p
yq_p = y + ly_p
xq_n = x - lx_p
yq_n = y - ly_p
while (temp.sum() > 0):
Pos1 = interp2(E * Mag, xq_p, yq_p) * mpoint
Neg1 = interp2(E * Mag, xq_n, yq_n) * mpoint
temp = 1 * np.logical_or(np.int64(Pos1 - High > 0),
np.int64(Neg1 - High > 0))
E = E + temp
mpoint = mpoint - temp
return E
def edgeLink(M, Mag, Ori):
# TODO: your code here
edgeMap = M * Mag
updatedMap = edgeMap
mean = np.mean(Mag)
median = np.median(Mag)
std_dev = np.std(Mag)
# print("mean",mean)
# print("median",median)
# print("dev",std_dev)
# low_threshold=int(max(0, (1.0 - sigma) * np.median(Mag)))
# high_threshold=int(min(255, (1.0 + sigma) * np.median(Mag)))
high_threshold = mean * 0.9
low_threshold = mean * 0.65
print("high", high_threshold)
print("low", low_threshold)
# high_threshold=np.mean(Mag)*0.8
# low_threshold=np.mean(Mag)
strongEdgeMap = (edgeMap > high_threshold) * 1
weakEdgeMap = np.logical_and(edgeMap > low_threshold,
edgeMap < high_threshold) * 1
X, Y = np.meshgrid(np.arange(M.shape[1]), np.arange(M.shape[0]))
OriEdge1 = Ori + (np.pi / 2)
x1 = X + np.cos(OriEdge1)
y1 = Y + np.sin(OriEdge1)
OriEdge2 = Ori - (np.pi / 2)
x2 = X + np.cos(OriEdge2)
y2 = Y + np.sin(OriEdge2)
edge1x = weakEdgeMap * x1
edge1y = weakEdgeMap * y1
edge2x = weakEdgeMap * x2
edge2y = weakEdgeMap * y2
strongEdgeMap_old = np.empty(strongEdgeMap.shape)
weakEdgeMap_new = weakEdgeMap
while True:
side1 = weakEdgeMap_new * (interp.interp2(updatedMap, edge1x, edge1y) >
high_threshold)
side2 = weakEdgeMap_new * (interp.interp2(updatedMap, edge2x, edge2y) >
high_threshold)
sides = np.asarray(side1 + side2, dtype=bool)
updatedMap = edgeMap + sides * high_threshold
weakEdgeMap_new = weakEdgeMap - sides
strongEdgeMap = np.asarray(strongEdgeMap + sides, dtype=bool)
edge1x = weakEdgeMap_new * x1
edge1y = weakEdgeMap_new * y1
edge2x = weakEdgeMap_new * x2
edge2y = weakEdgeMap_new * y2
# print(np.sum(strongEdgeMap))
if (strongEdgeMap_old == strongEdgeMap).all():
return strongEdgeMap
else:
strongEdgeMap_old = strongEdgeMap
def nonMaxSup(Mag, Ori): # TODO: your code here X,Y=np.meshgrid(np.arange(Mag.shape[1]),np.arange(Mag.shape[0])) cosTheta=np.cos(Ori) sinTheta=np.sin(Ori) xq=X+cosTheta yq=Y+sinTheta side1=interp.interp2(Mag,xq,yq) Ori1=Ori+np.pi cosTheta1=np.cos(Ori1) sinTheta1=np.sin(Ori1) xq1=X+cosTheta1 yq1=Y+sinTheta1 side2=interp.interp2(Mag,xq1,yq1) grad1=Mag>side2 grad2=Mag>side1 ans=(np.logical_and(grad1,grad2))*1 return ans
def nonMaxSup(Mag, Ori):
h, w = Mag.shape
lx = np.cos(Ori)
ly = np.sin(Ori)
x, y = np.meshgrid(np.arange(w), np.arange(h))
xq_p = x + lx
yq_p = y - ly
xq_n = x - lx
yq_n = y + ly
Pos = interp2(Mag, xq_p, yq_p)
Neg = interp2(Mag, xq_n, yq_n)
M = np.int64(Mag - Pos > 0) * np.int64(Mag - Neg > 0)
return M
def nonMaxSup(Mag, Ori):
# Construct interpolation for upper point and lower point
x, y = np.meshgrid(np.arange(Mag.shape[1]), np.arange(Mag.shape[0]))
x_pri = x + np.cos(Ori)
y_pri = y + np.sin(Ori)
Mag_point1 = interp.interp2(Mag, x_pri, y_pri)
X_pri = x - np.cos(Ori)
Y_pri = y - np.sin(Ori)
Mag_point2 = interp.interp2(Mag, X_pri, Y_pri)
# Compare three matrices: Mag, Mag_point1, Mag_point2
# Only when Mag > Mag_point1 and Mag > Mag_point2, M=1
# if replace 1 with Mag, we will get a picture looks like Mag, but thinner
M = np.where((Mag > Mag_point1) & (Mag > Mag_point2), 1, 0)
# plt.imshow(M, cmap='gray')
# plt.show()
return M
def im_transform_2d(img, t, x=None, y=None, center=(0.0, 0.0)):
img = np.asarray(img, dtype=np.float)
H, W = img.shape
if x is None:
x = np.arange(W, dtype=np.float) - center[0]
if y is None:
y = np.arange(H, dtype=np.float) - center[1]
Y, X = np.meshgrid(y, x, indexing='ij')
Xt, Yt = pt_transform_2d(t, X, Y, inverse=True)
oimg = interp.interp2(Xt, Yt, x, y, img)
return oimg
def edgeLink(M, Mag, Ori):
# set the high and low threshold
highThresholdRatio = 2.5
lowThresholdRatio = 1.1
Mag1 = np.copy(Mag)
highThreshold = np.mean(M * Mag1) * highThresholdRatio
lowThreshold = np.mean(M * Mag1) * lowThresholdRatio
# construct the next edge points' coordinate
x, y = np.meshgrid(np.arange(Mag.shape[1]), np.arange(Mag.shape[0]))
new_ori = np.tan(-1 / (np.tan(Ori) + 0.0001))
x_pri = x + np.cos(new_ori)
y_pri = y + np.sin(new_ori)
X_pri = x - np.cos(new_ori)
Y_pri = y - np.sin(new_ori)
# iterating until no further high threshold edge is detected
oldNumOne = 0
numOne = 0.1
while (oldNumOne != numOne):
oldNumOne = numOne
Mag_point1 = interp.interp2(Mag1, x_pri, y_pri)
Mag_point2 = interp.interp2(Mag1, X_pri, Y_pri)
Mag1 = np.where((M == 1) & (Mag1 >= highThreshold), highThreshold,
np.where((M == 1) & (Mag1 >= lowThreshold)
& ((Mag_point1 >= highThreshold) |
(Mag_point2 >= highThreshold)),
highThreshold, Mag1))
numOne = np.sum(Mag1 == highThreshold)
E = np.where(Mag1 == highThreshold, 1, 0)
# plt.imshow(E, cmap='gray')
# plt.show()
return E
def estimateFeatureTranslation(startX, startY, Ix, Iy, img1, img2):
prev_diff = float("inf")
su,sv = 0,0
meshy, meshx = np.meshgrid(np.arange(startY-5,startY+5), np.arange(startX-5, startX+5))
img2_window = img2[startY-5:startY+5,startX-5:startX+5]
while 1:
It = img2 - img1
pIx = interp.interp2(Ix, meshx, meshy)
pIy = interp.interp2(Iy, meshx, meshy)
pIt = interp.interp2(It, meshx, meshy)
xx = np.sum(np.multiply(pIx,pIx))
xy = np.sum(np.multiply(pIx,pIy))
yy = np.sum(np.multiply(pIy,pIy))
xt = np.sum(np.multiply(pIx,pIt))
yt = np.sum(np.multiply(pIy,pIt))
b = - np.array([xt,yt]).transpose()
A = np.array([[xx+10e-6,xy],[xy,yy+10e-6]])
uv = np.matmul(np.linalg.inv(A),b)
meshy,meshx = meshy+(uv[1]),meshx+(uv[0])
newImg = interp.interp2(img1, meshx, meshy)
diff = np.linalg.norm(img2_window - newImg)
if (prev_diff-diff) < 10e-6:
# return np.round(startX + su), np.round(startY + sv)
return startX + su, startY + sv
#return np.round(startX + uv[0]), np.round(startY + uv[1])
else:
su += uv[0]
sv += uv[1]
prev_diff = diff
def nonMaxSup(Mag, Ori):
# TODO: your code here
'''
Ori[Ori<0]=np.pi+ Ori[Ori<0]
Ori[Ori>7*(np.pi/8)]=np.pi-Ori[Ori>7*(np.pi/8)]
Ori[np.logical_and(Ori>=0,Ori<np.pi/8)]=0.0
Ori[np.logical_and(Ori>=np.pi/8,Ori<3*np.pi/8)]=np.pi/4
Ori[np.logical_and(Ori>=3*np.pi/8,Ori<5*(np.pi/8))]=np.pi/2
Ori[np.logical_and(Ori>=5*np.pi/8,Ori<7*np.pi/8)]=3*np.pi/4
#plt.imshow(ang1,cmap='gray')
#print(ang1)
fx=np.array([[-1,2,-1]])
fy=np.array([[-1],[2],[-1]])
fxy=np.array([[0,0,-1],[0,2,0],[-1,0,0]])
fyx=np.array([[-1,0,0],[0,2,0],[0,0,-1]])
ang1x=(Ori==0)
ang1y=(Ori==np.pi/2)
ang1xy=(Ori==np.pi/4)
ang1yx=(Ori==3*np.pi/4)
#edgex1=np.logical_and(signal.convolve2d(M,np.asarray([[-1,1,0]]),'same')>0, signal.convolve2d(M,np.asarray([[0,1,-1]]),'same')>0).astype(int)
edgex=np.logical_and(signal.convolve2d(Mag,np.asarray([[-1,1.1,0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0,1.1,-1]]),'same')>0).astype(int)
#edgex[edgex>0]=1.0
#edgex[edgex<=0]=0.0
edgex=edgex*ang1x
edgex=edgex*Mag
#print(edgex1)
#edgey=signal.convolve2d(M,fy,'same')
edgey=np.logical_and(signal.convolve2d(Mag,np.asarray([[-1],[1.1],[0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0],[1.1],[-1]]),'same')>0).astype(int)
#edgey[edgey>0]=1.0
#edgey[edgey<=0]=0.0
edgey=edgey*ang1y
edgey=edgey*Mag
#edgexy1=np.logical_and(signal.convolve2d(M,np.asarray([[0,0,-1],[0,1,0],[0,0,0]]),'same'), signal.convolve2d(M,np.asarray([[0,0,0],[0,1,0],[-1,0,0]]),'same')).astype(int)
edgexy=np.logical_and(signal.convolve2d(Mag,np.asarray([[0,0,-1],[0,1.1,0],[0,0,0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0,0,0],[0,1.1,0],[-1,0,0]]),'same')>0).astype(int)
#edgexy=signal.convolve2d(M,fxy,'same')
#edgexy[edgexy>0]=1.0
#edgexy[edgexy<=0]=0.0
edgexy=edgexy*ang1xy
edgexy=edgexy*Mag
#print(edgexy1)
edgeyx=np.logical_and(signal.convolve2d(Mag,np.asarray([[-1,0,0],[0,1.1,0],[0,0,0]]),'same')>0, signal.convolve2d(Mag,np.asarray([[0,0,0],[0,1.1,0],[0,0,-1]]),'same')>0).astype(int)
#edgeyx=signal.convolve2d(M,fyx,'same')
#edgeyx[edgeyx>0]=1.0
#edgeyx[edgeyx<0]=0.0
edgeyx=edgeyx*ang1yx
edgeyx=edgeyx*Mag
#plt.imshow(edgex,cmap='gray')
edge=edgex+edgey+edgexy+edgeyx
return edge
'''
#X, Y = np.meshgrid(np.arange(0, Mag.shape[1], 1), np.arange(0, Mag.shape[0], 1))
X, Y = np.meshgrid(np.arange(Mag.shape[1]), np.arange(Mag.shape[0]))
Ori = +1 * Ori
Xc = X + np.cos(Ori)
Ys = Y + np.sin(Ori)
Xc_neg = X - np.cos(Ori)
Ys_neg = Y - np.sin(Ori)
M_interp_pos = interp.interp2(Mag, Xc, Ys)
M_interp_neg = interp.interp2(Mag, Xc_neg, Ys_neg)
Magx = Mag > M_interp_pos
Magy = Mag > M_interp_neg
Mag1 = np.logical_and(Magx, Magy)
return Mag1
def morph_tri(im1, im2, im1_pts, im2_pts, warp_frac, dissolve_frac):
h1 = im1.shape[0]
w1 = im1.shape[1]
h2 = im2.shape[0]
w2 = im2.shape[1]
morphed_im = []
for t in range(len(warp_frac)):
warp_value = warp_frac[t]
dissolve_value = dissolve_frac[t]
# find triangles for intermediate image
# print("========intermediate image=========")
intermediate_pts = (1 - warp_value) * im1_pts + warp_value * im2_pts
Tri = Delaunay(intermediate_pts)
simplices = Tri.simplices
# find the inverse of matrix A for each intermediate triangle, #inverse_matrix=#triangles
# print("========inverse of matrix A for intermediate triangle=========")
temp = intermediate_pts[simplices].transpose(
(0, 2, 1)) #get coordinates of convex points
temp_1 = np.zeros([simplices.shape[0], 3, 3])
for t in range(len(temp)):
temp_1[t, :, :] = np.linalg.inv(np.vstack((temp[t], [1, 1, 1])))
# find matrix A for img 1, #matrix=#triangles
# print("========find matrix A for img 1=========")
temp_img1 = im1_pts[simplices].transpose((0, 2, 1))
temp_1_img1 = np.zeros([simplices.shape[0], 3, 3])
for t in range(len(temp_img1)):
temp_1_img1[t, :, :] = np.vstack((temp_img1[t], [1, 1, 1]))
# find matrix A for img 2, #matrix=#triangles
# print("========find matrix A for img 2=========")
temp_img2 = im2_pts[simplices].transpose((0, 2, 1))
temp_1_img2 = np.zeros([simplices.shape[0], 3, 3])
for t in range(len(temp_img2)):
temp_1_img2[t, :, :] = np.vstack((temp_img2[t], [1, 1, 1]))
# get alpha beta gamma
# print("========homogeneous coordinators=========")
x, y = np.meshgrid(np.arange(w1), np.arange(h1))
x = x.flatten()
y = y.flatten()
simplex = Tri.find_simplex(list(zip(
x, y))) # the index of triangle each pixel is in
pt_A = temp_1[simplex] #get interm_A_inverse for each point
alpha = pt_A[:, 0, 0] * x + pt_A[:, 0, 1] * y + pt_A[:, 0, 2]
beta = pt_A[:, 1, 0] * x + pt_A[:, 1, 1] * y + pt_A[:, 1, 2]
gamma = pt_A[:, 2, 0] * x + pt_A[:, 2, 1] * y + pt_A[:, 2, 2]
# get coordinate in im1
# print("========get coordinate in im1=========")
pt_A_img1 = temp_1_img1[simplex] #get im1_A matrix for each point
x_t_1 = pt_A_img1[:, 0,
0] * alpha + pt_A_img1[:, 0,
1] * beta + pt_A_img1[:, 0,
2] * gamma
y_t_1 = pt_A_img1[:, 1,
0] * alpha + pt_A_img1[:, 1,
1] * beta + pt_A_img1[:, 1,
2] * gamma
z_t_1 = alpha + beta + gamma
x_t_1 = x_t_1 / z_t_1
y_t_1 = y_t_1 / z_t_1
im1_warped = im1.copy()
im1_warped[:, :, 0] = np.reshape(interp2(im1[:, :, 0], x_t_1, y_t_1),
[h1, w1])
im1_warped[:, :, 1] = np.reshape(interp2(im1[:, :, 1], x_t_1, y_t_1),
[h1, w1])
im1_warped[:, :, 2] = np.reshape(interp2(im1[:, :, 2], x_t_1, y_t_1),
[h1, w1])
# print(sum(im1_warped[:,:,0] == im1[:,:,0]))
# print(sum(im1_warped[:,:,1] == im1[:,:,1]))
# print(sum(im1_warped[:,:,2] == im1[:,:,2]))
# get coordinate in im2
# print("========get coordinate in im2=========")
pt_A_img2 = temp_1_img2[simplex] #get im2_A matrix for each point
x_t_2 = pt_A_img2[:, 0,
0] * alpha + pt_A_img2[:, 0,
1] * beta + pt_A_img2[:, 0,
2] * gamma
y_t_2 = pt_A_img2[:, 1,
0] * alpha + pt_A_img2[:, 1,
1] * beta + pt_A_img2[:, 1,
2] * gamma
z_t_2 = alpha + beta + gamma
x_t_2 = x_t_2 / z_t_2
y_t_2 = y_t_2 / z_t_2
# im2 = im1
im2_warped = im2.copy()
im2_warped[:, :, 0] = np.reshape(interp2(im2[:, :, 0], x_t_2, y_t_2),
[h2, w2])
im2_warped[:, :, 1] = np.reshape(interp2(im2[:, :, 1], x_t_2, y_t_2),
[h2, w2])
im2_warped[:, :, 2] = np.reshape(interp2(im2[:, :, 2], x_t_2, y_t_2),
[h2, w2])
# print(sum(im2_warped[:,:,0] == im2[:,:,0]))
# print(sum(im2_warped[:,:,1] == im2[:,:,1]))
# print(sum(im2_warped[:,:,2] == im2[:,:,2]))
# cross-dissolve
# print("========cross-dissolve=========")
morphed = im1_warped.copy()
morphed[:, :,
0] = (1 - dissolve_value
) * im1_warped[:, :,
0] + dissolve_value * im2_warped[:, :, 0]
morphed[:, :,
1] = (1 - dissolve_value
) * im1_warped[:, :,
1] + dissolve_value * im2_warped[:, :, 1]
morphed[:, :,
2] = (1 - dissolve_value
) * im1_warped[:, :,
2] + dissolve_value * im2_warped[:, :, 2]
# print(sum(morphed[:,:,0] == im1[:,:,0]))
# print(sum(morphed[:,:,1] == im1[:,:,1]))
# print(sum(morphed[:,:,2] == im1[:,:,2]))
# print(sum(morphed[:,:,0] == im2[:,:,0]))
# print(sum(morphed[:,:,1] == im2[:,:,1]))
# print(sum(morphed[:,:,2] == im2[:,:,2]))
# print(morphed.max(), morphed.min())
# putting together the morphed images
# print("========put together=========")
morphed_im.append(morphed)
morphed_im = np.array(morphed_im)
return morphed_im.astype('uint8')
# find points in Delaunay triangles
target_face = np.zeros_like(img2, dtype=np.uint8)
for ind, triangle in enumerate(tri.simplices):
cartesian_coor = points[np.where(belong_to_tri == ind)]
A = np.vstack(
(img2_reference_points[triangle, :].T, np.array([1, 1, 1])))
b = np.hstack(
(cartesian_coor, np.ones([cartesian_coor.shape[0], 1])))
barycentric_coor = np.linalg.inv(A).dot(b.T)
img1_reference_points = np.vstack((landmarks1_trans, four_corners))
interp_position = barycentric_coor.T.dot(
img1_reference_points[triangle, :])
for k in range(3):
interp_1 = interp2(img1_trans[:, :, k],
np.array([interp_position[:, 0]]),
np.array([interp_position[:, 1]])).T
target_face[:, :,
k][cartesian_coor[:, 1],
cartesian_coor[:, 0]] = interp_1.reshape(-1, )
time2 = time.time()
# img1_reference_points = np.vstack((landmarks1_trans,four_corners))
# plt.figure(1)
# plt.imshow(cv2.cvtColor(img1_trans,cv2.COLOR_RGB2BGR))
# plt.triplot(img1_reference_points[:,0], img1_reference_points[:,1], tri.simplices.copy())
# plt.figure(2)
# plt.imshow(cv2.cvtColor(target_face,cv2.COLOR_RGB2BGR))
# plt.triplot(img2_reference_points[:,0], img2_reference_points[:,1], tri.simplices.copy())
# plt.figure(3)
# plt.imshow(cv2.cvtColor(img2,cv2.COLOR_RGB2BGR))
