def __init__(self, BasePath, ModelPath, NumFeatures):
self.BasePath = BasePath
self.ModelPath = ModelPath
InputImageList = []
for filename in sorted(glob.glob(self.BasePath + '/*.jpg')):
ImageTemp = cv2.imread(filename)
InputImageList.append(ImageTemp)
self.NumFeatures = NumFeatures
self.Images = np.array(InputImageList)
self.NumImages = len(InputImageList)
self.HelperFunctions = HelperFunctions()
self.Model = DeepNetwork()
self.ImageUtils = ImageUtils()
self.ImageSize = InputImageList[0].shape
self.ImgPH = tf.placeholder('float', shape=(1, 128, 128, 2))
def __init__(self, ModelPath):
self.HelperFunctions = ImageUtils()
self.FaceDetector = dlib.get_frontal_face_detector()
self.ShapePredictor = dlib.shape_predictor(ModelPath)
class FaceSwapUtils:
def __init__(self, ModelPath):
self.HelperFunctions = ImageUtils()
self.FaceDetector = dlib.get_frontal_face_detector()
self.ShapePredictor = dlib.shape_predictor(ModelPath)
def getTriangulationCorrespondence(self, LandMarkPointsTarget,
DelaunayTriangleList,
LandMarkPointsSrc):
DelaunayTriangleListSrc = []
resultShape = DelaunayTriangleList.shape
for triangle in DelaunayTriangleList:
a = (triangle[0], triangle[1])
idA = LandMarkPointsTarget.index(a)
b = (triangle[2], triangle[3])
idB = LandMarkPointsTarget.index(b)
c = (triangle[4], triangle[5])
idC = LandMarkPointsTarget.index(c)
triangleIndices = LandMarkPointsSrc[idA], LandMarkPointsSrc[
idB], LandMarkPointsSrc[idC]
DelaunayTriangleListSrc.append(triangleIndices)
DelaunayTriangleListSrc = np.array(DelaunayTriangleListSrc)
DelaunayTriangleListSrc = np.reshape(DelaunayTriangleListSrc,
resultShape)
return DelaunayTriangleListSrc
def getMatrixBary(self, DelaunayTriangleList):
BMatrix = []
BInvMatrix = []
for triangle in DelaunayTriangleList:
ax = triangle[0]
ay = triangle[1]
bx = triangle[2]
by = triangle[3]
cx = triangle[4]
cy = triangle[5]
B = np.array([[ax, bx, cx], [ay, by, cy], [1, 1, 1]])
BMatrix.append(B)
BInv = np.linalg.inv(B)
BInvMatrix.append(BInv)
BMatrix = np.array(BMatrix)
BInvMatrix = np.array(BInvMatrix)
return BMatrix, BInvMatrix
def Blending(self, TargetImg, SrcImg, Mask):
radius = 3
kernel = np.ones((radius, radius), np.uint8)
Mask = cv2.dilate(Mask, kernel, iterations=1)
r = cv2.boundingRect(Mask)
center = ((r[0] + int(r[2] / 2), r[1] + int(r[3] / 2)))
return cv2.seamlessClone(TargetImg, SrcImg, Mask, center,
cv2.NORMAL_CLONE)
def DetectFacialLandmarks(self, InputImage):
GrayImage = cv2.cvtColor(InputImage, cv2.COLOR_BGR2GRAY)
dets = self.FaceDetector(GrayImage, 1)
isFace = False
for (_, det) in enumerate(dets):
isFace = True
shape = self.ShapePredictor(GrayImage, det)
shape = self.HelperFunctions.dlibFODObjectToNumpy(shape)
DelaunayBBox = (0, 0, InputImage.shape[1], InputImage.shape[0])
LandMarkPoints = []
for (x, y) in shape:
LandMarkPoints.append((x, y))
if (debug):
print("x: {} y: {}".format(x, y))
cv2.circle(InputImage, (x, y), 7, (0, 255, 0), -1)
if (debug):
print("Face Landmark Points: {}".format(LandMarkPoints))
if (isFace):
return isFace, DelaunayBBox, LandMarkPoints
else:
return isFace, [], []
def DetectFacialLandmarks2(self, InputImage):
GrayImage = cv2.cvtColor(InputImage, cv2.COLOR_BGR2GRAY)
dets = self.FaceDetector(GrayImage, 1)
isFace = False
LandMarkPoints = []
for (idX, det) in enumerate(dets):
isFace = True
shape = self.ShapePredictor(GrayImage, det)
shape = self.HelperFunctions.dlibFODObjectToNumpy(shape)
DelaunayBBox = (0, 0, InputImage.shape[1], InputImage.shape[0])
LandMarkPoints1 = []
for (x, y) in shape:
LandMarkPoints1.append((x, y))
if (debug):
print("x: {} y: {}".format(x, y))
cv2.circle(InputImage, (x, y), 7, (0, 255, 0), -1)
LandMarkPoints.append(LandMarkPoints1)
if (idX == 1):
break
if (debug):
print("Face Landmark Points: {}".format(LandMarkPoints))
if (isFace):
return isFace, DelaunayBBox, LandMarkPoints
else:
return isFace, [0], [0]
def GetDelaunayTriangulation(self, DelaunayBBox, LandMarkPoints,
InputImage):
subdiv = cv2.Subdiv2D(DelaunayBBox)
subdiv.insert(LandMarkPoints)
DelaunayTriangleList = subdiv.getTriangleList()
DelaunayTriangleList = DelaunayTriangleList[DelaunayTriangleList.min(
axis=1) >= 0, :]
DelaunayTriangleList = DelaunayTriangleList[DelaunayTriangleList.max(
axis=1) <= max(InputImage.shape[0], InputImage.shape[1]), :]
DelaunayTriangleListN = []
for triangle in DelaunayTriangleList:
a = (triangle[0], triangle[1])
b = (triangle[2], triangle[3])
c = (triangle[4], triangle[5])
if (self.HelperFunctions.isGoodAngle(a, b, c, 140)):
DelaunayTriangleListN.append(triangle)
DelaunayTriangleListN = np.array(DelaunayTriangleListN)
if (debug):
# print("DelaunayTriangleList: {}".format(DelaunayTriangleList))
# print(type(DelaunayTriangleList))
for triangle in DelaunayTriangleList:
a = (triangle[0], triangle[1])
b = (triangle[2], triangle[3])
c = (triangle[4], triangle[5])
cv2.line(InputImage, a, b, (0, 0, 255), 1, cv2.LINE_AA, 0)
cv2.line(InputImage, b, c, (0, 0, 255), 1, cv2.LINE_AA, 0)
cv2.line(InputImage, c, a, (0, 0, 255), 1, cv2.LINE_AA, 0)
self.HelperFunctions.ShowImage(InputImage, 'Facial Landmarks')
return DelaunayTriangleList
def __init__(self):
self.ImageUtils = ImageUtils()
class Test:
def __init__(self):
self.ImageUtils = ImageUtils()
def SetupAll(self, BasePath):
"""
Inputs:
BasePath - Path to images
Outputs:
ImageSize - Size of the Image
DataPath - Paths of all images where testing will be run on
"""
# Image Input Shape
ImageSize = [32, 32, 3]
DataPath = []
NumImages = len(glob.glob(BasePath + '*.jpg'))
SkipFactor = 1
for count in range(1, NumImages + 1, SkipFactor):
DataPath.append(BasePath + str(count) + '.jpg')
return ImageSize, DataPath
def ReadImages(self, ImageSize, DataPath):
"""
Inputs:
ImageSize - Size of the Image
DataPath - Paths of all images where testing will be run on
Outputs:
I1Combined - I1 image after any standardization and/or cropping/resizing to ImageSize
I1 - Original I1 image for visualization purposes only
"""
ImageName = DataPath
I1 = cv2.imread(ImageName)
if (I1 is None):
# OpenCV returns empty list if image is not read!
print('ERROR: Image I1 cannot be read')
sys.exit()
##########################################################################
# Add any standardization or cropping/resizing if used in Training here!
##########################################################################
Im = self.ImageUtils.PreProcess(I1, 640, 480)
I1S, H4PtTruth1 = self.ImageUtils.CreateTrainingData(Im, 256, 256, 64)
I2S, H4PtTruth2 = self.ImageUtils.CreateTrainingData(Im, 256, 256, 64)
I1Combined = np.expand_dims(I1S, axis=0)
return I1Combined, I1
def TestOperation(self, ImgPH, ImageSize, ModelPath, DataPath,
LabelsPathPred):
"""
Inputs:
ImgPH is the Input Image placeholder
ImageSize is the size of the image
ModelPath - Path to load trained model from
DataPath - Paths of all images where testing will be run on
LabelsPathPred - Path to save predictions
Outputs:
Predictions written to ./TxtFiles/PredOut.txt
"""
Length = ImageSize[0]
# Predict output with forward pass, MiniBatchSize for Test is 1
_, prSoftMaxS = HomographyModel(ImgPH, ImageSize, 1)
# Setup Saver
Saver = tf.train.Saver()
with tf.Session() as sess:
Saver.restore(sess, ModelPath)
print('Number of parameters in this model are %d ' % np.sum([
np.prod(v.get_shape().as_list())
for v in tf.trainable_variables()
]))
OutSaveT = open(LabelsPathPred, 'w')
for count in tqdm(range(np.size(DataPath))):
DataPathNow = DataPath[count]
Img, ImgOrg = self.ReadImages(ImageSize, DataPathNow)
FeedDict = {ImgPH: Img}
PredT = np.argmax(sess.run(prSoftMaxS, FeedDict))
OutSaveT.write(str(PredT) + '\n')
OutSaveT.close()
def ReadLabels(self, LabelsPathTest, LabelsPathPred):
if (not (os.path.isfile(LabelsPathTest))):
print('ERROR: Test Labels do not exist in ' + LabelsPathTest)
sys.exit()
else:
LabelTest = open(LabelsPathTest, 'r')
LabelTest = LabelTest.read()
LabelTest = map(float, LabelTest.split())
if (not (os.path.isfile(LabelsPathPred))):
print('ERROR: Pred Labels do not exist in ' + LabelsPathPred)
sys.exit()
else:
LabelPred = open(LabelsPathPred, 'r')
LabelPred = LabelPred.read()
LabelPred = map(float, LabelPred.split())
return LabelTest, LabelPred
def __init__(self, ModelPath):
self.ModelPath = ModelPath
self.HelperFunctions = ImageUtils()
self.FaceSwapFunctions = FaceSwapUtils(self.ModelPath)
class FaceSwapVideoFaces:
def __init__(self, ModelPath):
self.ModelPath = ModelPath
self.HelperFunctions = ImageUtils()
self.FaceSwapFunctions = FaceSwapUtils(self.ModelPath)
def FaceWarpByTriangulation(self, frame, weight):
TargetImg = frame.copy()
SrcImg = frame.copy()
CloneSrcImg = TargetImg.copy()
Mask = np.zeros(TargetImg.shape, TargetImg.dtype)
isFace1, DelaunayBBoxTarget, LandMarkPoints = self.FaceSwapFunctions.DetectFacialLandmarks2(
TargetImg)
if (not isFace1 or len(LandMarkPoints) < 2):
return TargetImg
LandMarkPointsTarget = LandMarkPoints[0]
LandMarkPointsSrc = LandMarkPoints[1]
DelaunayTrianglesTarget = self.FaceSwapFunctions.GetDelaunayTriangulation(
DelaunayBBoxTarget, LandMarkPointsTarget, TargetImg)
DelaunayTrianglesTarget2 = self.FaceSwapFunctions.GetDelaunayTriangulation(
DelaunayBBoxTarget, LandMarkPointsSrc, TargetImg)
DelaunayTrianglesSrc = self.FaceSwapFunctions.getTriangulationCorrespondence(
LandMarkPointsTarget, DelaunayTrianglesTarget, LandMarkPointsSrc)
points = np.array(LandMarkPointsTarget)
hull = cv2.convexHull(points)
color = (255, 255, 255)
cv2.fillPoly(Mask, [hull], color)
Mask = cv2.cvtColor(Mask, cv2.COLOR_BGR2GRAY)
# self.HelperFunctions.ShowImage(Mask, 'Mask')
_, BInvMatrix = self.FaceSwapFunctions.getMatrixBary(
DelaunayTrianglesTarget)
AMatrix, _ = self.FaceSwapFunctions.getMatrixBary(DelaunayTrianglesSrc)
triangle_number = 0
for triangle in DelaunayTrianglesTarget:
xMin, yMin, xMax, yMax = self.HelperFunctions.getTriangleBBox(
triangle)
BInv = BInvMatrix[triangle_number]
for x in range(int(xMin), int(xMax + 1)):
for y in range(int(yMin), int(yMax + 1)):
BarycentricCoordinatesTarget = np.matmul(
BInv, np.array([[x], [y], [1]]))
if ((BarycentricCoordinatesTarget[0] >= 0
and BarycentricCoordinatesTarget[0] <= 1)
and (BarycentricCoordinatesTarget[1] >= 0
and BarycentricCoordinatesTarget[1] <= 1)
and (BarycentricCoordinatesTarget[2] >= 0
and BarycentricCoordinatesTarget[2] <= 1)):
PixelLocSrc = np.matmul(AMatrix[triangle_number],
BarycentricCoordinatesTarget)
PixelLocSrc = np.divide(PixelLocSrc, PixelLocSrc[2])
TargetImg[y][x] = weight * SrcImg[int(
PixelLocSrc[1])][int(PixelLocSrc[0])] + (
1 - weight) * TargetImg[y][x]
triangle_number += 1
TargetImg1 = self.FaceSwapFunctions.Blending(TargetImg, CloneSrcImg,
Mask)
CloneSrcImg = TargetImg1
# self.HelperFunctions.ShowImage(TargetImg1, 'TargetImg1')
#################################
Mask = np.zeros(TargetImg1.shape, TargetImg1.dtype)
DelaunayTrianglesSrc = self.FaceSwapFunctions.getTriangulationCorrespondence(
LandMarkPointsSrc, DelaunayTrianglesTarget2, LandMarkPointsTarget)
points = np.array(LandMarkPointsSrc)
hull = cv2.convexHull(points)
color = (255, 255, 255)
cv2.fillPoly(Mask, [hull], color)
Mask = cv2.cvtColor(Mask, cv2.COLOR_BGR2GRAY)
# self.HelperFunctions.ShowImage(Mask, 'Mask')
_, BInvMatrix = self.FaceSwapFunctions.getMatrixBary(
DelaunayTrianglesTarget2)
AMatrix, _ = self.FaceSwapFunctions.getMatrixBary(DelaunayTrianglesSrc)
triangle_number = 0
for triangle in DelaunayTrianglesTarget2:
xMin, yMin, xMax, yMax = self.HelperFunctions.getTriangleBBox(
triangle)
BInv = BInvMatrix[triangle_number]
for x in range(int(xMin), int(xMax + 1)):
for y in range(int(yMin), int(yMax + 1)):
BarycentricCoordinatesTarget = np.matmul(
BInv, np.array([[x], [y], [1]]))
if ((BarycentricCoordinatesTarget[0] >= 0
and BarycentricCoordinatesTarget[0] <= 1)
and (BarycentricCoordinatesTarget[1] >= 0
and BarycentricCoordinatesTarget[1] <= 1)
and (BarycentricCoordinatesTarget[2] >= 0
and BarycentricCoordinatesTarget[2] <= 1)):
PixelLocSrc = np.matmul(AMatrix[triangle_number],
BarycentricCoordinatesTarget)
PixelLocSrc = np.divide(PixelLocSrc, PixelLocSrc[2])
TargetImg1[y][x] = weight * SrcImg[int(
PixelLocSrc[1])][int(PixelLocSrc[0])] + (
1 - weight) * TargetImg1[y][x]
triangle_number += 1
# self.HelperFunctions.ShowImage(TargetImg1, 'Target Image')
FaceSwapOP = self.FaceSwapFunctions.Blending(TargetImg1, CloneSrcImg,
Mask)
if debug:
self.HelperFunctions.ShowImage(TargetImg, 'Target Image')
self.HelperFunctions.ShowImage(CloneSrcImg, 'Source Image')
FaceSwapOP = cv2.resize(FaceSwapOP, (720, 720))
self.HelperFunctions.ShowImage(
FaceSwapOP, 'FaceSwap using Delaunay Triangulation')
return FaceSwapOP
def FaceWarpByTPS(self, frame):
TargetImg = frame
SrcImg = frame.copy()
CloneSrcImg = TargetImg.copy()
isFace1, _, LandMarkPoints = self.FaceSwapFunctions.DetectFacialLandmarks2(
TargetImg)
if (not isFace1 or len(LandMarkPoints) < 2):
return TargetImg
LandMarkPointsTarget = LandMarkPoints[0]
LandMarkPointsSrc = LandMarkPoints[1]
p = len(LandMarkPointsTarget)
K = np.zeros([p, p])
# K matrix
for idX in range(p):
for idY in range(p):
K[idX][idY] = self.HelperFunctions.FuncUofR(
np.linalg.norm(np.array(LandMarkPointsTarget[idX]) -
np.array(LandMarkPointsTarget[idY]),
ord=2))
# P matrix
P = np.zeros([p, 3])
for i, (x, y) in enumerate(LandMarkPointsTarget):
P[i] = (x, y, 1)
# lambda I matrix
I = np.identity(p + 3)
lamb = 1e-4
# V matrix
vx = np.zeros([p + 3])
vy = np.zeros([p + 3])
for i, (x, y) in enumerate(LandMarkPointsSrc):
vx[i] = x
vy[i] = y
vx = np.reshape(vx, (p + 3, 1))
vy = np.reshape(vy, (p + 3, 1))
t1 = np.hstack((K, P))
t2 = np.hstack((np.transpose(P), np.zeros([3, 3])))
M = np.vstack((t1, t2))
resultX = np.dot(np.linalg.inv(M + lamb * I), vx)
resultY = np.dot(np.linalg.inv(M + lamb * I), vy)
wX = resultX[0:p]
axX = resultX[p]
axY = resultX[p + 1]
ax1 = resultX[p + 2]
wY = resultY[0:p]
ayX = resultY[p]
ayY = resultY[p + 1]
ay1 = resultY[p + 2]
points = np.array(LandMarkPointsTarget)
hull = cv2.convexHull(points)
Mask = np.zeros(TargetImg.shape, TargetImg.dtype)
if debug:
print("Points inside convex hull: {}".format(hull))
Mask = np.zeros((TargetImg.shape[0], TargetImg.shape[1], 3), np.uint8)
color = (255, 255, 255)
cv2.fillPoly(Mask, [hull], color)
Mask = Mask[:, :, 1]
ptsY = np.where(Mask == 255)[0]
ptsX = np.where(Mask == 255)[1]
ptsY = np.transpose(ptsY)
ptsX = np.transpose(ptsX)
pts = np.vstack((ptsX, ptsY))
pts = np.transpose(pts)
if debug:
print("K: {}".format(K))
print("P: {}".format(P))
print("M: {}".format(M))
print("vx: {}".format(vx))
print("vy: {}".format(vy))
print("resultX: {}".format(resultX))
print("resultY: {}".format(resultY))
self.HelperFunctions.ShowImage(Mask, 'Convex hull for face')
for Lpts in pts:
U1 = (points - Lpts)
U1 = np.linalg.norm(U1, ord=2, axis=1)
U1 = self.HelperFunctions.FuncUofR(U1)
wUX = np.matmul(np.transpose(wX), U1)
wUY = np.matmul(np.transpose(wY), U1)
fX = int(ax1 + axX * Lpts[0] + axY * Lpts[1] + wUX)
fY = int(ay1 + ayX * Lpts[0] + ayY * Lpts[1] + wUY)
if fX < 0 or fY < 0:
continue
TargetImg[Lpts[1]][Lpts[0]] = SrcImg[fY][fX]
TargetImg1 = self.FaceSwapFunctions.Blending(TargetImg, CloneSrcImg,
Mask)
CloneSrcImg = TargetImg1
########################
p = len(LandMarkPointsSrc)
K = np.zeros([p, p])
# K matrix
for idX in range(p):
for idY in range(p):
K[idX][idY] = self.HelperFunctions.FuncUofR(
np.linalg.norm(np.array(LandMarkPointsSrc[idX]) -
np.array(LandMarkPointsSrc[idY]),
ord=2))
# P matrix
P = np.zeros([p, 3])
for i, (x, y) in enumerate(LandMarkPointsSrc):
P[i] = (x, y, 1)
# lambda I matrix
I = np.identity(p + 3)
lamb = 1e-4
# V matrix
vx = np.zeros([p + 3])
vy = np.zeros([p + 3])
for i, (x, y) in enumerate(LandMarkPointsTarget):
vx[i] = x
vy[i] = y
vx = np.reshape(vx, (p + 3, 1))
vy = np.reshape(vy, (p + 3, 1))
t1 = np.hstack((K, P))
t2 = np.hstack((np.transpose(P), np.zeros([3, 3])))
M = np.vstack((t1, t2))
resultX = np.dot(np.linalg.inv(M + lamb * I), vx)
resultY = np.dot(np.linalg.inv(M + lamb * I), vy)
wX = resultX[0:p]
axX = resultX[p]
axY = resultX[p + 1]
ax1 = resultX[p + 2]
wY = resultY[0:p]
ayX = resultY[p]
ayY = resultY[p + 1]
ay1 = resultY[p + 2]
points = np.array(LandMarkPointsSrc)
hull = cv2.convexHull(points)
Mask = np.zeros(TargetImg.shape, TargetImg.dtype)
if debug:
print("Points inside convex hull: {}".format(hull))
Mask = np.zeros((TargetImg1.shape[0], TargetImg1.shape[1], 3),
np.uint8)
color = (255, 255, 255)
cv2.fillPoly(Mask, [hull], color)
Mask = Mask[:, :, 1]
ptsY = np.where(Mask == 255)[0]
ptsX = np.where(Mask == 255)[1]
ptsY = np.transpose(ptsY)
ptsX = np.transpose(ptsX)
pts = np.vstack((ptsX, ptsY))
pts = np.transpose(pts)
if debug:
print("K: {}".format(K))
print("P: {}".format(P))
print("M: {}".format(M))
print("vx: {}".format(vx))
print("vy: {}".format(vy))
print("resultX: {}".format(resultX))
print("resultY: {}".format(resultY))
self.HelperFunctions.ShowImage(Mask, 'Convex hull for face')
for Lpts in pts:
U1 = (points - Lpts)
U1 = np.linalg.norm(U1, ord=2, axis=1)
U1 = self.HelperFunctions.FuncUofR(U1)
wUX = np.matmul(np.transpose(wX), U1)
wUY = np.matmul(np.transpose(wY), U1)
fX = int(ax1 + axX * Lpts[0] + axY * Lpts[1] + wUX)
fY = int(ay1 + ayX * Lpts[0] + ayY * Lpts[1] + wUY)
if fX < 0 or fY < 0:
continue
TargetImg1[Lpts[1]][Lpts[0]] = SrcImg[fY][fX]
FaceSwapOP = self.FaceSwapFunctions.Blending(TargetImg1, CloneSrcImg,
Mask)
# self.HelperFunctions.ShowImage(
# FaceSwapOP, 'FaceSwap using Thin Plate Spline')
return FaceSwapOP
def __init__(self, Image_Path_1, Image_Path_2, ModelPath):
self.ModelPath = ModelPath
InputImageList = [cv2.imread(Image_Path_1), cv2.imread(Image_Path_2)]
self.Images = np.array(InputImageList)
self.HelperFunctions = ImageUtils()
self.FaceSwapFunctions = FaceSwapUtils(self.ModelPath)
def __init__(self, Image_Path, ModelPath):
self.ModelPath = ModelPath
# ImageTemp = cv2.resize(ImageTemp, (1000, 800))
self.Image = cv2.imread(Image_Path)
self.HelperFunctions = ImageUtils()
self.FaceSwapFunctions = FaceSwapUtils(self.ModelPath)
class Stitcher:
"""
Read a set of images for Panorama stitching
"""
def __init__(self, BasePath, ModelPath, NumFeatures):
self.BasePath = BasePath
self.ModelPath = ModelPath
InputImageList = []
for filename in sorted(glob.glob(self.BasePath + '/*.jpg')):
ImageTemp = cv2.imread(filename)
InputImageList.append(ImageTemp)
self.NumFeatures = NumFeatures
self.Images = np.array(InputImageList)
self.NumImages = len(InputImageList)
self.HelperFunctions = HelperFunctions()
self.Model = DeepNetwork()
self.ImageUtils = ImageUtils()
self.ImageSize = InputImageList[0].shape
self.ImgPH = tf.placeholder('float', shape=(1, 128, 128, 2))
"""
Obtain Homography using Deep Learning Model (Supervised and Unsupervised)
"""
def ExtractHomographyFromH4Pt(self, H4PtPred):
pts1 = np.float32([[0, 0], [self.ImageSize[1], 0],
[self.ImageSize[1], self.ImageSize[0]],
[0, self.ImageSize[0]]])
pts2 = np.float32(
[[0 + H4PtPred[0][0], 0 + H4PtPred[0][4]],
[self.ImageSize[1] + H4PtPred[0][1], 0 + H4PtPred[0][5]],
[
self.ImageSize[1] + H4PtPred[0][2],
self.ImageSize[0] + H4PtPred[0][6]
], [0 + H4PtPred[0][3], self.ImageSize[0] + H4PtPred[0][7]]])
HPred = cv2.getPerspectiveTransform(pts1, pts2)
return HPred
def EstimateHomographySupervised(self, Image1, Image2):
# Setup Saver
H4PtPred = self.Model.HomographyNet(self.ImgPH, False)
Saver = tf.train.Saver()
with tf.Session() as sess:
Saver.restore(sess, self.ModelPath)
print('Number of parameters in this model are %d ' % np.sum([
np.prod(v.get_shape().as_list())
for v in tf.trainable_variables()
]))
Image1 = np.float32(self.ImageUtils.PreProcess(Image1, 128, 128))
Image1 = self.ImageUtils.ImageStandardization(Image1)
Image2 = np.float32(self.ImageUtils.PreProcess(Image2, 128, 128))
Image2 = self.ImageUtils.ImageStandardization(Image2)
Images = np.dstack((Image1, Image2))
I1Batch = []
I1Batch.append(Images)
FeedDict = {self.ImgPH: I1Batch}
H4Pt = sess.run(H4PtPred, FeedDict)
print("H4pt: {}".format(H4Pt))
H = self.ExtractHomographyFromH4Pt(H4Pt)
Hinv = np.linalg.inv(H)
return H, Hinv
"""
Image Warping + Blending
Save Panorama output as mypano.png
"""
def RemoveBlackBoundary(self, ImageIn):
gray = cv2.cvtColor(ImageIn, cv2.COLOR_BGR2GRAY)
_, thresh = cv2.threshold(gray, 1, 255, cv2.THRESH_BINARY)
_, contours, _ = cv2.findContours(thresh, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnt = contours[0]
x, y, w, h = cv2.boundingRect(cnt)
ImageOut = ImageIn[y:y + h, x:x + w]
return ImageOut
def Warping(self, Img, Homography, NextShape):
nH, nW, _ = Img.shape
Borders = np.array([[0, nW, nW, 0], [0, 0, nH, nH], [1, 1, 1, 1]])
BordersNew = np.dot(Homography, Borders)
Ymin = min(BordersNew[1] / BordersNew[2])
Xmin = min(BordersNew[0] / BordersNew[2])
Ymax = max(BordersNew[1] / BordersNew[2])
Xmax = max(BordersNew[0] / BordersNew[2])
if Ymin < 0:
MatChange = np.array([[1, 0, -1 * Xmin], [0, 1, -1 * Ymin],
[0, 0, 1]])
Hnew = np.dot(MatChange, Homography)
h = int(round(Ymax - Ymin)) + NextShape[0]
else:
MatChange = np.array([[1, 0, -1 * Xmin], [0, 1, Ymin], [0, 0, 1]])
Hnew = np.dot(MatChange, Homography)
h = int(round(Ymax + Ymin)) + NextShape[0]
w = int(round(Xmax - Xmin)) + NextShape[1]
sz = (w, h)
PanoHolder = cv2.warpPerspective(Img, Hnew, dsize=sz)
return PanoHolder, int(Xmin), int(Ymin)
def Blender(self):
Pano = self.Images[0]
for NextImage in self.Images[1:2]:
H, Hinv = self.EstimateHomographySupervised(Pano, NextImage)
PanoHolder, oX, oY = self.Warping(Pano, H, NextImage.shape)
self.HelperFunctions.ShowImage(PanoHolder, 'PanoHolder')
oX = abs(oX)
oY = abs(oY)
for IdY in range(oY, NextImage.shape[0] + oY):
for IdX in range(oX, NextImage.shape[1] + oX):
y = IdY - oY
x = IdX - oX
PanoHolder[IdY, IdX, :] = NextImage[y, x, :]
# Pano = self.RemoveBlackBoundary(PanoHolder)
PanoResize = cv2.resize(Pano, (1280, 1024))
self.HelperFunctions.ShowImage(PanoResize, 'PanoResize')
PanoResize = cv2.GaussianBlur(PanoResize, (5, 5), 1.2)
return PanoResize
def main():
Parser = argparse.ArgumentParser()
Parser.add_argument(
'--Mode',
default='0',
help=
'FaceSwap Mode. 0: Swap faces in two images. 1: Swap largest face in video with user defined image. 2: Swap two faces in single video, Default:0'
)
Parser.add_argument(
'--FaceSwapMethod',
default='1',
help=
'FaceSwap FaceSwapMethod. 0: Triangulation. 1: Thin Plate Spline, Default:0'
)
Parser.add_argument(
'--Path1',
default='/home/rohith/CMSC733/git/FaceSwap/Data/anakin2.jpg',
help=
'Video path for Mode 1 and 2. Target image for Mode 0, Default:/home/rohith/CMSC733/git/FaceSwap/Data/elon.mp4'
)
Parser.add_argument(
'--Path2',
default='/home/rohith/CMSC733/git/FaceSwap/Data/rohith.jpeg',
help=
'Image path to be swapped for Mode 0 and 1. Dont care for Mode2, Default:/home/rohith/CMSC733/git/FaceSwap/Data/padme2.jpg'
)
Parser.add_argument(
'--ModelPath',
default=
'/home/rohith/CMSC733/git/FaceSwap/Models/shape_predictor_68_face_landmarks.dat',
help=
'Model path of dlib predictor, Default:/home/rohith/CMSC733/git/FaceSwap/Models/shape_predictor_68_face_landmarks.dat'
)
Args = Parser.parse_args()
Mode = Args.Mode
FaceSwapMethod = Args.FaceSwapMethod
Path1 = Args.Path1
Path2 = Args.Path2
ModelPath = Args.ModelPath
HelperFunctions = ImageUtils()
if Mode == '0':
if FaceSwapMethod == '0':
print(
"**********Triangulation method started for FaceSwap in two images**********"
)
swapFacesTri = FaceSwapImages(Path1, Path2, ModelPath)
faceSwapOP = swapFacesTri.FaceWarpByTriangulation(1)
HelperFunctions.ShowImage(faceSwapOP,
'Triangulation method FaceSwap')
print(
"**********Triangulation method ended for FaceSwap in two images**********"
)
elif FaceSwapMethod == '1':
print(
"**********TPS method started for FaceSwap in two images**********"
)
swapFacesTPS = FaceSwapImages(Path1, Path2, ModelPath)
faceSwapOP = swapFacesTPS.FaceWarpByTPS()
HelperFunctions.ShowImage(faceSwapOP, 'TPS method FaceSwap')
print(
"**********TPS method ended for FaceSwap in two images**********"
)
elif Mode == '1':
cap = cv2.VideoCapture(Path1)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
w = int(cap.get(3))
h = int(cap.get(4))
if FaceSwapMethod == '0':
print(
"**********Triangulation method started for swapping face in video with image**********"
)
swapFaces1 = FaceSwapVideoWithImage(Path2, ModelPath)
out1 = cv2.VideoWriter('DataOutputTri.avi', fourcc, 20.0, (w, h))
while (cap.isOpened()):
ret, frame = cap.read()
if ret == True:
frame = swapFaces1.FaceWarpByTriangulation(frame, 1)
out1.write(frame)
# cv2.imshow('frame', frame)
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
else:
break
cap.release()
out1.release()
cv2.destroyAllWindows()
print(
"**********Triangulation method ended for swapping face in video with image**********"
)
elif FaceSwapMethod == '1':
print(
"**********TPS method started for swapping face in video with image**********"
)
# i = 0
swapFaces2 = FaceSwapVideoWithImage(Path2, ModelPath)
out2 = cv2.VideoWriter('DataOutputTPS.avi', fourcc, 20.0, (w, h))
while (cap.isOpened()):
ret, frame = cap.read()
if ret == True:
frame = swapFaces2.FaceWarpByTPS(frame)
out2.write(frame)
# print(i)
# i += 1
# cv2.imshow('frame', frame)
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
else:
break
cap.release()
out2.release()
cv2.destroyAllWindows()
print(
"**********TPS method ended for swapping face in video with image**********"
)
elif Mode == '2':
cap = cv2.VideoCapture(Path1)
fourcc = cv2.VideoWriter_fourcc(*'XVID')
w = int(cap.get(3))
h = int(cap.get(4))
if FaceSwapMethod == '0':
print(
"**********Triangulation method started for swapping face in video**********"
)
swapFaces1 = FaceSwapVideoFaces(ModelPath)
out1 = cv2.VideoWriter('DataOutputTri.avi', fourcc, 20.0, (w, h))
while (cap.isOpened()):
ret, frame = cap.read()
if ret == True:
frame = swapFaces1.FaceWarpByTriangulation(frame, 1)
out1.write(frame)
# cv2.imshow('frame', frame)
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
else:
break
cap.release()
out1.release()
cv2.destroyAllWindows()
print(
"**********Triangulation method ended for swapping face in video**********"
)
elif FaceSwapMethod == '1':
print(
"**********TPS method started for swapping face in video**********"
)
i = 0
swapFaces2 = FaceSwapVideoFaces(ModelPath)
out2 = cv2.VideoWriter('DataOutputTPS.avi', fourcc, 20.0, (w, h))
while (cap.isOpened()):
ret, frame = cap.read()
if ret == True:
frame = swapFaces2.FaceWarpByTPS(frame)
out2.write(frame)
# print(i)
# i += 1
# cv2.imshow('frame', frame)
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
else:
break
cap.release()
out2.release()
cv2.destroyAllWindows()
print(
"**********TPS method ended for swapping face in video**********"
)