def dfaker(landmarks, face, channels=4):
""" Dfaker model mask
Embeds the mask into the face alpha channel
channels: 1, 3 or 4:
1 - Return a single channel mask
3 - Return a 3 channel mask
4 - Return the original image with the mask in the alpha channel
"""
padding = int(face.shape[0] * 0.1875)
coverage = face.shape[0] - (padding * 2)
logger.trace("face_shape: %s, coverage: %s, landmarks: %s", face.shape,
coverage, landmarks)
mat = umeyama(landmarks[17:], True)[0:2]
mat = mat.reshape(-1).reshape(2, 3)
mat = mat * coverage
mat[:, 2] += padding
mask = np.zeros(face.shape[0:2] + (1, ), dtype=np.float32)
hull = cv2.convexHull(landmarks).reshape(1, -1, 2) # pylint: disable=no-member
hull = cv2.transform(hull, mat).reshape(-1, 2) # pylint: disable=no-member
cv2.fillConvexPoly(mask, hull, 255.) # pylint: disable=no-member
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15, 15)) # pylint: disable=no-member
mask = cv2.dilate(mask, kernel, borderType=cv2.BORDER_REFLECT) # pylint: disable=no-member
mask = np.expand_dims(mask, axis=-1)
return merge_mask(face, mask, channels)
def random_warp(self, image):
""" get pair of random warped images from aligned face image """
logger.trace("Randomly warping image")
height, width = image.shape[0:2]
coverage = self.get_coverage(image)
try:
assert height == width and height % 2 == 0
except AssertionError as err:
msg = (
"Training images should be square with an even number of pixels across each "
"side. An image was found with width: {}, height: {}."
"\nMost likely this is a frame rather than a face within your training set. "
"\nMake sure that the only images within your training set are faces generated "
"from the Extract process.".format(width, height))
raise FaceswapError(msg) from err
range_ = np.linspace(height // 2 - coverage // 2,
height // 2 + coverage // 2,
5,
dtype='float32')
mapx = np.broadcast_to(range_, (5, 5)).copy()
mapy = mapx.T
# mapx, mapy = np.float32(np.meshgrid(range_,range_)) # instead of broadcast
pad = int(1.25 * self.input_size)
slices = slice(pad // 10, -pad // 10)
dst_slices = [
slice(0, (size + 1), (size // 4)) for size in self.output_sizes
]
interp = np.empty((2, self.input_size, self.input_size),
dtype='float32')
for i, map_ in enumerate([mapx, mapy]):
map_ = map_ + np.random.normal(size=(5, 5), scale=self.scale)
interp[i] = cv2.resize(map_, (pad, pad))[slices, slices] # pylint:disable=no-member
warped_image = cv2.remap( # pylint:disable=no-member
image, interp[0], interp[1], cv2.INTER_LINEAR) # pylint:disable=no-member
logger.trace("Warped image shape: %s", warped_image.shape)
src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1)
dst_points = [
np.mgrid[dst_slice, dst_slice] for dst_slice in dst_slices
]
mats = [
umeyama(src_points, True, dst_pts.T.reshape(-1, 2))[0:2]
for dst_pts in dst_points
]
target_images = [
cv2.warpAffine(
image, # pylint:disable=no-member
mat,
(self.output_sizes[idx], self.output_sizes[idx]))
for idx, mat in enumerate(mats)
]
logger.trace("Target image shapes: %s",
[tgt.shape for tgt in target_images])
return self.compile_images(warped_image, target_images)
def get_align_mat(face, size, should_align_eyes):
mat_umeyama = umeyama(numpy.array(face.landmarks_as_xy()[17:]), landmarks_2D, True)[0:2]
if should_align_eyes is False:
return mat_umeyama
mat_umeyama = mat_umeyama * size
# Convert to matrix
landmarks = numpy.matrix(face.landmarks_as_xy())
# cv2 expects points to be in the form np.array([ [[x1, y1]], [[x2, y2]], ... ]), we'll expand the dim
landmarks = numpy.expand_dims(landmarks, axis=1)
# Align the landmarks using umeyama
umeyama_landmarks = cv2.transform(landmarks, mat_umeyama, landmarks.shape)
# Determine a rotation matrix to align eyes horizontally
mat_align_eyes = align_eyes(umeyama_landmarks, size)
# Extend the 2x3 transform matrices to 3x3 so we can multiply them
# and combine them as one
mat_umeyama = numpy.matrix(mat_umeyama)
mat_umeyama.resize((3, 3))
mat_align_eyes = numpy.matrix(mat_align_eyes)
mat_align_eyes.resize((3, 3))
mat_umeyama[2] = mat_align_eyes[2] = [0, 0, 1]
# Combine the umeyama transform with the extra rotation matrix
transform_mat = mat_align_eyes * mat_umeyama
# Remove the extra row added, shape needs to be 2x3
transform_mat = numpy.delete(transform_mat, 2, 0)
transform_mat = transform_mat / size
return transform_mat
def get_align_mat(face, size, should_align_eyes):
mat_umeyama = umeyama(numpy.array(face.landmarks_as_xy()[17:]), landmarks_2D, True)[0:2]
if should_align_eyes is False:
return mat_umeyama
mat_umeyama = mat_umeyama * size
# Convert to matrix
landmarks = numpy.matrix(face.landmarks_as_xy())
# cv2 expects points to be in the form np.array([ [[x1, y1]], [[x2, y2]], ... ]), we'll expand the dim
landmarks = numpy.expand_dims(landmarks, axis=1)
# Align the landmarks using umeyama
umeyama_landmarks = cv2.transform(landmarks, mat_umeyama, landmarks.shape)
# Determine a rotation matrix to align eyes horizontally
mat_align_eyes = align_eyes(umeyama_landmarks, size)
# Extend the 2x3 transform matrices to 3x3 so we can multiply them
# and combine them as one
mat_umeyama = numpy.matrix(mat_umeyama)
mat_umeyama.resize((3, 3))
mat_align_eyes = numpy.matrix(mat_align_eyes)
mat_align_eyes.resize((3, 3))
mat_umeyama[2] = mat_align_eyes[2] = [0, 0, 1]
# Combine the umeyama transform with the extra rotation matrix
transform_mat = mat_align_eyes * mat_umeyama
# Remove the extra row added, shape needs to be 2x3
transform_mat = numpy.delete(transform_mat, 2, 0)
transform_mat = transform_mat / size
return transform_mat
def random_warp(self, image):
""" get pair of random warped images from aligned face image """
logger.trace("Randomly warping image")
height, width = image.shape[0:2]
coverage = self.get_coverage(image)
assert height == width and height % 2 == 0
range_ = np.linspace(height // 2 - coverage // 2,
height // 2 + coverage // 2,
5,
dtype='float32')
mapx = np.broadcast_to(range_, (5, 5)).copy()
mapy = mapx.T
# mapx, mapy = np.float32(np.meshgrid(range_,range_)) # instead of broadcast
pad = int(1.25 * self.input_size)
slices = slice(pad // 10, -pad // 10)
dst_slice = slice(0, (self.output_size + 1), (self.output_size // 4))
interp = np.empty((2, self.input_size, self.input_size),
dtype='float32')
####
for i, map_ in enumerate([mapx, mapy]):
map_ = map_ + np.random.normal(size=(5, 5), scale=self.scale)
interp[i] = cv2.resize(map_, (pad, pad))[slices, slices] # pylint:disable=no-member
warped_image = cv2.remap( # pylint:disable=no-member
image, interp[0], interp[1], cv2.INTER_LINEAR) # pylint:disable=no-member
logger.trace("Warped image shape: %s", warped_image.shape)
src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1)
dst_points = np.mgrid[dst_slice, dst_slice]
mat = umeyama(src_points, True, dst_points.T.reshape(-1, 2))[0:2]
target_image = cv2.warpAffine( # pylint:disable=no-member
image, mat, (self.output_size, self.output_size))
logger.trace("Target image shape: %s", target_image.shape)
warped_image, warped_mask = self.separate_mask(warped_image)
target_image, target_mask = self.separate_mask(target_image)
if target_mask is None:
logger.trace("Randomly warped image")
return [warped_image, target_image]
logger.trace("Target mask shape: %s", target_mask.shape)
logger.trace("Randomly warped image and mask")
return [warped_image, target_image, target_mask]
def random_warp(self, image):
""" get pair of random warped images from aligned face image """
logger.trace("Randomly warping image")
height, width = image.shape[0:2]
coverage = self.get_coverage(image)
assert height == width and height % 2 == 0
range_ = np.linspace(height // 2 - coverage // 2,
height // 2 + coverage // 2,
5, dtype='float32')
mapx = np.broadcast_to(range_, (5, 5)).copy()
mapy = mapx.T
# mapx, mapy = np.float32(np.meshgrid(range_,range_)) # instead of broadcast
pad = int(1.25 * self.input_size)
slices = slice(pad // 10, -pad // 10)
dst_slice = slice(0, (self.output_size + 1), (self.output_size // 4))
interp = np.empty((2, self.input_size, self.input_size), dtype='float32')
####
for i, map_ in enumerate([mapx, mapy]):
map_ = map_ + np.random.normal(size=(5, 5), scale=self.scale)
interp[i] = cv2.resize(map_, (pad, pad))[slices, slices] # pylint: disable=no-member
warped_image = cv2.remap( # pylint: disable=no-member
image, interp[0], interp[1], cv2.INTER_LINEAR) # pylint: disable=no-member
logger.trace("Warped image shape: %s", warped_image.shape)
src_points = np.stack([mapx.ravel(), mapy.ravel()], axis=-1)
dst_points = np.mgrid[dst_slice, dst_slice]
mat = umeyama(src_points, True, dst_points.T.reshape(-1, 2))[0:2]
target_image = cv2.warpAffine( # pylint: disable=no-member
image, mat, (self.output_size, self.output_size))
logger.trace("Target image shape: %s", target_image.shape)
warped_image, warped_mask = self.separate_mask(warped_image)
target_image, target_mask = self.separate_mask(target_image)
if target_mask is None:
logger.trace("Randomly warped image")
return [warped_image, target_image]
logger.trace("Target mask shape: %s", target_mask.shape)
logger.trace("Randomly warped image and mask")
return [warped_image, target_image, target_mask]
def get_align_mat(face, size, should_align_eyes):
""" Return the alignment Matrix """
logger.trace("size: %s, should_align_eyes: %s", size, should_align_eyes)
mat_umeyama = umeyama(np.array(face.landmarks_xy[17:]), True)[0:2]
if should_align_eyes is False:
return mat_umeyama
mat_umeyama = mat_umeyama * size
# Convert to matrix
landmarks = np.matrix(face.landmarks_xy)
# cv2 expects points to be in the form
# np.array([ [[x1, y1]], [[x2, y2]], ... ]), we'll expand the dim
landmarks = np.expand_dims(landmarks, axis=1)
# Align the landmarks using umeyama
umeyama_landmarks = cv2.transform( # pylint: disable=no-member
landmarks,
mat_umeyama,
landmarks.shape)
# Determine a rotation matrix to align eyes horizontally
mat_align_eyes = func_align_eyes(umeyama_landmarks, size)
# Extend the 2x3 transform matrices to 3x3 so we can multiply them
# and combine them as one
mat_umeyama = np.matrix(mat_umeyama)
mat_umeyama.resize((3, 3))
mat_align_eyes = np.matrix(mat_align_eyes)
mat_align_eyes.resize((3, 3))
mat_umeyama[2] = mat_align_eyes[2] = [0, 0, 1]
# Combine the umeyama transform with the extra rotation matrix
transform_mat = mat_align_eyes * mat_umeyama
# Remove the extra row added, shape needs to be 2x3
transform_mat = np.delete(transform_mat, 2, 0)
transform_mat = transform_mat / size
logger.trace("Returning: %s", transform_mat)
return transform_mat
def get_align_mat(face, size, should_align_eyes):
""" Return the alignment Matrix """
logger.trace("size: %s, should_align_eyes: %s", size, should_align_eyes)
mat_umeyama = umeyama(np.array(face.landmarks_as_xy[17:]), True)[0:2]
if should_align_eyes is False:
return mat_umeyama
mat_umeyama = mat_umeyama * size
# Convert to matrix
landmarks = np.matrix(face.landmarks_as_xy)
# cv2 expects points to be in the form
# np.array([ [[x1, y1]], [[x2, y2]], ... ]), we'll expand the dim
landmarks = np.expand_dims(landmarks, axis=1)
# Align the landmarks using umeyama
umeyama_landmarks = cv2.transform( # pylint: disable=no-member
landmarks,
mat_umeyama,
landmarks.shape)
# Determine a rotation matrix to align eyes horizontally
mat_align_eyes = func_align_eyes(umeyama_landmarks, size)
# Extend the 2x3 transform matrices to 3x3 so we can multiply them
# and combine them as one
mat_umeyama = np.matrix(mat_umeyama)
mat_umeyama.resize((3, 3))
mat_align_eyes = np.matrix(mat_align_eyes)
mat_align_eyes.resize((3, 3))
mat_umeyama[2] = mat_align_eyes[2] = [0, 0, 1]
# Combine the umeyama transform with the extra rotation matrix
transform_mat = mat_align_eyes * mat_umeyama
# Remove the extra row added, shape needs to be 2x3
transform_mat = np.delete(transform_mat, 2, 0)
transform_mat = transform_mat / size
logger.trace("Returning: %s", transform_mat)
return transform_mat
def dfaker(landmarks, face, channels=4):
""" Dfaker model mask
Embeds the mask into the face alpha channel
channels: 1, 3 or 4:
1 - Return a single channel mask
3 - Return a 3 channel mask
4 - Return the original image with the mask in the alpha channel
"""
padding = int(face.shape[0] * 0.1875)
coverage = face.shape[0] - (padding * 2)
logger.trace("face_shape: %s, coverage: %s, landmarks: %s", face.shape,
coverage, landmarks)
mat = umeyama(landmarks[17:], True)[0:2]
mat = np.array(mat.ravel()).reshape(2, 3)
mat = mat * coverage
mat[:, 2] += padding
points = np.array(landmarks).reshape((-1, 2))
facepoints = np.array(points).reshape((-1, 2))
mask = np.zeros_like(face, dtype=np.uint8)
hull = cv2.convexHull(facepoints.astype(int)) # pylint: disable=no-member
hull = cv2.transform(
hull.reshape(1, -1, 2), # pylint: disable=no-member
mat).reshape(-1, 2).astype(int)
cv2.fillConvexPoly(mask, hull, (255, 255, 255)) # pylint: disable=no-member
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (15, 15)) # pylint: disable=no-member
mask = cv2.dilate(
mask, # pylint: disable=no-member
kernel,
iterations=1,
borderType=cv2.BORDER_REFLECT) # pylint: disable=no-member
mask = mask[:, :, :1]
return merge_mask(face, mask, channels)
def get_align_mat(face):
return umeyama(numpy.array(face.landmarksAsXY()[17:]), landmarks_2D,
True)[0:2]
def get_align_mat(face):
""" Return the alignment Matrix """
mat_umeyama = umeyama(np.array(face.landmarks_xy[17:]), True)[0:2]
return mat_umeyama
