def load_feed_face(self, image, size=64, coverage_ratio=0.625, dtype=None):
""" Align a face in the correct dimensions for feeding into a model.
Parameters
----------
image: numpy.ndarray
The image that contains the face to be aligned
size: int
The size of the face in pixels to be fed into the model
coverage_ratio: float, optional
the ratio of the extracted image that was used for training. Default: `0.625`
dtype: str, optional
Optionally set a ``dtype`` for the final face to be formatted in. Default: ``None``
Notes
-----
This method must be executed to get access to the following `properties`:
- :func:`feed_face`
- :func:`feed_interpolators`
"""
logger.trace("Loading feed face: (size: %s, coverage_ratio: %s, dtype: %s)",
size, coverage_ratio, dtype)
self.feed["size"] = size
self.feed["padding"] = self._padding_from_coverage(size, coverage_ratio)
self.feed["matrix"] = get_align_mat(self)
face = AlignerExtract().transform(image, self.feed["matrix"], size, self.feed["padding"])
face = np.clip(face[:, :, :3] / 255., 0., 1.)
self.feed["face"] = face if dtype is None else face.astype(dtype)
logger.trace("Loaded feed face. (face_shape: %s, matrix: %s)",
self.feed_face.shape, self._feed_matrix)
def process(self, output_item):
""" Detect and move blurry face """
extractor = AlignerExtract()
for face in output_item["detected_faces"]:
aligned_landmarks = face.aligned_landmarks
resized_face = face.aligned_face
size = face.aligned["size"]
feature_mask = extractor.get_feature_mask(
aligned_landmarks / size,
size, 48)
feature_mask = cv2.blur( # pylint: disable=no-member
feature_mask, (10, 10))
isolated_face = cv2.multiply( # pylint: disable=no-member
feature_mask,
resized_face.astype(float)).astype(np.uint8)
blurry, focus_measure = is_blurry(isolated_face, self.blur_thresh)
if blurry:
blur_folder = output_item["output_file"].parts[:-1]
blur_folder = get_folder(Path(*blur_folder) / Path("blurry"))
frame_name = output_item["output_file"].parts[-1]
output_item["output_file"] = blur_folder / Path(frame_name)
if self.verbose:
print("{}'s focus measure of {} was below the blur "
"threshold, moving to \"blurry\"".format(
frame_name, focus_measure))
def reference_matrix(self):
""" Return matrix for transforming output face back to image """
mat = AlignerExtract().transform_matrix(self.reference["matrix"],
self.reference["size"],
self.reference["padding"])
logger.trace("Returning: %s", mat)
return mat
def process(self, output_item):
""" Detect and move blurry face """
extractor = AlignerExtract()
for idx, detected_face in enumerate(output_item["detected_faces"]):
frame_name = detected_face["file_location"].parts[-1]
face = detected_face["face"]
logger.trace("Checking for blurriness. Frame: '%s', Face: %s",
frame_name, idx)
aligned_landmarks = face.aligned_landmarks
resized_face = face.aligned_face
size = face.aligned["size"]
padding = int(size * 0.1875)
feature_mask = extractor.get_feature_mask(aligned_landmarks / size,
size, padding)
feature_mask = cv2.blur(feature_mask, (10, 10))
isolated_face = cv2.multiply(feature_mask,
resized_face.astype(float)).astype(
np.uint8)
blurry, focus_measure = self.is_blurry(isolated_face)
if blurry:
blur_folder = detected_face["file_location"].parts[:-1]
blur_folder = get_folder(Path(*blur_folder) / Path("blurry"))
detected_face["file_location"] = blur_folder / Path(frame_name)
logger.verbose(
"%s's focus measure of %s was below the blur threshold, "
"moving to 'blurry'", frame_name,
"{0:.2f}".format(focus_measure))
def adjusted_matrix(self):
""" Return adjusted matrix for size/padding combination """
mat = AlignerExtract().transform_matrix(self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
logger.trace("Returning: %s", mat)
return mat
def load_reference_face(self,
image,
size=64,
coverage_ratio=0.625,
dtype=None):
""" Return a face in the correct dimensions for reference to the output from a NN
Coverage ratio should be the ratio of the extracted image that was used for
training """
logger.trace(
"Loading reference face: (size: %s, coverage_ratio: %s, dtype: %s)",
size, coverage_ratio, dtype)
self.reference["size"] = size
self.reference["padding"] = self.padding_from_coverage(
size, coverage_ratio)
self.reference["matrix"] = get_align_mat(self,
size,
should_align_eyes=False)
face = np.clip(
AlignerExtract().transform(image, self.reference["matrix"], size,
self.reference["padding"])[:, :, :3] /
255.0, 0.0, 1.0)
self.reference["face"] = face if dtype is None else face.astype(dtype)
logger.trace("Loaded reference face. (face_shape: %s, matrix: %s)",
self.reference_face.shape, self.reference_matrix)
def feed_matrix(self):
""" Return matrix for transforming feed face back to image """
mat = AlignerExtract().transform_matrix(self.feed["matrix"],
self.feed["size"],
self.feed["padding"])
logger.trace("Returning: %s", mat)
return mat
def load_aligned(self, image, size=256, align_eyes=False, dtype=None):
""" No need to load aligned information for all uses of this
class, so only call this to load the information for easy
reference to aligned properties for this face """
# Don't reload an already aligned face:
if self.aligned:
logger.trace(
"Skipping alignment calculation for already aligned face")
else:
logger.trace(
"Loading aligned face: (size: %s, align_eyes: %s, dtype: %s)",
size, align_eyes, dtype)
padding = int(size * self.extract_ratio) // 2
self.aligned["size"] = size
self.aligned["padding"] = padding
self.aligned["align_eyes"] = align_eyes
self.aligned["matrix"] = get_align_mat(self, size, align_eyes)
self.aligned["face"] = None
if image is not None and self.aligned["face"] is None:
logger.trace("Getting aligned face")
face = AlignerExtract().transform(image, self.aligned["matrix"],
size, padding)
self.aligned["face"] = face if dtype is None else face.astype(
dtype)
logger.trace(
"Loaded aligned face: %s",
{key: val
for key, val in self.aligned.items() if key != "face"})
def reference_landmarks(self):
""" Return the landmarks location transposed to reference face """
landmarks = AlignerExtract().transform_points(
self.landmarksXY, self.reference["matrix"], self.reference["size"],
self.reference["padding"])
logger.trace("Returning: %s", landmarks)
return landmarks
def original_roi(self):
""" Return the square aligned box location on the original
image """
roi = AlignerExtract().get_original_roi(self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
logger.trace("Returning: %s", roi)
return roi
def original_roi(self):
""" numpy.ndarray: The location of the extracted face box within the original frame.
Only available after :func:`load_aligned` has been called, otherwise returns ``None``"""
if not self.aligned:
return None
roi = AlignerExtract().get_original_roi(self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
logger.trace("Returning: %s", roi)
return roi
def _adjusted_matrix(self):
""" numpy.ndarray: Adjusted matrix for size/padding combination. Only available after
:func:`load_aligned` has been called, otherwise returns ``None``"""
if not self.aligned:
return None
mat = AlignerExtract().transform_matrix(self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
logger.trace("Returning: %s", mat)
return mat
def load_aligned(self, image, size=256, padding=48, align_eyes=False):
""" No need to load aligned information for all uses of this
class, so only call this to load the information for easy
reference to aligned properties for this face """
self.aligned["size"] = size
self.aligned["padding"] = padding
self.aligned["align_eyes"] = align_eyes
self.aligned["matrix"] = get_align_mat(self, size, align_eyes)
self.aligned["face"] = AlignerExtract().transform(
image, self.aligned["matrix"], size, padding)
def feed_matrix(self):
""" numpy.ndarray: The adjusted matrix face sized for feeding into a model. Only available
after :func:`load_feed_face` has been called with an image, otherwise returns ``None`` """
if not self.feed:
return None
mat = AlignerExtract().transform_matrix(self.feed["matrix"],
self.feed["size"],
self.feed["padding"])
logger.trace("Returning: %s", mat)
return mat
def load_aligned(self, image, size=256, align_eyes=False, dtype=None):
""" Align a face from a given image.
Aligning a face is a relatively expensive task and is not required for all uses of
the :class:`~lib.faces_detect.DetectedFace` object, so call this function explicitly to
load an aligned face.
This method plugs into :mod:`lib.aligner` to perform face alignment based on this face's
``landmarks_xy``. If the face has already been aligned, then this function will return
having performed no action.
Parameters
----------
image: numpy.ndarray
The image that contains the face to be aligned
size: int
The size of the output face in pixels
align_eyes: bool, optional
Optionally perform additional alignment to align eyes. Default: `False`
dtype: str, optional
Optionally set a ``dtype`` for the final face to be formatted in. Default: ``None``
Notes
-----
This method must be executed to get access to the following `properties`:
- :func:`original_roi`
- :func:`aligned_landmarks`
- :func:`aligned_face`
- :func:`adjusted_interpolators`
"""
if self.aligned:
# Don't reload an already aligned face
logger.trace(
"Skipping alignment calculation for already aligned face")
else:
logger.trace(
"Loading aligned face: (size: %s, align_eyes: %s, dtype: %s)",
size, align_eyes, dtype)
padding = int(size * self._extract_ratio) // 2
self.aligned["size"] = size
self.aligned["padding"] = padding
self.aligned["align_eyes"] = align_eyes
self.aligned["matrix"] = get_align_mat(self, size, align_eyes)
self.aligned["face"] = None
if image is not None and self.aligned["face"] is None:
logger.trace("Getting aligned face")
face = AlignerExtract().transform(image, self.aligned["matrix"],
size, padding)
self.aligned["face"] = face if dtype is None else face.astype(
dtype)
logger.trace(
"Loaded aligned face: %s",
{key: val
for key, val in self.aligned.items() if key != "face"})
def crop_destination_faces(self):
""" Update the main faces dict with new destination faces based on source matrices """
logger.debug("Updating destination faces")
self.faces["dst"] = list()
destination = self.destination if self.destination else [
np.ones_like(src["image"]) for src in self.source
]
for idx, image in enumerate(destination):
self.faces["dst"].append(AlignerExtract().transform(
image, self.faces["matrix"][idx], self.size, self.padding))
logger.debug("Updated destination faces")
def reference_matrix(self):
""" numpy.ndarray: The adjusted matrix face sized for reference against a face coming out of
a model. Only available after :func:`load_reference_face` has been called, otherwise
returns ``None``"""
if not self.reference:
return None
mat = AlignerExtract().transform_matrix(self.reference["matrix"],
self.reference["size"],
self.reference["padding"])
logger.trace("Returning: %s", mat)
return mat
def aligned_landmarks(self):
""" numpy.ndarray: The 68 point landmarks location transposed to the extracted face box.
Only available after :func:`load_aligned` has been called, otherwise returns ``None``"""
if not self.aligned:
return None
landmarks = AlignerExtract().transform_points(self.landmarks_xy,
self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
logger.trace("Returning: %s", landmarks)
return landmarks
def load_feed_face(self,
image,
size=64,
coverage_ratio=0.625,
dtype=None,
is_aligned_face=False):
""" Align a face in the correct dimensions for feeding into a model.
Parameters
----------
image: numpy.ndarray
The image that contains the face to be aligned
size: int
The size of the face in pixels to be fed into the model
coverage_ratio: float, optional
the ratio of the extracted image that was used for training. Default: `0.625`
dtype: str, optional
Optionally set a ``dtype`` for the final face to be formatted in. Default: ``None``
is_aligned_face: bool, optional
Indicates that the :attr:`image` is an aligned face rather than a frame.
Default: ``False``
Notes
-----
This method must be executed to get access to the following `properties`:
- :func:`feed_face`
- :func:`feed_interpolators`
"""
logger.trace(
"Loading feed face: (size: %s, coverage_ratio: %s, dtype: %s, "
"is_aligned_face: %s)", size, coverage_ratio, dtype,
is_aligned_face)
self.feed["size"] = size
self.feed["padding"] = self._padding_from_coverage(
size, coverage_ratio)
self.feed["matrix"] = get_align_mat(self)
if is_aligned_face:
original_size = image.shape[0]
interp = cv2.INTER_CUBIC if original_size < size else cv2.INTER_AREA
face = cv2.resize(image, (size, size), interpolation=interp)
else:
face = AlignerExtract().transform(image, self.feed["matrix"], size,
self.feed["padding"])
self.feed["face"] = face if dtype is None else face.astype(dtype)
logger.trace("Loaded feed face. (face_shape: %s, matrix: %s)",
self.feed_face.shape, self.feed_matrix)
def load_aligned(self, image, size=256, padding=48, align_eyes=False):
""" No need to load aligned information for all uses of this
class, so only call this to load the information for easy
reference to aligned properties for this face """
logger.trace(
"Loading aligned face: (size: %s, padding: %s, align_eyes: %s)",
size, padding, align_eyes)
self.aligned["size"] = size
self.aligned["padding"] = padding
self.aligned["align_eyes"] = align_eyes
self.aligned["matrix"] = get_align_mat(self, size, align_eyes)
self.aligned["face"] = AlignerExtract().transform(
image, self.aligned["matrix"], size, padding)
logger.trace(
"Loaded aligned face: %s",
{key: val
for key, val in self.aligned.items() if key != "face"})
def crop_source_faces(self):
""" Update the main faces dict with new source faces and matrices """
logger.debug("Updating source faces")
self.faces = dict()
for image in self.source:
detected_face = image["detected_faces"][0]
src_img = image["image"]
detected_face.load_aligned(src_img, self.size, align_eyes=False)
matrix = detected_face.aligned["matrix"]
self.faces.setdefault("filenames", list()).append(
os.path.splitext(image["filename"])[0])
self.faces.setdefault("matrix", list()).append(matrix)
self.faces.setdefault("src",
list()).append(AlignerExtract().transform(
src_img, matrix, self.size,
self.padding))
self.update_source = False
logger.debug("Updated source faces")
def align_eyes(face, image):
""" Re-extract a face with the pupils forced to be absolutely horizontally aligned """
umeyama_landmarks = face.aligned_landmarks
left_eye_center = umeyama_landmarks[42:48].mean(axis=0)
right_eye_center = umeyama_landmarks[36:42].mean(axis=0)
d_y = right_eye_center[1] - left_eye_center[1]
d_x = right_eye_center[0] - left_eye_center[0]
theta = np.pi - np.arctan2(d_y, d_x)
rot_cos = np.cos(theta)
rot_sin = np.sin(theta)
rotation_matrix = np.array([[rot_cos, -rot_sin, 0.],
[rot_sin, rot_cos, 0.], [0., 0., 1.]])
mat_umeyama = np.concatenate(
(face.aligned["matrix"], np.array([[0., 0., 1.]])), axis=0)
corrected_mat = np.dot(rotation_matrix, mat_umeyama)
face.aligned["matrix"] = corrected_mat[:2]
face.aligned["face"] = AlignerExtract().transform(
image, face.aligned["matrix"], face.aligned["size"],
int(face.aligned["size"] * 0.375) // 2)
logger.trace("Adjusted matrix: %s", face.aligned["matrix"])
return face
def adjusted_matrix(self):
""" Return adjusted matrix for size/padding combination """
return AlignerExtract().transform_matrix(self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
def original_roi(self):
""" Return the square aligned box location on the original
image """
return AlignerExtract().get_original_roi(self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
def aligned_landmarks(self):
""" Return the landmarks location transposed to extracted face """
return AlignerExtract().transform_points(self.landmarksXY,
self.aligned["matrix"],
self.aligned["size"],
self.aligned["padding"])
