def _alignments_faces(self, frame_name, image):
""" Return detected faces from an alignments file.
Parameters
----------
frame_name: str
The name of the frame to return the detected faces for
image: :class:`numpy.ndarray`
The frame that the detected faces exist in
Returns
-------
list
List of :class:`lib.align.DetectedFace` objects
"""
if not self._check_alignments(frame_name):
return list()
faces = self._alignments.get_faces_in_frame(frame_name)
detected_faces = list()
for rawface in faces:
face = DetectedFace()
face.from_alignment(rawface, image=image)
detected_faces.append(face)
return detected_faces
def _annotate_image(self, frame_name):
""" Annotate the frame with each face that appears in the alignments file.
Parameters
----------
frame_name: str
The full path to the original frame
"""
logger.trace("Annotating frame: '%s'", frame_name)
image = self._frames.load_image(frame_name)
for idx, alignment in enumerate(
self._alignments.get_faces_in_frame(frame_name)):
face = DetectedFace()
face.from_alignment(alignment, image=image)
# Bounding Box
cv2.rectangle(image, (face.left, face.top),
(face.right, face.bottom), (255, 0, 0), 1)
self._annotate_landmarks(
image,
np.rint(face.landmarks_xy).astype("int32"))
self._annotate_extract_boxes(image, face, idx)
self._annotate_pose(image, face) # Pose (head is still loaded)
self._frames.save_image(self._output_folder, frame_name, image)
def _add_aligned_face(self, filename, alignments, image_size):
""" Add a :class:`lib.align.AlignedFace` object to the cache.
Parameters
----------
filename: str
The file path for the current image
alignments: dict
The alignments for a single face, extracted from a PNG header
image_size: int
The pixel size of the image loaded from disk
Returns
-------
:class:`lib.align.DetectedFace`
The Detected Face object that was used to create the Aligned Face
"""
if self._size is None:
self._size = get_centered_size(
"legacy" if self._extract_version == 1.0 else "head",
self._centering, image_size)
detected_face = DetectedFace()
detected_face.from_png_meta(alignments)
aligned_face = AlignedFace(detected_face.landmarks_xy,
centering=self._centering,
size=self._size,
is_aligned=True)
logger.trace("Caching aligned face for: %s", filename)
self._cache[os.path.basename(filename)]["aligned_face"] = aligned_face
return detected_face
def add(self, frame_index, pnt_x, width, pnt_y, height):
""" Add a :class:`~lib.align.DetectedFace` object to the current frame with the
given dimensions.
Parameters
----------
frame_index: int
The frame that the face is being set for
pnt_x: int
The left point of the bounding box
width: int
The width of the bounding box
pnt_y: int
The top point of the bounding box
height: int
The height of the bounding box
"""
face = DetectedFace()
faces = self._faces_at_frame_index(frame_index)
faces.append(face)
face_index = len(faces) - 1
self.bounding_box(frame_index, face_index, pnt_x, width, pnt_y, height, aligner="cv2-dnn")
face.load_aligned(None)
self._tk_face_count_changed.set(True)
def load(self):
""" Load the faces from the alignments file, convert to
:class:`~lib.align.DetectedFace`. objects and add to :attr:`_frame_faces`. """
for key in sorted(self._alignments.data):
this_frame_faces = []
for item in self._alignments.data[key]["faces"]:
face = DetectedFace()
face.from_alignment(item, with_thumb=True)
this_frame_faces.append(face)
self._frame_faces.append(this_frame_faces)
def _update_png_headers(self):
""" Update the EXIF iTXt field of any face PNGs that have had their face index changed.
Notes
-----
This could be quicker if parellizing in threads, however, Windows (at least) does not seem
to like this and has a tendency to throw permission errors, so this remains single threaded
for now.
"""
to_update = [ # Items whose face index has changed
x for x in self._items.file_list_sorted if x["face_index"] !=
self._items.items[x["source_filename"]].index(x["face_index"])
]
for file_info in tqdm(to_update,
desc="Updating PNG Headers",
leave=False):
frame = file_info["source_filename"]
face_index = file_info["face_index"]
new_index = self._items.items[frame].index(face_index)
fullpath = os.path.join(self._items.folder,
file_info["current_filename"])
logger.debug(
"Updating png header for '%s': face index from %s to %s",
fullpath, face_index, new_index)
# Update file_list_sorted for rename task
orig_filename = "{}_{}.png".format(
os.path.splitext(frame)[0], new_index)
file_info["face_index"] = new_index
file_info["original_filename"] = orig_filename
face = DetectedFace()
face.from_alignment(
self._alignments.get_faces_in_frame(frame)[new_index])
meta = dict(
alignments=face.to_png_meta(),
source=dict(
alignments_version=file_info["alignments_version"],
original_filename=orig_filename,
face_index=new_index,
source_filename=frame,
source_is_video=file_info["source_is_video"],
source_frame_dims=file_info.get("source_frame_dims")))
update_existing_metadata(fullpath, meta)
logger.info(
"%s Extracted face(s) had their header information updated",
len(to_update))
def estimate_blur_fft(cls, image, metadata=None):
""" Estimate the amount of blur a fft filtered image has.
Parameters
----------
image: :class:`numpy.ndarray`
Use Fourier Transform to analyze the frequency characteristics of the masked
face using 2D Discrete Fourier Transform (DFT) filter to find the frequency domain.
A mean value is assigned to the magnitude spectrum and returns a blur score.
Adapted from https://www.pyimagesearch.com/2020/06/15/
opencv-fast-fourier-transform-fft-for-blur-detection-in-images-and-video-streams/
metadata: dict, optional
The metadata for the face image or ``None`` if no metadata is available. If metadata is
provided the face will be masked by the "components" mask prior to calculating blur.
Default:``None``
Returns
-------
float
The estimated fft blur score for the face
"""
if metadata is not None:
alignments = metadata["alignments"]
det_face = DetectedFace()
det_face.from_png_meta(alignments)
aln_face = AlignedFace(np.array(alignments["landmarks_xy"],
dtype="float32"),
image=image,
centering="legacy",
size=256,
is_aligned=True)
mask = det_face.mask["components"]
mask.set_sub_crop(aln_face.pose.offset[mask.stored_centering] * -1,
centering="legacy")
mask = cv2.resize(mask.mask, (256, 256),
interpolation=cv2.INTER_CUBIC)[..., None]
image = np.minimum(aln_face.face, mask)
if image.ndim == 3:
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
height, width = image.shape
c_height, c_width = (int(height / 2.0), int(width / 2.0))
fft = np.fft.fft2(image)
fft_shift = np.fft.fftshift(fft)
fft_shift[c_height - 75:c_height + 75, c_width - 75:c_width + 75] = 0
ifft_shift = np.fft.ifftshift(fft_shift)
shift_back = np.fft.ifft2(ifft_shift)
magnitude = np.log(np.abs(shift_back))
score = np.mean(magnitude)
return score
def _get_detected_face(alignment):
""" Convert an alignment dict item to a detected_face object
Parameters
----------
alignment: dict
The alignment dict for a face
Returns
-------
:class:`lib.FacesDetect.detected_face`:
The corresponding detected_face object for the alignment
"""
detected_face = DetectedFace()
detected_face.from_alignment(alignment)
return detected_face
def extract_one_face(self, alignment, image):
""" Extract one face from image """
logger.trace("Extracting one face: (frame: '%s', alignment: %s)",
self.current_frame, alignment)
face = DetectedFace()
face.from_alignment(alignment, image=image)
face.load_aligned(image, size=self.size, centering="head")
face.thumbnail = generate_thumbnail(face.aligned.face, size=80, quality=60)
return face
def estimate_blur(cls, image, metadata=None):
""" Estimate the amount of blur an image has with the variance of the Laplacian.
Normalize by pixel number to offset the effect of image size on pixel gradients & variance.
Parameters
----------
image: :class:`numpy.ndarray`
The face image to calculate blur for
metadata: dict, optional
The metadata for the face image or ``None`` if no metadata is available. If metadata is
provided the face will be masked by the "components" mask prior to calculating blur.
Default:``None``
Returns
-------
float
The estimated blur score for the face
"""
if metadata is not None:
alignments = metadata["alignments"]
det_face = DetectedFace()
det_face.from_png_meta(alignments)
aln_face = AlignedFace(np.array(alignments["landmarks_xy"],
dtype="float32"),
image=image,
centering="legacy",
size=256,
is_aligned=True)
mask = det_face.mask["components"]
mask.set_sub_crop(aln_face.pose.offset[mask.stored_centering] * -1,
centering="legacy")
mask = cv2.resize(mask.mask, (256, 256),
interpolation=cv2.INTER_CUBIC)[..., None]
image = np.minimum(aln_face.face, mask)
if image.ndim == 3:
image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
blur_map = cv2.Laplacian(image, cv2.CV_32F)
score = np.var(blur_map) / np.sqrt(image.shape[0] * image.shape[1])
return score
def _add_remove_faces(cls, alignments, faces):
""" On a revert, ensure that the alignments and detected face object counts for each frame
are in sync. """
num_alignments = len(alignments)
num_faces = len(faces)
if num_alignments == num_faces:
retval = False
elif num_alignments > num_faces:
faces.extend([DetectedFace() for _ in range(num_faces, num_alignments)])
retval = True
else:
del faces[num_alignments:]
retval = True
return retval
def load(self):
""" Load the faces from the alignments file, convert to
:class:`~lib.align.DetectedFace`. objects and add to :attr:`_frame_faces`. """
for key in sorted(self._alignments.data):
this_frame_faces = []
for item in self._alignments.data[key]["faces"]:
face = DetectedFace()
face.from_alignment(item, with_thumb=True)
face.load_aligned(None)
_ = face.aligned.average_distance # cache the distances
this_frame_faces.append(face)
self._frame_faces.append(this_frame_faces)
self._sorted_frame_names = sorted(self._alignments.data)
def alignment_dict(filename, image):
""" Set the image to an ExtractMedia object for alignment """
height, width = image.shape[:2]
face = DetectedFace(x=0, w=width, y=0, h=height)
return ExtractMedia(filename, image, detected_faces=[face])
def process(self):
""" Run the job to remove faces from an alignments file that do not exist within a faces
folder. """
logger.info("[REMOVE FACES FROM ALIGNMENTS]") # Tidy up cli output
frame_face_indices = self._items.items
if not frame_face_indices:
logger.error(
"No matching faces found in your faces folder. This would remove all "
"faces from your alignments file. Process aborted.")
return
pre_face_count = self._alignments.faces_count
self._alignments.filter_faces(frame_face_indices, filter_out=False)
del_count = pre_face_count - self._alignments.faces_count
if del_count == 0:
logger.info("No changes made to alignments file. Exiting")
return
logger.info("%s alignment(s) were removed from alignments file",
del_count)
# PNG Header Updates
updated_headers = 0
for file_info in tqdm(self._items.file_list_sorted,
desc="Updating PNG Headers"):
frame = file_info["source_filename"]
face_index = file_info["face_index"]
new_index = frame_face_indices[frame].index(face_index)
if new_index == face_index: # face index has not changed
continue
fullpath = os.path.join(self._items.folder,
file_info["current_filename"])
logger.debug(
"Updating png header for '%s': face index from %s to %s",
fullpath, face_index, new_index)
# Update file_list_sorted for rename task
orig_filename = "{}_{}.png".format(
os.path.splitext(frame)[0], new_index)
file_info["face_index"] = new_index
file_info["original_filename"] = orig_filename
face = DetectedFace()
face.from_alignment(
self._alignments.get_faces_in_frame(frame)[new_index])
meta = dict(alignments=face.to_png_meta(),
source=dict(
alignments_version=file_info["alignments_version"],
original_filename=orig_filename,
face_index=new_index,
source_filename=frame,
source_is_video=file_info["source_is_video"]))
update_existing_metadata(fullpath, meta)
updated_headers += 1
logger.info(
"%s Extracted face(s) had their header information updated",
updated_headers)
self._alignments.save()
rename = Rename(self._alignments, None, self._items)
rename.process()
def to_detected_face(left, top, right, bottom):
""" Return a :class:`~lib.align.DetectedFace` object for the bounding box """
return DetectedFace(x=int(round(left)),
w=int(round(right - left)),
y=int(round(top)),
h=int(round(bottom - top)))
def __init__(self, arguments):
logger.debug("Initializing %s: (args: %s)", self.__class__.__name__, arguments)
self._args = arguments
# load faces
faces_alignments = AlignmentsBase(self._args.faces_align_dir)
print()
print(f'Faces alignments: {len(faces_alignments._data.keys())}')
print(faces_alignments._data.keys())
self._faces = {}
faces_loader = ImagesLoader(self._args.faces_dir)
for filename, image in faces_loader.load():
face_name = os.path.basename(filename)
faces = faces_alignments.get_faces_in_frame(face_name)
detected_faces = list()
for rawface in faces:
face = DetectedFace()
face.from_alignment(rawface, image=image)
feed_face = AlignedFace(face.landmarks_xy,
image=image,
centering='face',
size=image.shape[0],
coverage_ratio=1.0,
dtype="float32")
detected_faces.append(feed_face)
self._faces[face_name] = (filename, image, detected_faces)
print('Faces:', len(self._faces))
print(self._faces.keys())
print()
self._patch_threads = None
self._images = ImagesLoader(self._args.input_dir, fast_count=True)
self._alignments = Alignments(self._args, False, self._images.is_video)
if self._alignments.version == 1.0:
logger.error("The alignments file format has been updated since the given alignments "
"file was generated. You need to update the file to proceed.")
logger.error("To do this run the 'Alignments Tool' > 'Extract' Job.")
sys.exit(1)
self._opts = OptionalActions(self._args, self._images.file_list, self._alignments)
self._add_queues()
self._disk_io = DiskIO(self._alignments, self._images, arguments)
self._predictor = Predict(self._disk_io.load_queue, self._queue_size, self._faces, arguments)
self._validate()
get_folder(self._args.output_dir)
configfile = self._args.configfile if hasattr(self._args, "configfile") else None
self._converter = Converter(self._predictor.output_size,
self._predictor.coverage_ratio,
self._predictor.centering,
self._disk_io.draw_transparent,
self._disk_io.pre_encode,
arguments,
configfile=configfile)
logger.debug("Initialized %s", self.__class__.__name__)
