def _alignments_faces(self, frame_name, image): """ Return detected faces from an alignments file. Parameters ---------- frame_name: str The name of the frame to return the detected faces for image: :class:`numpy.ndarray` The frame that the detected faces exist in Returns ------- list List of :class:`lib.align.DetectedFace` objects """ if not self._check_alignments(frame_name): return list() faces = self._alignments.get_faces_in_frame(frame_name) detected_faces = list() for rawface in faces: face = DetectedFace() face.from_alignment(rawface, image=image) detected_faces.append(face) return detected_faces
def _annotate_image(self, frame_name): """ Annotate the frame with each face that appears in the alignments file. Parameters ---------- frame_name: str The full path to the original frame """ logger.trace("Annotating frame: '%s'", frame_name) image = self._frames.load_image(frame_name) for idx, alignment in enumerate( self._alignments.get_faces_in_frame(frame_name)): face = DetectedFace() face.from_alignment(alignment, image=image) # Bounding Box cv2.rectangle(image, (face.left, face.top), (face.right, face.bottom), (255, 0, 0), 1) self._annotate_landmarks( image, np.rint(face.landmarks_xy).astype("int32")) self._annotate_extract_boxes(image, face, idx) self._annotate_pose(image, face) # Pose (head is still loaded) self._frames.save_image(self._output_folder, frame_name, image)
def _add_aligned_face(self, filename, alignments, image_size): """ Add a :class:`lib.align.AlignedFace` object to the cache. Parameters ---------- filename: str The file path for the current image alignments: dict The alignments for a single face, extracted from a PNG header image_size: int The pixel size of the image loaded from disk Returns ------- :class:`lib.align.DetectedFace` The Detected Face object that was used to create the Aligned Face """ if self._size is None: self._size = get_centered_size( "legacy" if self._extract_version == 1.0 else "head", self._centering, image_size) detected_face = DetectedFace() detected_face.from_png_meta(alignments) aligned_face = AlignedFace(detected_face.landmarks_xy, centering=self._centering, size=self._size, is_aligned=True) logger.trace("Caching aligned face for: %s", filename) self._cache[os.path.basename(filename)]["aligned_face"] = aligned_face return detected_face
def add(self, frame_index, pnt_x, width, pnt_y, height): """ Add a :class:`~lib.align.DetectedFace` object to the current frame with the given dimensions. Parameters ---------- frame_index: int The frame that the face is being set for pnt_x: int The left point of the bounding box width: int The width of the bounding box pnt_y: int The top point of the bounding box height: int The height of the bounding box """ face = DetectedFace() faces = self._faces_at_frame_index(frame_index) faces.append(face) face_index = len(faces) - 1 self.bounding_box(frame_index, face_index, pnt_x, width, pnt_y, height, aligner="cv2-dnn") face.load_aligned(None) self._tk_face_count_changed.set(True)
def load(self): """ Load the faces from the alignments file, convert to :class:`~lib.align.DetectedFace`. objects and add to :attr:`_frame_faces`. """ for key in sorted(self._alignments.data): this_frame_faces = [] for item in self._alignments.data[key]["faces"]: face = DetectedFace() face.from_alignment(item, with_thumb=True) this_frame_faces.append(face) self._frame_faces.append(this_frame_faces)
def _update_png_headers(self): """ Update the EXIF iTXt field of any face PNGs that have had their face index changed. Notes ----- This could be quicker if parellizing in threads, however, Windows (at least) does not seem to like this and has a tendency to throw permission errors, so this remains single threaded for now. """ to_update = [ # Items whose face index has changed x for x in self._items.file_list_sorted if x["face_index"] != self._items.items[x["source_filename"]].index(x["face_index"]) ] for file_info in tqdm(to_update, desc="Updating PNG Headers", leave=False): frame = file_info["source_filename"] face_index = file_info["face_index"] new_index = self._items.items[frame].index(face_index) fullpath = os.path.join(self._items.folder, file_info["current_filename"]) logger.debug( "Updating png header for '%s': face index from %s to %s", fullpath, face_index, new_index) # Update file_list_sorted for rename task orig_filename = "{}_{}.png".format( os.path.splitext(frame)[0], new_index) file_info["face_index"] = new_index file_info["original_filename"] = orig_filename face = DetectedFace() face.from_alignment( self._alignments.get_faces_in_frame(frame)[new_index]) meta = dict( alignments=face.to_png_meta(), source=dict( alignments_version=file_info["alignments_version"], original_filename=orig_filename, face_index=new_index, source_filename=frame, source_is_video=file_info["source_is_video"], source_frame_dims=file_info.get("source_frame_dims"))) update_existing_metadata(fullpath, meta) logger.info( "%s Extracted face(s) had their header information updated", len(to_update))
def estimate_blur_fft(cls, image, metadata=None): """ Estimate the amount of blur a fft filtered image has. Parameters ---------- image: :class:`numpy.ndarray` Use Fourier Transform to analyze the frequency characteristics of the masked face using 2D Discrete Fourier Transform (DFT) filter to find the frequency domain. A mean value is assigned to the magnitude spectrum and returns a blur score. Adapted from https://www.pyimagesearch.com/2020/06/15/ opencv-fast-fourier-transform-fft-for-blur-detection-in-images-and-video-streams/ metadata: dict, optional The metadata for the face image or ``None`` if no metadata is available. If metadata is provided the face will be masked by the "components" mask prior to calculating blur. Default:``None`` Returns ------- float The estimated fft blur score for the face """ if metadata is not None: alignments = metadata["alignments"] det_face = DetectedFace() det_face.from_png_meta(alignments) aln_face = AlignedFace(np.array(alignments["landmarks_xy"], dtype="float32"), image=image, centering="legacy", size=256, is_aligned=True) mask = det_face.mask["components"] mask.set_sub_crop(aln_face.pose.offset[mask.stored_centering] * -1, centering="legacy") mask = cv2.resize(mask.mask, (256, 256), interpolation=cv2.INTER_CUBIC)[..., None] image = np.minimum(aln_face.face, mask) if image.ndim == 3: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) height, width = image.shape c_height, c_width = (int(height / 2.0), int(width / 2.0)) fft = np.fft.fft2(image) fft_shift = np.fft.fftshift(fft) fft_shift[c_height - 75:c_height + 75, c_width - 75:c_width + 75] = 0 ifft_shift = np.fft.ifftshift(fft_shift) shift_back = np.fft.ifft2(ifft_shift) magnitude = np.log(np.abs(shift_back)) score = np.mean(magnitude) return score
def _get_detected_face(alignment): """ Convert an alignment dict item to a detected_face object Parameters ---------- alignment: dict The alignment dict for a face Returns ------- :class:`lib.FacesDetect.detected_face`: The corresponding detected_face object for the alignment """ detected_face = DetectedFace() detected_face.from_alignment(alignment) return detected_face
def extract_one_face(self, alignment, image): """ Extract one face from image """ logger.trace("Extracting one face: (frame: '%s', alignment: %s)", self.current_frame, alignment) face = DetectedFace() face.from_alignment(alignment, image=image) face.load_aligned(image, size=self.size, centering="head") face.thumbnail = generate_thumbnail(face.aligned.face, size=80, quality=60) return face
def estimate_blur(cls, image, metadata=None): """ Estimate the amount of blur an image has with the variance of the Laplacian. Normalize by pixel number to offset the effect of image size on pixel gradients & variance. Parameters ---------- image: :class:`numpy.ndarray` The face image to calculate blur for metadata: dict, optional The metadata for the face image or ``None`` if no metadata is available. If metadata is provided the face will be masked by the "components" mask prior to calculating blur. Default:``None`` Returns ------- float The estimated blur score for the face """ if metadata is not None: alignments = metadata["alignments"] det_face = DetectedFace() det_face.from_png_meta(alignments) aln_face = AlignedFace(np.array(alignments["landmarks_xy"], dtype="float32"), image=image, centering="legacy", size=256, is_aligned=True) mask = det_face.mask["components"] mask.set_sub_crop(aln_face.pose.offset[mask.stored_centering] * -1, centering="legacy") mask = cv2.resize(mask.mask, (256, 256), interpolation=cv2.INTER_CUBIC)[..., None] image = np.minimum(aln_face.face, mask) if image.ndim == 3: image = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) blur_map = cv2.Laplacian(image, cv2.CV_32F) score = np.var(blur_map) / np.sqrt(image.shape[0] * image.shape[1]) return score
def _add_remove_faces(cls, alignments, faces): """ On a revert, ensure that the alignments and detected face object counts for each frame are in sync. """ num_alignments = len(alignments) num_faces = len(faces) if num_alignments == num_faces: retval = False elif num_alignments > num_faces: faces.extend([DetectedFace() for _ in range(num_faces, num_alignments)]) retval = True else: del faces[num_alignments:] retval = True return retval
def load(self): """ Load the faces from the alignments file, convert to :class:`~lib.align.DetectedFace`. objects and add to :attr:`_frame_faces`. """ for key in sorted(self._alignments.data): this_frame_faces = [] for item in self._alignments.data[key]["faces"]: face = DetectedFace() face.from_alignment(item, with_thumb=True) face.load_aligned(None) _ = face.aligned.average_distance # cache the distances this_frame_faces.append(face) self._frame_faces.append(this_frame_faces) self._sorted_frame_names = sorted(self._alignments.data)
def alignment_dict(filename, image): """ Set the image to an ExtractMedia object for alignment """ height, width = image.shape[:2] face = DetectedFace(x=0, w=width, y=0, h=height) return ExtractMedia(filename, image, detected_faces=[face])
def process(self): """ Run the job to remove faces from an alignments file that do not exist within a faces folder. """ logger.info("[REMOVE FACES FROM ALIGNMENTS]") # Tidy up cli output frame_face_indices = self._items.items if not frame_face_indices: logger.error( "No matching faces found in your faces folder. This would remove all " "faces from your alignments file. Process aborted.") return pre_face_count = self._alignments.faces_count self._alignments.filter_faces(frame_face_indices, filter_out=False) del_count = pre_face_count - self._alignments.faces_count if del_count == 0: logger.info("No changes made to alignments file. Exiting") return logger.info("%s alignment(s) were removed from alignments file", del_count) # PNG Header Updates updated_headers = 0 for file_info in tqdm(self._items.file_list_sorted, desc="Updating PNG Headers"): frame = file_info["source_filename"] face_index = file_info["face_index"] new_index = frame_face_indices[frame].index(face_index) if new_index == face_index: # face index has not changed continue fullpath = os.path.join(self._items.folder, file_info["current_filename"]) logger.debug( "Updating png header for '%s': face index from %s to %s", fullpath, face_index, new_index) # Update file_list_sorted for rename task orig_filename = "{}_{}.png".format( os.path.splitext(frame)[0], new_index) file_info["face_index"] = new_index file_info["original_filename"] = orig_filename face = DetectedFace() face.from_alignment( self._alignments.get_faces_in_frame(frame)[new_index]) meta = dict(alignments=face.to_png_meta(), source=dict( alignments_version=file_info["alignments_version"], original_filename=orig_filename, face_index=new_index, source_filename=frame, source_is_video=file_info["source_is_video"])) update_existing_metadata(fullpath, meta) updated_headers += 1 logger.info( "%s Extracted face(s) had their header information updated", updated_headers) self._alignments.save() rename = Rename(self._alignments, None, self._items) rename.process()
def to_detected_face(left, top, right, bottom): """ Return a :class:`~lib.align.DetectedFace` object for the bounding box """ return DetectedFace(x=int(round(left)), w=int(round(right - left)), y=int(round(top)), h=int(round(bottom - top)))
def __init__(self, arguments): logger.debug("Initializing %s: (args: %s)", self.__class__.__name__, arguments) self._args = arguments # load faces faces_alignments = AlignmentsBase(self._args.faces_align_dir) print() print(f'Faces alignments: {len(faces_alignments._data.keys())}') print(faces_alignments._data.keys()) self._faces = {} faces_loader = ImagesLoader(self._args.faces_dir) for filename, image in faces_loader.load(): face_name = os.path.basename(filename) faces = faces_alignments.get_faces_in_frame(face_name) detected_faces = list() for rawface in faces: face = DetectedFace() face.from_alignment(rawface, image=image) feed_face = AlignedFace(face.landmarks_xy, image=image, centering='face', size=image.shape[0], coverage_ratio=1.0, dtype="float32") detected_faces.append(feed_face) self._faces[face_name] = (filename, image, detected_faces) print('Faces:', len(self._faces)) print(self._faces.keys()) print() self._patch_threads = None self._images = ImagesLoader(self._args.input_dir, fast_count=True) self._alignments = Alignments(self._args, False, self._images.is_video) if self._alignments.version == 1.0: logger.error("The alignments file format has been updated since the given alignments " "file was generated. You need to update the file to proceed.") logger.error("To do this run the 'Alignments Tool' > 'Extract' Job.") sys.exit(1) self._opts = OptionalActions(self._args, self._images.file_list, self._alignments) self._add_queues() self._disk_io = DiskIO(self._alignments, self._images, arguments) self._predictor = Predict(self._disk_io.load_queue, self._queue_size, self._faces, arguments) self._validate() get_folder(self._args.output_dir) configfile = self._args.configfile if hasattr(self._args, "configfile") else None self._converter = Converter(self._predictor.output_size, self._predictor.coverage_ratio, self._predictor.centering, self._disk_io.draw_transparent, self._disk_io.pre_encode, arguments, configfile=configfile) logger.debug("Initialized %s", self.__class__.__name__)