def process_data(self, data): filepath = Path(data[0]) try: dflimg = DFLIMG.load(filepath) if dflimg is None or not dflimg.has_data(): self.log_err(f"{filepath.name} is not a dfl image file") return [1, [str(filepath)]] bgr = cv2_imread(str(filepath)) if bgr is None: raise Exception("Unable to load %s" % (filepath.name)) gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) if self.faster: source_rect = dflimg.get_source_rect() sharpness = mathlib.polygon_area( np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32), np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32)) else: face_mask = LandmarksProcessor.get_image_hull_mask( gray.shape, dflimg.get_landmarks()) sharpness = estimate_sharpness( (gray[..., None] * face_mask).astype(np.uint8)) pitch, yaw, roll = LandmarksProcessor.estimate_pitch_yaw_roll( dflimg.get_landmarks(), size=dflimg.get_shape()[1]) hist = cv2.calcHist([gray], [0], None, [256], [0, 256]) except Exception as e: self.log_err(e) return [1, [str(filepath)]] return [0, [str(filepath), sharpness, hist, yaw, pitch]]
def get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type): _, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ) ] ) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ), np.round( l_c - bt_diag_vec*mod ), ] ) # get area of face square in global space area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array( [ np.round( l_c + [-side,-side] ), np.round( l_c + [ side,-side] ), np.round( l_c + [ side, side] ) ] ) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) mat = cv2.getAffineTransform(l_t,pts2) return mat
def sort_by_face_source_rect_size(input_path): io.log_info("Sorting by face rect size...") img_list = [] trash_img_list = [] for filepath in io.progress_bar_generator( pathex.get_image_paths(input_path), "Loading"): filepath = Path(filepath) dflimg = DFLIMG.load(filepath) if dflimg is None or not dflimg.has_data(): io.log_err(f"{filepath.name} is not a dfl image file") trash_img_list.append([str(filepath)]) continue source_rect = dflimg.get_source_rect() rect_area = mathlib.polygon_area( np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32), np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32)) img_list.append([str(filepath), rect_area]) io.log_info("Sorting...") img_list = sorted(img_list, key=operator.itemgetter(1), reverse=True) return img_list, trash_img_list
def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array(image_landmarks) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama( np.concatenate([image_landmarks[17:49], image_landmarks[54:55]]), landmarks_2D_new, True)[0:2] # get corner points in global space l_p = transform_points( np.float32([(0, 0), (1, 0), (1, 1), (0, 1), (0.5, 0.5)]), mat, True) l_c = l_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (l_p[2] - l_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (l_p[1] - l_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value padding, remove_align = FaceType_to_padding_remove_align.get( face_type, 0.0) mod = (1.0 / scale) * (npla.norm(l_p[0] - l_p[2]) * (padding * np.sqrt(2.0) + 0.5)) # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array([ np.round(l_c - tb_diag_vec * mod), np.round(l_c + bt_diag_vec * mod), np.round(l_c + tb_diag_vec * mod) ]) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array([ np.round(l_c - tb_diag_vec * mod), np.round(l_c + bt_diag_vec * mod), np.round(l_c + tb_diag_vec * mod), np.round(l_c - bt_diag_vec * mod), ]) # get area of face square in global space area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1]) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array([ np.round(l_c + [-side, -side]), np.round(l_c + [side, -side]), np.round(l_c + [side, side]) ]) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size))) mat = cv2.getAffineTransform(l_t, pts2) return mat
def dfl_img_area(dfl_img): source_rect = dfl_img.get_source_rect() from core import mathlib import numpy as np rect_area = mathlib.polygon_area( np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32), np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32)) return rect_area
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0, full_face_align_top=True): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array (image_landmarks) padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True) l_c = l_p[4] tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) ) if not remove_align: l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ) ] ) else: l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ), np.round( l_c + bt_diag_vec*mod ), np.round( l_c + tb_diag_vec*mod ), np.round( l_c - bt_diag_vec*mod ), ] ) area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) side = np.float32(math.sqrt(area) / 2) l_t = np.array( [ np.round( l_c + [-side,-side] ), np.round( l_c + [ side,-side] ), np.round( l_c + [ side, side] ) ] ) pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) mat = cv2.getAffineTransform(l_t,pts2) #if remove_align: # bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True) # #import code # #code.interact(local=dict(globals(), **locals())) # area = mathlib.polygon_area(bbox[:,0], bbox[:,1] ) # side = math.sqrt(area) / 2 # center = transform_points ( [(output_size/2,output_size/2)], mat, True) # pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] )) # pts2 = np.float32([[0,0],[output_size,0],[0,output_size]]) # mat = cv2.getAffineTransform(pts1,pts2) return mat
def final_stage( data, image, face_type, image_size, extract_from_dflimg=False, output_debug_path=None, final_output_path=None, ): data.final_output_files = [] filepath = data.filepath rects = data.rects landmarks = data.landmarks if output_debug_path is not None: debug_image = image.copy() if extract_from_dflimg and len(rects) != 1: #if re-extracting from dflimg and more than 1 or zero faces detected - dont process and just copy it print("extract_from_dflimg and len(rects) != 1", filepath) output_filepath = final_output_path / filepath.name if filepath != str(output_file): shutil.copy(str(filepath), str(output_filepath)) data.final_output_files.append(output_filepath) else: face_idx = 0 for rect, image_landmarks in zip(rects, landmarks): if extract_from_dflimg and face_idx > 1: #cannot extract more than 1 face from dflimg break if image_landmarks is None: continue rect = np.array(rect) if face_type == FaceType.MARK_ONLY: image_to_face_mat = None face_image = image face_image_landmarks = image_landmarks else: image_to_face_mat = LandmarksProcessor.get_transform_mat( image_landmarks, image_size, face_type) face_image = cv2.warpAffine(image, image_to_face_mat, (image_size, image_size), cv2.INTER_LANCZOS4) face_image_landmarks = LandmarksProcessor.transform_points( image_landmarks, image_to_face_mat) landmarks_bbox = LandmarksProcessor.transform_points( [(0, 0), (0, image_size - 1), (image_size - 1, image_size - 1), (image_size - 1, 0)], image_to_face_mat, True) rect_area = mathlib.polygon_area( np.array(rect[[0, 2, 2, 0]]), np.array(rect[[1, 1, 3, 3]])) landmarks_area = mathlib.polygon_area( landmarks_bbox[:, 0], landmarks_bbox[:, 1]) if not data.manual and face_type <= FaceType.FULL_NO_ALIGN and landmarks_area > 4 * rect_area: #get rid of faces which umeyama-landmark-area > 4*detector-rect-area continue if output_debug_path is not None: LandmarksProcessor.draw_rect_landmarks( debug_image, rect, image_landmarks, image_size, face_type, transparent_mask=True) output_path = final_output_path if data.force_output_path is not None: output_path = data.force_output_path if extract_from_dflimg and filepath.suffix == '.jpg': #if extracting from dflimg and jpg copy it in order not to lose quality output_filepath = output_path / filepath.name if filepath != output_filepath: shutil.copy(str(filepath), str(output_filepath)) else: output_filepath = output_path / f"{filepath.stem}_{face_idx}.jpg" cv2_imwrite(output_filepath, face_image, [int(cv2.IMWRITE_JPEG_QUALITY), 100]) DFLJPG.embed_data( output_filepath, face_type=FaceType.toString(face_type), landmarks=face_image_landmarks.tolist(), source_filename=filepath.name, source_rect=rect, source_landmarks=image_landmarks.tolist(), image_to_face_mat=image_to_face_mat) data.final_output_files.append(output_filepath) face_idx += 1 data.faces_detected = face_idx if output_debug_path is not None: cv2_imwrite(output_debug_path / (filepath.stem + '.jpg'), debug_image, [int(cv2.IMWRITE_JPEG_QUALITY), 50]) return data
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0): if not isinstance(image_landmarks, np.ndarray): image_landmarks = np.array (image_landmarks) # estimate landmarks transform from global space to local aligned space with bounds [0..1] mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] # get corner points in global space g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True) g_c = g_p[4] # calc diagonal vectors between corners in global space tb_diag_vec = (g_p[2]-g_p[0]).astype(np.float32) tb_diag_vec /= npla.norm(tb_diag_vec) bt_diag_vec = (g_p[1]-g_p[3]).astype(np.float32) bt_diag_vec /= npla.norm(bt_diag_vec) # calc modifier of diagonal vectors for scale and padding value padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0) mod = (1.0 / scale)* ( npla.norm(g_p[0]-g_p[2])*(padding*np.sqrt(2.0) + 0.5) ) if face_type == FaceType.WHOLE_FACE: # adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead vec = (g_p[0]-g_p[3]).astype(np.float32) vec_len = npla.norm(vec) vec /= vec_len g_c += vec*vec_len*0.07 elif face_type == FaceType.HEAD: mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2] # assuming image_landmarks are 3D_Landmarks extracted for HEAD, # adjust horizontal offset according to estimated yaw yaw = estimate_averaged_yaw(transform_points (image_landmarks, mat, False)) hvec = (g_p[0]-g_p[1]).astype(np.float32) hvec_len = npla.norm(hvec) hvec /= hvec_len yaw *= np.abs(math.tanh(yaw*2)) # Damp near zero g_c -= hvec * (yaw * hvec_len / 2.0) # adjust vertical offset for HEAD, 50% below vvec = (g_p[0]-g_p[3]).astype(np.float32) vvec_len = npla.norm(vvec) vvec /= vvec_len g_c += vvec*vvec_len*0.50 # calc 3 points in global space to estimate 2d affine transform if not remove_align: l_t = np.array( [ g_c - tb_diag_vec*mod, g_c + bt_diag_vec*mod, g_c + tb_diag_vec*mod ] ) else: # remove_align - face will be centered in the frame but not aligned l_t = np.array( [ g_c - tb_diag_vec*mod, g_c + bt_diag_vec*mod, g_c + tb_diag_vec*mod, g_c - bt_diag_vec*mod, ] ) # get area of face square in global space area = mathlib.polygon_area(l_t[:,0], l_t[:,1] ) # calc side of square side = np.float32(math.sqrt(area) / 2) # calc 3 points with unrotated square l_t = np.array( [ g_c + [-side,-side], g_c + [ side,-side], g_c + [ side, side] ] ) # calc affine transform from 3 global space points to 3 local space points size of 'output_size' pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) )) mat = cv2.getAffineTransform(l_t,pts2) return mat
def final_stage( data, image, face_type, image_size, jpeg_quality, output_debug_path=None, final_output_path=None, ): data.final_output_files = [] filepath = data.filepath rects = data.rects landmarks = data.landmarks if output_debug_path is not None: debug_image = image.copy() face_idx = 0 for rect, image_landmarks in zip(rects, landmarks): if image_landmarks is None: continue rect = np.array(rect) if face_type == FaceType.MARK_ONLY: image_to_face_mat = None face_image = image face_image_landmarks = image_landmarks else: image_to_face_mat = LandmarksProcessor.get_transform_mat( image_landmarks, image_size, face_type) face_image = cv2.warpAffine(image, image_to_face_mat, (image_size, image_size), cv2.INTER_LANCZOS4) face_image_landmarks = LandmarksProcessor.transform_points( image_landmarks, image_to_face_mat) landmarks_bbox = LandmarksProcessor.transform_points( [(0, 0), (0, image_size - 1), (image_size - 1, image_size - 1), (image_size - 1, 0)], image_to_face_mat, True) rect_area = mathlib.polygon_area( np.array(rect[[0, 2, 2, 0]]).astype(np.float32), np.array(rect[[1, 1, 3, 3]]).astype(np.float32)) landmarks_area = mathlib.polygon_area( landmarks_bbox[:, 0].astype(np.float32), landmarks_bbox[:, 1].astype(np.float32)) if not data.manual and face_type <= FaceType.FULL_NO_ALIGN and landmarks_area > 4 * rect_area: #get rid of faces which umeyama-landmark-area > 4*detector-rect-area continue if output_debug_path is not None: LandmarksProcessor.draw_rect_landmarks( debug_image, rect, image_landmarks, face_type, image_size, transparent_mask=True) output_path = final_output_path if data.force_output_path is not None: output_path = data.force_output_path output_filepath = output_path / f"{filepath.stem}_{face_idx}.jpg" cv2_imwrite(output_filepath, face_image, [int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality]) dflimg = DFLJPG.load(output_filepath) dflimg.set_face_type(FaceType.toString(face_type)) dflimg.set_landmarks(face_image_landmarks.tolist()) dflimg.set_source_filename(filepath.name) dflimg.set_source_rect(rect) dflimg.set_source_landmarks(image_landmarks.tolist()) dflimg.set_image_to_face_mat(image_to_face_mat) dflimg.save() data.final_output_files.append(output_filepath) face_idx += 1 data.faces_detected = face_idx if output_debug_path is not None: cv2_imwrite(output_debug_path / (filepath.stem + '.jpg'), debug_image, [int(cv2.IMWRITE_JPEG_QUALITY), 50]) return data