def process_data(self, data):
filepath = Path(data[0])
try:
dflimg = DFLIMG.load(filepath)
if dflimg is None or not dflimg.has_data():
self.log_err(f"{filepath.name} is not a dfl image file")
return [1, [str(filepath)]]
bgr = cv2_imread(str(filepath))
if bgr is None:
raise Exception("Unable to load %s" % (filepath.name))
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
if self.faster:
source_rect = dflimg.get_source_rect()
sharpness = mathlib.polygon_area(
np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32),
np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32))
else:
face_mask = LandmarksProcessor.get_image_hull_mask(
gray.shape, dflimg.get_landmarks())
sharpness = estimate_sharpness(
(gray[..., None] * face_mask).astype(np.uint8))
pitch, yaw, roll = LandmarksProcessor.estimate_pitch_yaw_roll(
dflimg.get_landmarks(), size=dflimg.get_shape()[1])
hist = cv2.calcHist([gray], [0], None, [256], [0, 256])
except Exception as e:
self.log_err(e)
return [1, [str(filepath)]]
return [0, [str(filepath), sharpness, hist, yaw, pitch]]
def get_transform_mat_by_data (l_c, tb_diag_vec, bt_diag_vec, mod, output_size, face_type):
_, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ) ] )
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ),
np.round( l_c - bt_diag_vec*mod ),
] )
# get area of face square in global space
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array( [ np.round( l_c + [-side,-side] ),
np.round( l_c + [ side,-side] ),
np.round( l_c + [ side, side] ) ] )
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
mat = cv2.getAffineTransform(l_t,pts2)
return mat
def sort_by_face_source_rect_size(input_path):
io.log_info("Sorting by face rect size...")
img_list = []
trash_img_list = []
for filepath in io.progress_bar_generator(
pathex.get_image_paths(input_path), "Loading"):
filepath = Path(filepath)
dflimg = DFLIMG.load(filepath)
if dflimg is None or not dflimg.has_data():
io.log_err(f"{filepath.name} is not a dfl image file")
trash_img_list.append([str(filepath)])
continue
source_rect = dflimg.get_source_rect()
rect_area = mathlib.polygon_area(
np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32),
np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32))
img_list.append([str(filepath), rect_area])
io.log_info("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(1), reverse=True)
return img_list, trash_img_list
def get_transform_mat(image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array(image_landmarks)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama(
np.concatenate([image_landmarks[17:49], image_landmarks[54:55]]),
landmarks_2D_new, True)[0:2]
# get corner points in global space
l_p = transform_points(
np.float32([(0, 0), (1, 0), (1, 1), (0, 1), (0.5, 0.5)]), mat, True)
l_c = l_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (l_p[2] - l_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (l_p[1] - l_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
padding, remove_align = FaceType_to_padding_remove_align.get(
face_type, 0.0)
mod = (1.0 / scale) * (npla.norm(l_p[0] - l_p[2]) *
(padding * np.sqrt(2.0) + 0.5))
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array([
np.round(l_c - tb_diag_vec * mod),
np.round(l_c + bt_diag_vec * mod),
np.round(l_c + tb_diag_vec * mod)
])
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array([
np.round(l_c - tb_diag_vec * mod),
np.round(l_c + bt_diag_vec * mod),
np.round(l_c + tb_diag_vec * mod),
np.round(l_c - bt_diag_vec * mod),
])
# get area of face square in global space
area = mathlib.polygon_area(l_t[:, 0], l_t[:, 1])
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array([
np.round(l_c + [-side, -side]),
np.round(l_c + [side, -side]),
np.round(l_c + [side, side])
])
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(((0, 0), (output_size, 0), (output_size, output_size)))
mat = cv2.getAffineTransform(l_t, pts2)
return mat
def dfl_img_area(dfl_img):
source_rect = dfl_img.get_source_rect()
from core import mathlib
import numpy as np
rect_area = mathlib.polygon_area(
np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32),
np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32))
return rect_area
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0, full_face_align_top=True):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
l_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5)]) , mat, True)
l_c = l_p[4]
tb_diag_vec = (l_p[2]-l_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (l_p[1]-l_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
mod = (1.0 / scale)* ( npla.norm(l_p[0]-l_p[2])*(padding*np.sqrt(2.0) + 0.5) )
if not remove_align:
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ) ] )
else:
l_t = np.array( [ np.round( l_c - tb_diag_vec*mod ),
np.round( l_c + bt_diag_vec*mod ),
np.round( l_c + tb_diag_vec*mod ),
np.round( l_c - bt_diag_vec*mod ),
] )
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
side = np.float32(math.sqrt(area) / 2)
l_t = np.array( [ np.round( l_c + [-side,-side] ),
np.round( l_c + [ side,-side] ),
np.round( l_c + [ side, side] ) ] )
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
mat = cv2.getAffineTransform(l_t,pts2)
#if remove_align:
# bbox = transform_points ( [ (0,0), (0,output_size), (output_size, output_size), (output_size,0) ], mat, True)
# #import code
# #code.interact(local=dict(globals(), **locals()))
# area = mathlib.polygon_area(bbox[:,0], bbox[:,1] )
# side = math.sqrt(area) / 2
# center = transform_points ( [(output_size/2,output_size/2)], mat, True)
# pts1 = np.float32(( center+[-side,-side], center+[side,-side], center+[side,-side] ))
# pts2 = np.float32([[0,0],[output_size,0],[0,output_size]])
# mat = cv2.getAffineTransform(pts1,pts2)
return mat
def final_stage(
data,
image,
face_type,
image_size,
extract_from_dflimg=False,
output_debug_path=None,
final_output_path=None,
):
data.final_output_files = []
filepath = data.filepath
rects = data.rects
landmarks = data.landmarks
if output_debug_path is not None:
debug_image = image.copy()
if extract_from_dflimg and len(rects) != 1:
#if re-extracting from dflimg and more than 1 or zero faces detected - dont process and just copy it
print("extract_from_dflimg and len(rects) != 1", filepath)
output_filepath = final_output_path / filepath.name
if filepath != str(output_file):
shutil.copy(str(filepath), str(output_filepath))
data.final_output_files.append(output_filepath)
else:
face_idx = 0
for rect, image_landmarks in zip(rects, landmarks):
if extract_from_dflimg and face_idx > 1:
#cannot extract more than 1 face from dflimg
break
if image_landmarks is None:
continue
rect = np.array(rect)
if face_type == FaceType.MARK_ONLY:
image_to_face_mat = None
face_image = image
face_image_landmarks = image_landmarks
else:
image_to_face_mat = LandmarksProcessor.get_transform_mat(
image_landmarks, image_size, face_type)
face_image = cv2.warpAffine(image, image_to_face_mat,
(image_size, image_size),
cv2.INTER_LANCZOS4)
face_image_landmarks = LandmarksProcessor.transform_points(
image_landmarks, image_to_face_mat)
landmarks_bbox = LandmarksProcessor.transform_points(
[(0, 0), (0, image_size - 1),
(image_size - 1, image_size - 1),
(image_size - 1, 0)], image_to_face_mat, True)
rect_area = mathlib.polygon_area(
np.array(rect[[0, 2, 2, 0]]),
np.array(rect[[1, 1, 3, 3]]))
landmarks_area = mathlib.polygon_area(
landmarks_bbox[:, 0], landmarks_bbox[:, 1])
if not data.manual and face_type <= FaceType.FULL_NO_ALIGN and landmarks_area > 4 * rect_area: #get rid of faces which umeyama-landmark-area > 4*detector-rect-area
continue
if output_debug_path is not None:
LandmarksProcessor.draw_rect_landmarks(
debug_image,
rect,
image_landmarks,
image_size,
face_type,
transparent_mask=True)
output_path = final_output_path
if data.force_output_path is not None:
output_path = data.force_output_path
if extract_from_dflimg and filepath.suffix == '.jpg':
#if extracting from dflimg and jpg copy it in order not to lose quality
output_filepath = output_path / filepath.name
if filepath != output_filepath:
shutil.copy(str(filepath), str(output_filepath))
else:
output_filepath = output_path / f"{filepath.stem}_{face_idx}.jpg"
cv2_imwrite(output_filepath, face_image,
[int(cv2.IMWRITE_JPEG_QUALITY), 100])
DFLJPG.embed_data(
output_filepath,
face_type=FaceType.toString(face_type),
landmarks=face_image_landmarks.tolist(),
source_filename=filepath.name,
source_rect=rect,
source_landmarks=image_landmarks.tolist(),
image_to_face_mat=image_to_face_mat)
data.final_output_files.append(output_filepath)
face_idx += 1
data.faces_detected = face_idx
if output_debug_path is not None:
cv2_imwrite(output_debug_path / (filepath.stem + '.jpg'),
debug_image, [int(cv2.IMWRITE_JPEG_QUALITY), 50])
return data
def get_transform_mat (image_landmarks, output_size, face_type, scale=1.0):
if not isinstance(image_landmarks, np.ndarray):
image_landmarks = np.array (image_landmarks)
# estimate landmarks transform from global space to local aligned space with bounds [0..1]
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
# get corner points in global space
g_p = transform_points ( np.float32([(0,0),(1,0),(1,1),(0,1),(0.5,0.5) ]) , mat, True)
g_c = g_p[4]
# calc diagonal vectors between corners in global space
tb_diag_vec = (g_p[2]-g_p[0]).astype(np.float32)
tb_diag_vec /= npla.norm(tb_diag_vec)
bt_diag_vec = (g_p[1]-g_p[3]).astype(np.float32)
bt_diag_vec /= npla.norm(bt_diag_vec)
# calc modifier of diagonal vectors for scale and padding value
padding, remove_align = FaceType_to_padding_remove_align.get(face_type, 0.0)
mod = (1.0 / scale)* ( npla.norm(g_p[0]-g_p[2])*(padding*np.sqrt(2.0) + 0.5) )
if face_type == FaceType.WHOLE_FACE:
# adjust vertical offset for WHOLE_FACE, 7% below in order to cover more forehead
vec = (g_p[0]-g_p[3]).astype(np.float32)
vec_len = npla.norm(vec)
vec /= vec_len
g_c += vec*vec_len*0.07
elif face_type == FaceType.HEAD:
mat = umeyama( np.concatenate ( [ image_landmarks[17:49] , image_landmarks[54:55] ] ) , landmarks_2D_new, True)[0:2]
# assuming image_landmarks are 3D_Landmarks extracted for HEAD,
# adjust horizontal offset according to estimated yaw
yaw = estimate_averaged_yaw(transform_points (image_landmarks, mat, False))
hvec = (g_p[0]-g_p[1]).astype(np.float32)
hvec_len = npla.norm(hvec)
hvec /= hvec_len
yaw *= np.abs(math.tanh(yaw*2)) # Damp near zero
g_c -= hvec * (yaw * hvec_len / 2.0)
# adjust vertical offset for HEAD, 50% below
vvec = (g_p[0]-g_p[3]).astype(np.float32)
vvec_len = npla.norm(vvec)
vvec /= vvec_len
g_c += vvec*vvec_len*0.50
# calc 3 points in global space to estimate 2d affine transform
if not remove_align:
l_t = np.array( [ g_c - tb_diag_vec*mod,
g_c + bt_diag_vec*mod,
g_c + tb_diag_vec*mod ] )
else:
# remove_align - face will be centered in the frame but not aligned
l_t = np.array( [ g_c - tb_diag_vec*mod,
g_c + bt_diag_vec*mod,
g_c + tb_diag_vec*mod,
g_c - bt_diag_vec*mod,
] )
# get area of face square in global space
area = mathlib.polygon_area(l_t[:,0], l_t[:,1] )
# calc side of square
side = np.float32(math.sqrt(area) / 2)
# calc 3 points with unrotated square
l_t = np.array( [ g_c + [-side,-side],
g_c + [ side,-side],
g_c + [ side, side] ] )
# calc affine transform from 3 global space points to 3 local space points size of 'output_size'
pts2 = np.float32(( (0,0),(output_size,0),(output_size,output_size) ))
mat = cv2.getAffineTransform(l_t,pts2)
return mat
def final_stage(
data,
image,
face_type,
image_size,
jpeg_quality,
output_debug_path=None,
final_output_path=None,
):
data.final_output_files = []
filepath = data.filepath
rects = data.rects
landmarks = data.landmarks
if output_debug_path is not None:
debug_image = image.copy()
face_idx = 0
for rect, image_landmarks in zip(rects, landmarks):
if image_landmarks is None:
continue
rect = np.array(rect)
if face_type == FaceType.MARK_ONLY:
image_to_face_mat = None
face_image = image
face_image_landmarks = image_landmarks
else:
image_to_face_mat = LandmarksProcessor.get_transform_mat(
image_landmarks, image_size, face_type)
face_image = cv2.warpAffine(image, image_to_face_mat,
(image_size, image_size),
cv2.INTER_LANCZOS4)
face_image_landmarks = LandmarksProcessor.transform_points(
image_landmarks, image_to_face_mat)
landmarks_bbox = LandmarksProcessor.transform_points(
[(0, 0), (0, image_size - 1),
(image_size - 1, image_size - 1),
(image_size - 1, 0)], image_to_face_mat, True)
rect_area = mathlib.polygon_area(
np.array(rect[[0, 2, 2, 0]]).astype(np.float32),
np.array(rect[[1, 1, 3, 3]]).astype(np.float32))
landmarks_area = mathlib.polygon_area(
landmarks_bbox[:, 0].astype(np.float32),
landmarks_bbox[:, 1].astype(np.float32))
if not data.manual and face_type <= FaceType.FULL_NO_ALIGN and landmarks_area > 4 * rect_area: #get rid of faces which umeyama-landmark-area > 4*detector-rect-area
continue
if output_debug_path is not None:
LandmarksProcessor.draw_rect_landmarks(
debug_image,
rect,
image_landmarks,
face_type,
image_size,
transparent_mask=True)
output_path = final_output_path
if data.force_output_path is not None:
output_path = data.force_output_path
output_filepath = output_path / f"{filepath.stem}_{face_idx}.jpg"
cv2_imwrite(output_filepath, face_image,
[int(cv2.IMWRITE_JPEG_QUALITY), jpeg_quality])
dflimg = DFLJPG.load(output_filepath)
dflimg.set_face_type(FaceType.toString(face_type))
dflimg.set_landmarks(face_image_landmarks.tolist())
dflimg.set_source_filename(filepath.name)
dflimg.set_source_rect(rect)
dflimg.set_source_landmarks(image_landmarks.tolist())
dflimg.set_image_to_face_mat(image_to_face_mat)
dflimg.save()
data.final_output_files.append(output_filepath)
face_idx += 1
data.faces_detected = face_idx
if output_debug_path is not None:
cv2_imwrite(output_debug_path / (filepath.stem + '.jpg'),
debug_image, [int(cv2.IMWRITE_JPEG_QUALITY), 50])
return data