def get_full_face_mask():
if sample.eyebrows_expand_mod is not None:
full_face_mask = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape,
sample_landmarks,
eyebrows_expand_mod=sample.eyebrows_expand_mod)
else:
full_face_mask = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape, sample_landmarks)
return np.clip(full_face_mask, 0, 1)
def onClientProcessData(self, data):
filepath = Path(data[0])
try:
if filepath.suffix != '.png':
raise Exception(
"%s is not a png file required for sort_final" %
(filepath.name))
dflpng = DFLPNG.load(str(filepath), print_on_no_embedded_data=True)
if dflpng is None:
raise Exception("")
bgr = cv2.imread(str(filepath))
if bgr is None:
raise Exception("Unable to load %s" % (filepath.name))
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
gray_masked = (gray * LandmarksProcessor.get_image_hull_mask(
bgr.shape, dflpng.get_landmarks())[:, :, 0]).astype(np.uint8)
sharpness = estimate_sharpness(gray_masked)
hist = cv2.calcHist([gray], [0], None, [256], [0, 256])
except Exception as e:
print(e)
return [1, [str(filepath)]]
return [0, [str(filepath), sharpness, hist, dflpng.get_yaw_value()]]
def sort_by_hist_dissim(input_path):
print("Sorting by histogram dissimilarity...")
img_list = []
for filename_path in tqdm(Path_utils.get_image_paths(input_path),
desc="Loading"):
image = cv2.imread(filename_path)
dflpng = DFLPNG.load(str(filename_path))
if dflpng is not None:
face_mask = LandmarksProcessor.get_image_hull_mask(
image, dflpng.get_landmarks())
image = (image * face_mask).astype(np.uint8)
img_list.append([
filename_path,
cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None,
[256], [0, 256]), 0
])
img_list = HistDissimSubprocessor(img_list).process()
print("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True)
return img_list
def process_data(self, data):
filepath = Path(data[0])
try:
if filepath.suffix == '.png':
dflimg = DFLPNG.load( str(filepath) )
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load( str(filepath) )
else:
dflimg = None
if dflimg is None:
self.log_err("%s is not a dfl image file" % (filepath.name))
return [ 1, [str(filepath)] ]
bgr = cv2_imread(str(filepath))
if bgr is None:
raise Exception ("Unable to load %s" % (filepath.name) )
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
gray_masked = ( gray * LandmarksProcessor.get_image_hull_mask (bgr.shape, dflimg.get_landmarks() )[:,:,0] ).astype(np.uint8)
sharpness = estimate_sharpness(gray_masked)
pitch, yaw = LandmarksProcessor.estimate_pitch_yaw ( dflimg.get_landmarks() )
hist = cv2.calcHist([gray], [0], None, [256], [0, 256])
except Exception as e:
self.log_err (e)
return [ 1, [str(filepath)] ]
return [ 0, [str(filepath), sharpness, hist, yaw ] ]
def sort_by_hist_dissim(input_path):
io.log_info("Sorting by histogram dissimilarity...")
img_list = []
trash_img_list = []
for filepath in io.progress_bar_generator(
pathex.get_image_paths(input_path), "Loading"):
filepath = Path(filepath)
dflimg = DFLIMG.load(filepath)
image = cv2_imread(str(filepath))
if dflimg is not None:
face_mask = LandmarksProcessor.get_image_hull_mask(
image.shape, dflimg.get_landmarks())
image = (image * face_mask).astype(np.uint8)
img_list.append([
str(filepath),
cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None,
[256], [0, 256]), 0
])
img_list = HistDissimSubprocessor(img_list).run()
io.log_info("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True)
return img_list, trash_img_list
def sort_by_hist_dissim(input_path):
io.log_info ("Sorting by histogram dissimilarity...")
img_list = []
trash_img_list = []
for filepath in io.progress_bar_generator( Path_utils.get_image_paths(input_path), "Loading"):
filepath = Path(filepath)
if filepath.suffix == '.png':
dflimg = DFLPNG.load( str(filepath) )
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load ( str(filepath) )
else:
dflimg = None
if dflimg is None:
io.log_err ("%s is not a dfl image file" % (filepath.name) )
trash_img_list.append ([str(filepath)])
continue
image = cv2_imread(str(filepath))
face_mask = LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks())
image = (image*face_mask).astype(np.uint8)
img_list.append ([str(filepath), cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None, [256], [0, 256]), 0 ])
img_list = HistDissimSubprocessor(img_list).run()
io.log_info ("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True)
return img_list, trash_img_list
def onProcessSample(self, sample, debug):
image = sample.load_image()
if debug:
LandmarksProcessor.draw_landmarks(image, sample.landmarks,
(0, 1, 0))
hull_mask = LandmarksProcessor.get_image_hull_mask(
image, sample.landmarks)
s_image = np.concatenate((image, hull_mask), axis=2)
warped_img, target_img = HalfFaceTrainingDataGenerator.warp(
s_image, sample.landmarks, self.warped_size, self.target_size)
if self.random_flip and np.random.randint(2) == 1:
warped_img = warped_img[:, ::-1, :]
target_img = target_img[:, ::-1, :]
if debug:
return (self.target_size[1], image, target_img,
warped_img[..., 0:3], warped_img[..., 0:3] *
np.expand_dims(warped_img[..., 3], 2))
else:
return (warped_img, target_img)
def sort_by_hist_dissim(input_path):
print("Sorting by histogram dissimilarity...")
img_list = []
for filepath in tqdm(Path_utils.get_image_paths(input_path),
desc="Loading",
ascii=True):
filepath = Path(filepath)
if filepath.suffix == '.png':
dflimg = DFLPNG.load(str(filepath), print_on_no_embedded_data=True)
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load(str(filepath), print_on_no_embedded_data=True)
else:
print("%s is not a dfl image file" % (filepath.name))
continue
image = cv2.imread(str(filepath))
face_mask = LandmarksProcessor.get_image_hull_mask(
image.shape, dflimg.get_landmarks())
image = (image * face_mask).astype(np.uint8)
img_list.append([
str(filepath),
cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None,
[256], [0, 256]), 0
])
img_list = HistDissimSubprocessor(img_list).process()
print("Sorting...")
img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True)
return img_list
def cli_convert_face (self, img_bgr, img_face_landmarks, debug, avaperator_face_bgr=None, **kwargs):
if debug:
debugs = [img_bgr.copy()]
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask (img_bgr.shape, img_face_landmarks)
img_face_mask_aaa = np.repeat(img_face_mask_a, 3, -1)
output_size = self.predictor_input_size
face_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, output_size, face_type=FaceType.FULL)
dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC )
predictor_input_dst_face_mask_a_0 = cv2.resize (dst_face_mask_a_0, (self.predictor_input_size,self.predictor_input_size), cv2.INTER_CUBIC )
prd_inp_dst_face_mask_a = predictor_input_dst_face_mask_a_0[...,np.newaxis]
prd_inp_avaperator_face_bgr = cv2.resize (avaperator_face_bgr, (self.predictor_input_size,self.predictor_input_size), cv2.INTER_CUBIC )
prd_face_bgr = self.predictor_func ( prd_inp_avaperator_face_bgr, prd_inp_dst_face_mask_a )
out_img = img_bgr.copy()
out_img = cv2.warpAffine( prd_face_bgr, face_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT )
out_img = np.clip(out_img, 0.0, 1.0)
if debug:
debugs += [out_img.copy()]
out_img = np.clip( img_bgr*(1-img_face_mask_aaa) + (out_img*img_face_mask_aaa) , 0, 1.0 )
if debug:
debugs += [out_img.copy()]
return debugs if debug else out_img
def process_data(self, data):
filepath = Path(data[0])
try:
dflimg = DFLIMG.load(filepath)
if dflimg is None or not dflimg.has_data():
self.log_err(f"{filepath.name} is not a dfl image file")
return [1, [str(filepath)]]
bgr = cv2_imread(str(filepath))
if bgr is None:
raise Exception("Unable to load %s" % (filepath.name))
gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY)
if self.faster:
source_rect = dflimg.get_source_rect()
sharpness = mathlib.polygon_area(
np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32),
np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32))
else:
face_mask = LandmarksProcessor.get_image_hull_mask(
gray.shape, dflimg.get_landmarks())
sharpness = estimate_sharpness(
(gray[..., None] * face_mask).astype(np.uint8))
pitch, yaw, roll = LandmarksProcessor.estimate_pitch_yaw_roll(
dflimg.get_landmarks(), size=dflimg.get_shape()[1])
hist = cv2.calcHist([gray], [0], None, [256], [0, 256])
except Exception as e:
self.log_err(e)
return [1, [str(filepath)]]
return [0, [str(filepath), sharpness, hist, yaw, pitch]]
def apply_celebamaskhq(input_dir ):
input_path = Path(input_dir)
img_path = input_path / 'aligned'
mask_path = input_path / 'mask'
if not img_path.exists():
raise ValueError(f'{str(img_path)} directory not found. Please ensure it exists.')
CelebAMASKHQSubprocessor(pathex.get_image_paths(img_path),
pathex.get_image_paths(mask_path, subdirs=True) ).run()
return
paths_to_extract = []
for filename in io.progress_bar_generator(pathex.get_image_paths(img_path), desc="Processing"):
filepath = Path(filename)
dflimg = DFLIMG.load(filepath)
if dflimg is not None:
paths_to_extract.append (filepath)
image_to_face_mat = dflimg.get_image_to_face_mat()
src_filename = dflimg.get_source_filename()
#img = cv2_imread(filename)
h,w,c = dflimg.get_shape()
fanseg_mask = LandmarksProcessor.get_image_hull_mask( (h,w,c), dflimg.get_landmarks() )
idx_name = '%.5d' % int(src_filename.split('.')[0])
idx_files = [ x for x in masks_files if idx_name in x ]
skin_files = [ x for x in idx_files if 'skin' in x ]
eye_glass_files = [ x for x in idx_files if 'eye_g' in x ]
for files, is_invert in [ (skin_files,False),
(eye_glass_files,True) ]:
if len(files) > 0:
mask = cv2_imread(files[0])
mask = mask[...,0]
mask[mask == 255] = 1
mask = mask.astype(np.float32)
mask = cv2.resize(mask, (1024,1024) )
mask = cv2.warpAffine(mask, image_to_face_mat, (w, h), cv2.INTER_LANCZOS4)
if not is_invert:
fanseg_mask *= mask[...,None]
else:
fanseg_mask *= (1-mask[...,None])
#cv2.imshow("", (fanseg_mask*255).astype(np.uint8) )
#cv2.waitKey(0)
dflimg.embed_and_set (filename, fanseg_mask=fanseg_mask)
def get_full_face_mask():
xseg_mask = sample.get_xseg_mask()
if xseg_mask is not None:
if xseg_mask.shape[0] != h or xseg_mask.shape[1] != w:
xseg_mask = cv2.resize(xseg_mask, (w,h), interpolation=cv2.INTER_CUBIC)
xseg_mask = imagelib.normalize_channels(xseg_mask, 1)
return np.clip(xseg_mask, 0, 1)
else:
full_face_mask = LandmarksProcessor.get_image_hull_mask (sample_bgr.shape, sample_landmarks, eyebrows_expand_mod=sample.eyebrows_expand_mod )
return np.clip(full_face_mask, 0, 1)
def onClientProcessData(self, data):
filename_path = Path( data[0] )
dflpng = DFLPNG.load( str(filename_path), print_on_no_embedded_data=True )
if dflpng is not None:
image = cv2.imread( str(filename_path) )
image = ( image * \
LandmarksProcessor.get_image_hull_mask (image, dflpng.get_landmarks()) \
).astype(np.uint8)
return [ str(filename_path), estimate_sharpness( image ) ]
else:
return [ str(filename_path), 0 ]
def process_data(self, data):
filepath = Path( data[0] )
dflimg = DFLIMG.load (filepath)
if dflimg is None or not dflimg.has_data():
self.log_err (f"{filepath.name} is not a dfl image file")
return [ str(filepath), 0 ]
else:
image = cv2_imread( str(filepath) )
face_mask = LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks())
image = (image*face_mask).astype(np.uint8)
return [ str(filepath), estimate_sharpness(image) ]
def onClientProcessData(self, data):
filepath = Path(data[0])
if filepath.suffix == '.png':
dflimg = DFLPNG.load(str(filepath), print_on_no_embedded_data=True)
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load(str(filepath), print_on_no_embedded_data=True)
else:
print("%s is not a dfl image file" % (filepath.name))
dflimg = None
if dflimg is not None:
image = cv2.imread(str(filepath))
image = ( image * \
LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks()) \
).astype(np.uint8)
return [str(filepath), estimate_sharpness(image)]
else:
return [str(filepath), 0]
def process_data(self, data):
filepath = Path( data[0] )
if filepath.suffix == '.png':
dflimg = DFLPNG.load( str(filepath) )
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load ( str(filepath) )
else:
dflimg = None
if dflimg is not None:
image = cv2_imread( str(filepath) )
image = ( image * \
LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks()) \
).astype(np.uint8)
return [ str(filepath), estimate_sharpness( image ) ]
else:
self.log_err ("%s is not a dfl image file" % (filepath.name) )
return [ str(filepath), 0 ]
def process_data(self, data):
filename = data[0]
filepath = Path(filename)
if filepath.suffix == '.png':
dflimg = DFLPNG.load(str(filepath))
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load(str(filepath))
else:
dflimg = None
image_to_face_mat = dflimg.get_image_to_face_mat()
src_filename = dflimg.get_source_filename()
img = cv2_imread(filename)
h, w, c = img.shape
fanseg_mask = LandmarksProcessor.get_image_hull_mask(
img.shape, dflimg.get_landmarks())
idx_name = '%.5d' % int(src_filename.split('.')[0])
idx_files = [x for x in self.masks_files_paths if idx_name in x]
skin_files = [x for x in idx_files if 'skin' in x]
eye_glass_files = [x for x in idx_files if 'eye_g' in x]
for files, is_invert in [(skin_files, False),
(eye_glass_files, True)]:
if len(files) > 0:
mask = cv2_imread(files[0])
mask = mask[..., 0]
mask[mask == 255] = 1
mask = mask.astype(np.float32)
mask = cv2.resize(mask, (1024, 1024))
mask = cv2.warpAffine(mask, image_to_face_mat, (w, h),
cv2.INTER_LANCZOS4)
if not is_invert:
fanseg_mask *= mask[..., None]
else:
fanseg_mask *= (1 - mask[..., None])
dflimg.embed_and_set(filename, fanseg_mask=fanseg_mask)
return 1
def process_data(self, data):
filepath = Path(data[0])
dflimg = DFLIMG.load(filepath)
if dflimg is None or not dflimg.has_data():
self.log_err(f"{filepath.name} is not a dfl image file")
return [str(filepath), 0]
else:
image = cv2_imread(str(filepath))
face_mask = LandmarksProcessor.get_image_hull_mask(
image.shape, dflimg.get_landmarks())
image = (image * face_mask).astype(np.uint8)
if self.estimate_motion_blur:
value = cv2.Laplacian(image, cv2.CV_64F, ksize=11).var()
else:
value = estimate_sharpness(image)
return [str(filepath), value]
def convert_face(self, img_bgr, img_face_landmarks, debug):
if debug:
debugs = [img_bgr.copy()]
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr, img_face_landmarks)
face_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, self.output_size, face_type=self.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
self.output_size,
face_type=self.face_type,
scale=self.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat,
(self.output_size, self.output_size),
flags=cv2.INTER_LANCZOS4)
dst_face_mask_a_0 = cv2.warpAffine(
img_face_mask_a,
face_mat, (self.output_size, self.output_size),
flags=cv2.INTER_LANCZOS4)
predictor_input_bgr = cv2.resize(
dst_face_bgr,
(self.predictor_input_size, self.predictor_input_size))
predictor_input_mask_a_0 = cv2.resize(
dst_face_mask_a_0,
(self.predictor_input_size, self.predictor_input_size))
predictor_input_mask_a = np.expand_dims(predictor_input_mask_a_0, -1)
predicted_bgra = self.predictor(
np.concatenate((predictor_input_bgr, predictor_input_mask_a), -1))
prd_face_bgr = np.clip(predicted_bgra[:, :, 0:3], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted_bgra[:, :, 3], 0.0, 1.0)
prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0
prd_face_mask_a = np.expand_dims(prd_face_mask_a_0, axis=-1)
prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1)
img_prd_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa,
face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=float),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4)
img_prd_face_mask_aaa = np.clip(img_prd_face_mask_aaa, 0.0, 1.0)
img_face_mask_aaa = img_prd_face_mask_aaa
if debug:
debugs += [img_face_mask_aaa.copy()]
img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0
img_face_mask_flatten_aaa = img_face_mask_aaa.copy()
img_face_mask_flatten_aaa[img_face_mask_flatten_aaa > 0.9] = 1.0
maxregion = np.argwhere(img_face_mask_flatten_aaa == 1.0)
out_img = img_bgr.copy()
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
if debug:
print("maxregion.size: %d, minx:%d, maxx:%d miny:%d, maxy:%d" %
(maxregion.size, minx, maxx, miny, maxy))
lenx = maxx - minx
leny = maxy - miny
if lenx >= 4 and leny >= 4:
masky = int(minx + (lenx // 2))
maskx = int(miny + (leny // 2))
lowest_len = min(lenx, leny)
if debug:
print("lowest_len = %f" % (lowest_len))
ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) *
0.01 * self.erode_mask_modifier)
blur = int(lowest_len * 0.10 * 0.01 * self.blur_mask_modifier)
if debug:
print("erode_size = %d, blur_size = %d" % (ero, blur))
img_mask_blurry_aaa = img_face_mask_aaa
if self.erode_mask:
if ero > 0:
img_mask_blurry_aaa = cv2.erode(
img_mask_blurry_aaa,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
(ero, ero)),
iterations=1)
elif ero < 0:
img_mask_blurry_aaa = cv2.dilate(
img_mask_blurry_aaa,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
if self.blur_mask and blur > 0:
img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa,
(blur, blur))
img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0)
if self.clip_border_mask_per > 0:
prd_border_rect_mask_a = np.ones(
prd_face_mask_a.shape, dtype=prd_face_mask_a.dtype)
prd_border_size = int(prd_border_rect_mask_a.shape[1] *
self.clip_border_mask_per)
prd_border_rect_mask_a[0:prd_border_size, :, :] = 0
prd_border_rect_mask_a[-prd_border_size:, :, :] = 0
prd_border_rect_mask_a[:, 0:prd_border_size, :] = 0
prd_border_rect_mask_a[:, -prd_border_size:, :] = 0
prd_border_rect_mask_a = np.expand_dims(
cv2.blur(prd_border_rect_mask_a,
(prd_border_size, prd_border_size)), -1)
if self.mode == 'hist-match-bw':
prd_face_bgr = cv2.cvtColor(prd_face_bgr,
cv2.COLOR_BGR2GRAY)
prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1),
(3, ), -1)
if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
if debug:
debugs += [
cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
]
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=prd_face_bgr.dtype)
if self.masked_hist_match:
hist_mask_a *= prd_face_mask_a
hist_match_1 = prd_face_bgr * hist_mask_a + (
1.0 - hist_mask_a) * np.ones(prd_face_bgr.shape[:2] +
(1, ),
dtype=prd_face_bgr.dtype)
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + (
1.0 - hist_mask_a) * np.ones(prd_face_bgr.shape[:2] +
(1, ),
dtype=prd_face_bgr.dtype)
hist_match_2[hist_match_1 > 1.0] = 1.0
new_prd_face_bgr = image_utils.color_hist_match(
hist_match_1, hist_match_2)
prd_face_bgr = new_prd_face_bgr
if self.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(np.float32)
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size, out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
if debug:
debugs += [out_img.copy()]
debugs += [img_mask_blurry_aaa.copy()]
if self.mode == 'seamless' or self.mode == 'seamless-hist-match':
out_img = np.clip(
img_bgr * (1 - img_face_mask_aaa) +
(out_img * img_face_mask_aaa), 0, 1.0)
if debug:
debugs += [out_img.copy()]
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8),
(img_bgr * 255).astype(np.uint8),
(img_face_mask_flatten_aaa * 255).astype(np.uint8),
(masky, maskx), cv2.NORMAL_CLONE)
out_img = out_img.astype(np.float32) / 255.0
if debug:
debugs += [out_img.copy()]
if self.clip_border_mask_per > 0:
img_prd_border_rect_mask_a = cv2.warpAffine(
prd_border_rect_mask_a, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
img_prd_border_rect_mask_a = np.expand_dims(
img_prd_border_rect_mask_a, -1)
out_img = out_img * img_prd_border_rect_mask_a + img_bgr * (
1.0 - img_prd_border_rect_mask_a)
img_mask_blurry_aaa *= img_prd_border_rect_mask_a
out_img = np.clip(
img_bgr * (1 - img_mask_blurry_aaa) +
(out_img * img_mask_blurry_aaa), 0, 1.0)
if self.mode == 'seamless-hist-match':
out_face_bgr = cv2.warpAffine(
out_img, face_mat,
(self.output_size, self.output_size))
new_out_face_bgr = image_utils.color_hist_match(
out_face_bgr, dst_face_bgr)
new_out = cv2.warpAffine(
new_out_face_bgr, face_mat, img_size, img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_mask_blurry_aaa) +
(new_out * img_mask_blurry_aaa), 0, 1.0)
if self.transfercolor:
if self.TFLabConverter is None:
self.TFLabConverter = image_utils.TFLabConverter()
img_lab_l, img_lab_a, img_lab_b = np.split(
self.TFLabConverter.bgr2lab(img_bgr), 3, axis=-1)
out_img_lab_l, out_img_lab_a, out_img_lab_b = np.split(
self.TFLabConverter.bgr2lab(out_img), 3, axis=-1)
out_img = self.TFLabConverter.lab2bgr(
np.concatenate([out_img_lab_l, img_lab_a, img_lab_b],
axis=-1))
if self.final_image_color_degrade_power != 0:
if debug:
debugs += [out_img.copy()]
out_img_reduced = image_utils.reduce_colors(out_img, 256)
if self.final_image_color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = self.final_image_color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
if self.alpha:
new_image = out_img.copy()
new_image = (new_image * 255).astype(
np.uint8) #convert image to int
b_channel, g_channel, r_channel = cv2.split(
new_image) #splitting RGB
alpha_channel = img_mask_blurry_aaa.copy(
) #making copy of alpha channel
alpha_channel = (alpha_channel * 255).astype(np.uint8)
alpha_channel, tmp2, tmp3 = cv2.split(
alpha_channel
) #splitting alpha to three channels, they all same in original alpha channel, we need just one
out_img = cv2.merge(
(b_channel, g_channel, r_channel,
alpha_channel)) #mergin RGB with alpha
out_img = out_img.astype(np.float32) / 255.0
if debug:
debugs += [out_img.copy()]
return debugs if debug else out_img
def MergeMaskedFace(predictor_func, predictor_input_shape, face_enhancer_func,
xseg_256_extract_func, cfg, frame_info, img_bgr_uint8,
img_bgr, img_face_landmarks):
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
input_size = predictor_input_shape[0]
mask_subres_size = input_size * 4
output_size = input_size
if cfg.super_resolution_power != 0:
output_size *= 4
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
if mask_subres_size == output_size:
face_mask_output_mat = face_output_mat
else:
face_mask_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
mask_subres_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_bgr = np.clip(dst_face_bgr, 0, 1)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size))
predicted = predictor_func(predictor_input_bgr)
prd_face_bgr = np.clip(predicted[0], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0)
prd_face_dst_mask_a_0 = np.clip(predicted[2], 0, 1.0)
if cfg.super_resolution_power != 0:
prd_face_bgr_enhanced = face_enhancer_func(prd_face_bgr,
is_tanh=True,
preserve_size=False)
mod = cfg.super_resolution_power / 100.0
prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * (
1.0 - mod) + prd_face_bgr_enhanced * mod
prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
if cfg.super_resolution_power != 0:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
prd_face_dst_mask_a_0 = cv2.resize(prd_face_dst_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
if cfg.mask_mode == 1: #dst
wrk_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
elif cfg.mask_mode == 2: #learned-prd
wrk_face_mask_a_0 = prd_face_mask_a_0
elif cfg.mask_mode == 3: #learned-dst
wrk_face_mask_a_0 = prd_face_dst_mask_a_0
elif cfg.mask_mode == 4: #learned-prd*learned-dst
wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0
elif cfg.mask_mode == 5: #learned-prd+learned-dst
wrk_face_mask_a_0 = np.clip(prd_face_mask_a_0 + prd_face_dst_mask_a_0,
0, 1)
elif cfg.mask_mode >= 6 and cfg.mask_mode <= 9: #XSeg modes
if cfg.mask_mode == 6 or cfg.mask_mode == 8 or cfg.mask_mode == 9:
# obtain XSeg-prd
prd_face_xseg_bgr = cv2.resize(prd_face_bgr,
(xseg_input_size, ) * 2,
interpolation=cv2.INTER_CUBIC)
prd_face_xseg_mask = xseg_256_extract_func(prd_face_xseg_bgr)
X_prd_face_mask_a_0 = cv2.resize(prd_face_xseg_mask,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
if cfg.mask_mode >= 7 and cfg.mask_mode <= 9:
# obtain XSeg-dst
xseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, xseg_input_size, face_type=cfg.face_type)
dst_face_xseg_bgr = cv2.warpAffine(img_bgr,
xseg_mat,
(xseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_xseg_mask = xseg_256_extract_func(dst_face_xseg_bgr)
X_dst_face_mask_a_0 = cv2.resize(dst_face_xseg_mask,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
if cfg.mask_mode == 6: #'XSeg-prd'
wrk_face_mask_a_0 = X_prd_face_mask_a_0
elif cfg.mask_mode == 7: #'XSeg-dst'
wrk_face_mask_a_0 = X_dst_face_mask_a_0
elif cfg.mask_mode == 8: #'XSeg-prd*XSeg-dst'
wrk_face_mask_a_0 = X_prd_face_mask_a_0 * X_dst_face_mask_a_0
elif cfg.mask_mode == 9: #learned-prd*learned-dst*XSeg-prd*XSeg-dst
wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0 * X_prd_face_mask_a_0 * X_dst_face_mask_a_0
wrk_face_mask_a_0[wrk_face_mask_a_0 < (1.0 /
255.0)] = 0.0 # get rid of noise
# resize to mask_subres_size
if wrk_face_mask_a_0.shape[0] != mask_subres_size:
wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0,
(mask_subres_size, mask_subres_size),
interpolation=cv2.INTER_CUBIC)
# process mask in local predicted space
if 'raw' not in cfg.mode:
# add zero pad
wrk_face_mask_a_0 = np.pad(wrk_face_mask_a_0, input_size)
ero = cfg.erode_mask_modifier
blur = cfg.blur_mask_modifier
if ero > 0:
wrk_face_mask_a_0 = cv2.erode(wrk_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
wrk_face_mask_a_0 = cv2.dilate(wrk_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
# clip eroded/dilated mask in actual predict area
# pad with half blur size in order to accuratelly fade to zero at the boundary
clip_size = input_size + blur // 2
wrk_face_mask_a_0[:clip_size, :] = 0
wrk_face_mask_a_0[-clip_size:, :] = 0
wrk_face_mask_a_0[:, :clip_size] = 0
wrk_face_mask_a_0[:, -clip_size:] = 0
if blur > 0:
blur = blur + (1 - blur % 2)
wrk_face_mask_a_0 = cv2.GaussianBlur(wrk_face_mask_a_0,
(blur, blur), 0)
wrk_face_mask_a_0 = wrk_face_mask_a_0[input_size:-input_size,
input_size:-input_size]
wrk_face_mask_a_0 = np.clip(wrk_face_mask_a_0, 0, 1)
img_face_mask_a = cv2.warpAffine(wrk_face_mask_a_0,
face_mask_output_mat,
img_size,
np.zeros(img_bgr.shape[0:2],
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)[..., None]
img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0)
img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise
if wrk_face_mask_a_0.shape[0] != output_size:
wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
wrk_face_mask_a = wrk_face_mask_a_0[..., None]
out_img = None
out_merging_mask_a = None
if cfg.mode == 'original':
return img_bgr, img_face_mask_a
elif 'raw' in cfg.mode:
if cfg.mode == 'raw-rgb':
out_img_face = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img_face_mask = cv2.warpAffine(
np.ones_like(prd_face_bgr), face_output_mat, img_size,
np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img = img_bgr * (
1 - out_img_face_mask) + out_img_face * out_img_face_mask
out_merging_mask_a = img_face_mask_a
elif cfg.mode == 'raw-predict':
out_img = prd_face_bgr
out_merging_mask_a = wrk_face_mask_a
else:
raise ValueError(f"undefined raw type {cfg.mode}")
out_img = np.clip(out_img, 0.0, 1.0)
else:
# Process if the mask meets minimum size
maxregion = np.argwhere(img_face_mask_a >= 0.1)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
wrk_face_mask_area_a = wrk_face_mask_a.copy()
wrk_face_mask_area_a[wrk_face_mask_area_a > 0] = 1.0
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1: #rct
prd_face_bgr = imagelib.reinhard_color_transfer(
np.clip(prd_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
np.clip(dst_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
prd_face_bgr = imagelib.color_transfer_sot(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a,
steps=10,
batch_size=30)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
prd_face_bgr = imagelib.color_transfer_mix(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
if cfg.mode == 'hist-match':
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= wrk_face_mask_area_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2,
cfg.hist_match_threshold).astype(dtype=np.float32)
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_a = None
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_a.copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[
img_face_seamless_mask_a <= i / 10.0] = 0.0
break
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.empty_like(img_bgr),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img = np.clip(out_img, 0.0, 1.0)
if 'seamless' in cfg.mode:
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
cfg_mp = cfg.motion_blur_power / 100.0
out_img = img_bgr * (1 - img_face_mask_a) + (out_img *
img_face_mask_a)
if ('seamless' in cfg.mode and cfg.color_transfer_mode != 0) or \
cfg.mode == 'seamless-hist-match' or \
cfg_mp != 0 or \
cfg.blursharpen_amount != 0 or \
cfg.image_denoise_power != 0 or \
cfg.bicubic_degrade_power != 0:
out_face_bgr = cv2.warpAffine(out_img,
face_mat,
(output_size, output_size),
flags=cv2.INTER_CUBIC)
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
out_face_bgr = imagelib.reinhard_color_transfer(
np.clip(
out_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
np.clip(
dst_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8))
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0,
1.0)
elif cfg.color_transfer_mode == 2: #lct
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
out_face_bgr = imagelib.color_transfer_sot(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a,
steps=10,
batch_size=30)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
out_face_bgr = imagelib.color_transfer_mix(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr,
cfg.hist_match_threshold)
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_power != 0:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.blursharpen_amount != 0:
out_face_bgr = imagelib.blursharpen(
out_face_bgr, cfg.sharpen_mode, 3,
cfg.blursharpen_amount)
if cfg.image_denoise_power != 0:
n = cfg.image_denoise_power
while n > 0:
img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
if int(n / 100) != 0:
img_bgr = img_bgr_denoised
else:
pass_power = (n % 100) / 100.0
img_bgr = img_bgr * (
1.0 -
pass_power) + img_bgr_denoised * pass_power
n = max(n - 10, 0)
if cfg.bicubic_degrade_power != 0:
p = 1.0 - cfg.bicubic_degrade_power / 101.0
img_bgr_downscaled = cv2.resize(
img_bgr,
(int(img_size[0] * p), int(img_size[1] * p)),
interpolation=cv2.INTER_CUBIC)
img_bgr = cv2.resize(img_bgr_downscaled,
img_size,
interpolation=cv2.INTER_CUBIC)
new_out = cv2.warpAffine(
out_face_bgr, face_mat, img_size,
np.empty_like(img_bgr),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img = np.clip(
img_bgr * (1 - img_face_mask_a) +
(new_out * img_face_mask_a), 0, 1.0)
if cfg.color_degrade_power != 0:
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
out_merging_mask_a = img_face_mask_a
if out_img is None:
out_img = img_bgr.copy()
return out_img, out_merging_mask_a
def process(samples,
sample_process_options,
output_sample_types,
debug,
ct_sample=None):
SPTF = SampleProcessor.Types
sample_rnd_seed = np.random.randint(0x80000000)
outputs = []
for sample in samples:
sample_bgr = sample.load_bgr()
ct_sample_bgr = None
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range,
rnd_seed=sample_rnd_seed)
outputs_sample = []
for opts in output_sample_types:
resolution = opts.get('resolution', 0)
types = opts.get('types', [])
motion_blur = opts.get('motion_blur', None)
gaussian_blur = opts.get('gaussian_blur', None)
ct_mode = opts.get('ct_mode', 'None')
normalize_tanh = opts.get('normalize_tanh', False)
data_format = opts.get('data_format', 'NHWC')
img_type = SPTF.NONE
target_face_type = SPTF.NONE
mode_type = SPTF.NONE
for t in types:
if t >= SPTF.IMG_TYPE_BEGIN and t < SPTF.IMG_TYPE_END:
img_type = t
elif t >= SPTF.FACE_TYPE_BEGIN and t < SPTF.FACE_TYPE_END:
target_face_type = t
elif t >= SPTF.MODE_BEGIN and t < SPTF.MODE_END:
mode_type = t
if mode_type == SPTF.MODE_FACE_MASK_HULL and not is_face_sample:
raise ValueError(
"MODE_FACE_MASK_HULL applicable only for face samples")
if mode_type == SPTF.MODE_FACE_MASK_STRUCT and not is_face_sample:
raise ValueError(
"MODE_FACE_MASK_STRUCT applicable only for face samples"
)
if is_face_sample:
if target_face_type == SPTF.NONE:
raise ValueError(
"target face type must be defined for face samples"
)
can_warp = (img_type == SPTF.IMG_WARPED
or img_type == SPTF.IMG_WARPED_TRANSFORMED)
can_transform = (img_type == SPTF.IMG_WARPED_TRANSFORMED
or img_type == SPTF.IMG_TRANSFORMED)
if img_type == SPTF.NONE:
raise ValueError('expected IMG_ type')
if img_type == SPTF.IMG_LANDMARKS_ARRAY:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
out_sample = l
elif img_type == SPTF.IMG_PITCH_YAW_ROLL or img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch_yaw_roll = sample.get_pitch_yaw_roll()
if params['flip']:
yaw = -yaw
if img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch = np.clip((pitch / math.pi) / 2.0 + 0.5, 0, 1)
yaw = np.clip((yaw / math.pi) / 2.0 + 0.5, 0, 1)
roll = np.clip((roll / math.pi) / 2.0 + 0.5, 0, 1)
out_sample = (pitch, yaw, roll)
else:
if mode_type == SPTF.NONE:
raise ValueError('expected MODE_ type')
if mode_type == SPTF.MODE_FACE_MASK_HULL:
if sample.eyebrows_expand_mod is not None:
img = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.eyebrows_expand_mod)
else:
img = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape, sample.landmarks)
if sample.ie_polys is not None:
sample.ie_polys.overlay_mask(img)
elif mode_type == SPTF.MODE_FACE_MASK_STRUCT:
if sample.eyebrows_expand_mod is not None:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.eyebrows_expand_mod)
else:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape, sample.landmarks)
else:
img = sample_bgr
if motion_blur is not None:
chance, mb_max_size = motion_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
img = imagelib.LinearMotionBlur(
img,
np.random.randint(mb_max_size) + 1,
np.random.randint(360))
if gaussian_blur is not None:
chance, kernel_max_size = gaussian_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
img = cv2.GaussianBlur(
img,
(np.random.randint(kernel_max_size) * 2 +
1, ) * 2, 0)
if is_face_sample:
target_ft = SampleProcessor.SPTF_FACETYPE_TO_FACETYPE[
target_face_type]
if target_ft > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
%
(sample.filename, sample.face_type, target_ft))
if sample.face_type == FaceType.MARK_ONLY:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0], target_ft)
if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT:
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=False)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)[..., None]
else:
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=True)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
else:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, target_ft)
if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT:
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=False)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_CONSTANT,
flags=cv2.INTER_CUBIC)[..., None]
else:
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=True)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_REPLICATE,
flags=cv2.INTER_CUBIC)
else:
img = imagelib.warp_by_params(params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=True)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT:
out_sample = np.clip(img.astype(np.float32), 0, 1)
else:
img = np.clip(img.astype(np.float32), 0, 1)
if ct_mode is not None and ct_sample is not None:
if ct_sample_bgr is None:
ct_sample_bgr = ct_sample.load_bgr()
img = imagelib.color_transfer(
ct_mode, img,
cv2.resize(ct_sample_bgr,
(resolution, resolution),
cv2.INTER_LINEAR))
if mode_type == SPTF.MODE_BGR:
out_sample = img
elif mode_type == SPTF.MODE_BGR_SHUFFLE:
rnd_state = np.random.RandomState(sample_rnd_seed)
out_sample = np.take(img,
rnd_state.permutation(
img.shape[-1]),
axis=-1)
elif mode_type == SPTF.MODE_BGR_RANDOM_HSV_SHIFT:
rnd_state = np.random.RandomState(sample_rnd_seed)
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
h, s, v = cv2.split(hsv)
h = (h + rnd_state.randint(360)) % 360
s = np.clip(s + rnd_state.random() - 0.5, 0, 1)
v = np.clip(v + rnd_state.random() - 0.5, 0, 1)
hsv = cv2.merge([h, s, v])
out_sample = np.clip(
cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0, 1)
elif mode_type == SPTF.MODE_G:
out_sample = cv2.cvtColor(img,
cv2.COLOR_BGR2GRAY)[...,
None]
elif mode_type == SPTF.MODE_GGG:
out_sample = np.repeat(
np.expand_dims(
cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1)
if not debug:
if normalize_tanh:
out_sample = np.clip(out_sample * 2.0 - 1.0, -1.0,
1.0)
if data_format == "NCHW":
out_sample = np.transpose(out_sample, (2, 0, 1))
outputs_sample.append(out_sample)
outputs += [outputs_sample]
return outputs
def mask_editor_main(input_dir,
confirmed_dir=None,
skipped_dir=None,
no_default_mask=False):
input_path = Path(input_dir)
confirmed_path = Path(confirmed_dir)
skipped_path = Path(skipped_dir)
if not input_path.exists():
raise ValueError('Input directory not found. Please ensure it exists.')
if not confirmed_path.exists():
confirmed_path.mkdir(parents=True)
if not skipped_path.exists():
skipped_path.mkdir(parents=True)
if not no_default_mask:
eyebrows_expand_mod = np.clip(
io.input_int(
"Default eyebrows expand modifier? (0..400, skip:100) : ",
100), 0, 400) / 100.0
else:
eyebrows_expand_mod = None
wnd_name = "MaskEditor tool"
io.named_window(wnd_name)
io.capture_mouse(wnd_name)
io.capture_keys(wnd_name)
cached_images = {}
image_paths = [Path(x) for x in Path_utils.get_image_paths(input_path)]
done_paths = []
done_images_types = {}
image_paths_total = len(image_paths)
saved_ie_polys = IEPolys()
zoom_factor = 1.0
preview_images_count = 9
target_wh = 256
do_prev_count = 0
do_save_move_count = 0
do_save_count = 0
do_skip_move_count = 0
do_skip_count = 0
def jobs_count():
return do_prev_count + do_save_move_count + do_save_count + do_skip_move_count + do_skip_count
is_exit = False
while not is_exit:
if len(image_paths) > 0:
filepath = image_paths.pop(0)
else:
filepath = None
next_image_paths = image_paths[0:preview_images_count]
next_image_paths_names = [path.name for path in next_image_paths]
prev_image_paths = done_paths[-preview_images_count:]
prev_image_paths_names = [path.name for path in prev_image_paths]
for key in list(cached_images.keys()):
if key not in prev_image_paths_names and \
key not in next_image_paths_names:
cached_images.pop(key)
for paths in [prev_image_paths, next_image_paths]:
for path in paths:
if path.name not in cached_images:
cached_images[path.name] = cv2_imread(str(path)) / 255.0
if filepath is not None:
if filepath.suffix == '.png':
dflimg = DFLPNG.load(str(filepath))
elif filepath.suffix == '.jpg':
dflimg = DFLJPG.load(str(filepath))
else:
dflimg = None
if dflimg is None:
io.log_err("%s is not a dfl image file" % (filepath.name))
continue
else:
lmrks = dflimg.get_landmarks()
ie_polys = dflimg.get_ie_polys()
fanseg_mask = dflimg.get_fanseg_mask()
if filepath.name in cached_images:
img = cached_images[filepath.name]
else:
img = cached_images[filepath.name] = cv2_imread(
str(filepath)) / 255.0
if fanseg_mask is not None:
mask = fanseg_mask
else:
if no_default_mask:
mask = np.zeros((target_wh, target_wh, 3))
else:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape,
lmrks,
eyebrows_expand_mod=eyebrows_expand_mod)
else:
img = np.zeros((target_wh, target_wh, 3))
mask = np.ones((target_wh, target_wh, 3))
ie_polys = None
def get_status_lines_func():
return [
'Progress: %d / %d . Current file: %s' %
(len(done_paths), image_paths_total,
str(filepath.name) if filepath is not None else "end"),
'[Left mouse button] - mark include mask.',
'[Right mouse button] - mark exclude mask.',
'[Middle mouse button] - finish current poly.',
'[Mouse wheel] - undo/redo poly or point. [+ctrl] - undo to begin/redo to end',
'[r] - applies edits made to last saved image.',
'[q] - prev image. [w] - skip and move to %s. [e] - save and move to %s. '
% (skipped_path.name, confirmed_path.name),
'[z] - prev image. [x] - skip. [c] - save. ',
'hold [shift] - speed up the frame counter by 10.',
'[-/+] - window zoom [esc] - quit',
]
try:
ed = MaskEditor(img,
[(done_images_types[name], cached_images[name])
for name in prev_image_paths_names],
[(0, cached_images[name])
for name in next_image_paths_names], mask,
ie_polys, get_status_lines_func)
except Exception as e:
print(e)
continue
next = False
while not next:
io.process_messages(0.005)
if jobs_count() == 0:
for (x, y, ev, flags) in io.get_mouse_events(wnd_name):
x, y = int(x / zoom_factor), int(y / zoom_factor)
ed.set_mouse_pos(x, y)
if filepath is not None:
if ev == io.EVENT_LBUTTONDOWN:
ed.mask_point(1)
elif ev == io.EVENT_RBUTTONDOWN:
ed.mask_point(0)
elif ev == io.EVENT_MBUTTONDOWN:
ed.mask_finish()
elif ev == io.EVENT_MOUSEWHEEL:
if flags & 0x80000000 != 0:
if flags & 0x8 != 0:
ed.undo_to_begin_point()
else:
ed.undo_point()
else:
if flags & 0x8 != 0:
ed.redo_to_end_point()
else:
ed.redo_point()
for key, chr_key, ctrl_pressed, alt_pressed, shift_pressed in io.get_key_events(
wnd_name):
if chr_key == 'q' or chr_key == 'z':
do_prev_count = 1 if not shift_pressed else 10
elif chr_key == '-':
zoom_factor = np.clip(zoom_factor - 0.1, 0.1, 4.0)
ed.set_screen_changed()
elif chr_key == '+':
zoom_factor = np.clip(zoom_factor + 0.1, 0.1, 4.0)
ed.set_screen_changed()
elif key == 27: #esc
is_exit = True
next = True
break
elif filepath is not None:
if chr_key == 'e':
saved_ie_polys = ed.ie_polys
do_save_move_count = 1 if not shift_pressed else 10
elif chr_key == 'c':
saved_ie_polys = ed.ie_polys
do_save_count = 1 if not shift_pressed else 10
elif chr_key == 'w':
do_skip_move_count = 1 if not shift_pressed else 10
elif chr_key == 'x':
do_skip_count = 1 if not shift_pressed else 10
elif chr_key == 'r' and saved_ie_polys != None:
ed.set_ie_polys(saved_ie_polys)
if do_prev_count > 0:
do_prev_count -= 1
if len(done_paths) > 0:
if filepath is not None:
image_paths.insert(0, filepath)
filepath = done_paths.pop(-1)
done_images_types[filepath.name] = 0
if filepath.parent != input_path:
new_filename_path = input_path / filepath.name
filepath.rename(new_filename_path)
image_paths.insert(0, new_filename_path)
else:
image_paths.insert(0, filepath)
next = True
elif filepath is not None:
if do_save_move_count > 0:
do_save_move_count -= 1
ed.mask_finish()
dflimg.embed_and_set(
str(filepath),
ie_polys=ed.get_ie_polys(),
eyebrows_expand_mod=eyebrows_expand_mod)
done_paths += [confirmed_path / filepath.name]
done_images_types[filepath.name] = 2
filepath.rename(done_paths[-1])
next = True
elif do_save_count > 0:
do_save_count -= 1
ed.mask_finish()
dflimg.embed_and_set(
str(filepath),
ie_polys=ed.get_ie_polys(),
eyebrows_expand_mod=eyebrows_expand_mod)
done_paths += [filepath]
done_images_types[filepath.name] = 2
next = True
elif do_skip_move_count > 0:
do_skip_move_count -= 1
done_paths += [skipped_path / filepath.name]
done_images_types[filepath.name] = 1
filepath.rename(done_paths[-1])
next = True
elif do_skip_count > 0:
do_skip_count -= 1
done_paths += [filepath]
done_images_types[filepath.name] = 1
next = True
else:
do_save_move_count = do_save_count = do_skip_move_count = do_skip_count = 0
if jobs_count() == 0:
if ed.switch_screen_changed():
screen = ed.make_screen()
if zoom_factor != 1.0:
h, w, c = screen.shape
screen = cv2.resize(
screen,
(int(w * zoom_factor), int(h * zoom_factor)))
io.show_image(wnd_name, screen)
io.process_messages(0.005)
io.destroy_all_windows()
def convert_face(self, img_bgr, img_face_landmarks, debug):
if (self.mask_mode == 3
or self.mask_mode == 4) and self.fan_seg == None:
self.fan_seg = FANSegmentator(256,
FaceType.toString(FaceType.FULL))
if self.over_res != 1:
img_bgr = cv2.resize(img_bgr, (img_bgr.shape[1] * self.over_res,
img_bgr.shape[0] * self.over_res))
img_face_landmarks = img_face_landmarks * self.over_res
if debug:
debugs = [img_bgr.copy()]
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
face_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, self.output_size, face_type=self.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
self.output_size,
face_type=self.face_type,
scale=self.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat,
(self.output_size, self.output_size),
flags=cv2.INTER_LANCZOS4)
dst_face_mask_a_0 = cv2.warpAffine(
img_face_mask_a,
face_mat, (self.output_size, self.output_size),
flags=cv2.INTER_LANCZOS4)
predictor_input_bgr = cv2.resize(
dst_face_bgr,
(self.predictor_input_size, self.predictor_input_size))
predictor_input_mask_a_0 = cv2.resize(
dst_face_mask_a_0,
(self.predictor_input_size, self.predictor_input_size))
predictor_input_mask_a = np.expand_dims(predictor_input_mask_a_0, -1)
predicted_bgra = self.predictor_func(
np.concatenate((predictor_input_bgr, predictor_input_mask_a), -1))
prd_face_bgr = np.clip(predicted_bgra[:, :, 0:3], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted_bgra[:, :, 3], 0.0, 1.0)
if self.mask_mode == 2: #dst
prd_face_mask_a_0 = predictor_input_mask_a_0
elif self.mask_mode == 3: #FAN-prd
prd_face_bgr_256 = cv2.resize(prd_face_bgr, (256, 256))
prd_face_bgr_256_mask = self.fan_seg.extract_from_bgr(
np.expand_dims(prd_face_bgr_256, 0))[0]
prd_face_mask_a_0 = cv2.resize(
prd_face_bgr_256_mask,
(self.predictor_input_size, self.predictor_input_size))
elif self.mask_mode == 4: #FAN-dst
face_256_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, 256, face_type=FaceType.FULL)
dst_face_256_bgr = cv2.warpAffine(img_bgr,
face_256_mat, (256, 256),
flags=cv2.INTER_LANCZOS4)
dst_face_256_mask = self.fan_seg.extract_from_bgr(
np.expand_dims(dst_face_256_bgr, 0))[0]
prd_face_mask_a_0 = cv2.resize(
dst_face_256_mask,
(self.predictor_input_size, self.predictor_input_size))
prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0
prd_face_mask_a = np.expand_dims(prd_face_mask_a_0, axis=-1)
prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1)
img_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa,
face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4)
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0.0, 1.0)
img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 #get rid of noise
if debug:
debugs += [img_face_mask_aaa.copy()]
if 'seamless' in self.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_aaa = None
for i in range(9, 0, -1):
a = img_face_mask_aaa > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_aaa = img_face_mask_aaa.copy()
img_face_seamless_mask_aaa[a] = 1.0
img_face_seamless_mask_aaa[img_face_seamless_mask_aaa <= i /
10.0] = 0.0
out_img = img_bgr.copy()
if self.mode == 'raw':
if self.raw_mode == 'rgb' or self.raw_mode == 'rgb-mask':
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size, out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
if self.raw_mode == 'rgb-mask':
out_img = np.concatenate(
[out_img,
np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1)
if self.raw_mode == 'mask-only':
out_img = img_face_mask_aaa
if self.raw_mode == 'predicted-only':
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.zeros(out_img.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
elif ('seamless' not in self.mode) or (img_face_seamless_mask_aaa
is not None):
#averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
ar = []
for i in range(1, 10):
maxregion = np.argwhere(img_face_mask_aaa > i / 10.0)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
maskx = (minx + (lenx / 2))
masky = (miny + (leny / 2))
if lenx >= 4 and leny >= 4:
ar += [[lenx, leny, maskx, masky]]
if len(ar) > 0:
lenx, leny, maskx, masky = np.mean(ar, axis=0)
if debug:
io.log_info("lenx/leny:(%d/%d) maskx/masky:(%f/%f)" %
(lenx, leny, maskx, masky))
maskx = int(maskx)
masky = int(masky)
lowest_len = min(lenx, leny)
if debug:
io.log_info("lowest_len = %f" % (lowest_len))
img_mask_blurry_aaa = img_face_mask_aaa
if self.erode_mask_modifier != 0:
ero = int(lowest_len *
(0.126 - lowest_len * 0.00004551365) * 0.01 *
self.erode_mask_modifier)
if debug:
io.log_info("erode_size = %d" % (ero))
if ero > 0:
img_mask_blurry_aaa = cv2.erode(
img_mask_blurry_aaa,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
(ero, ero)),
iterations=1)
elif ero < 0:
img_mask_blurry_aaa = cv2.dilate(
img_mask_blurry_aaa,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
if self.seamless_erode_mask_modifier != 0:
ero = int(lowest_len *
(0.126 - lowest_len * 0.00004551365) * 0.01 *
self.seamless_erode_mask_modifier)
if debug:
io.log_info("seamless_erode_size = %d" % (ero))
if ero > 0:
img_face_seamless_mask_aaa = cv2.erode(
img_face_seamless_mask_aaa,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
(ero, ero)),
iterations=1)
elif ero < 0:
img_face_seamless_mask_aaa = cv2.dilate(
img_face_seamless_mask_aaa,
cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
img_face_seamless_mask_aaa = np.clip(
img_face_seamless_mask_aaa, 0, 1)
if self.clip_hborder_mask_per > 0: #clip hborder before blur
prd_hborder_rect_mask_a = np.ones(prd_face_mask_a.shape,
dtype=np.float32)
prd_border_size = int(prd_hborder_rect_mask_a.shape[1] *
self.clip_hborder_mask_per)
prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0
prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0
prd_hborder_rect_mask_a = np.expand_dims(
cv2.blur(prd_hborder_rect_mask_a,
(prd_border_size, prd_border_size)), -1)
img_prd_hborder_rect_mask_a = cv2.warpAffine(
prd_hborder_rect_mask_a, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4)
img_prd_hborder_rect_mask_a = np.expand_dims(
img_prd_hborder_rect_mask_a, -1)
img_mask_blurry_aaa *= img_prd_hborder_rect_mask_a
img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0)
if debug:
debugs += [img_mask_blurry_aaa.copy()]
if self.blur_mask_modifier > 0:
blur = int(lowest_len * 0.10 * 0.01 *
self.blur_mask_modifier)
if debug:
io.log_info("blur_size = %d" % (blur))
if blur > 0:
img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa,
(blur, blur))
img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0)
if debug:
debugs += [img_mask_blurry_aaa.copy()]
if self.color_transfer_mode is not None:
if self.color_transfer_mode == 'rct':
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
prd_face_bgr = image_utils.reinhard_color_transfer(
np.clip((prd_face_bgr * 255).astype(np.uint8), 0,
255),
np.clip((dst_face_bgr * 255).astype(np.uint8), 0,
255),
source_mask=prd_face_mask_a,
target_mask=prd_face_mask_a)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
elif self.color_transfer_mode == 'lct':
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
prd_face_bgr = image_utils.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
if self.mode == 'hist-match-bw':
prd_face_bgr = cv2.cvtColor(prd_face_bgr,
cv2.COLOR_BGR2GRAY)
prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1),
(3, ), -1)
if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
if debug:
debugs += [
cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
]
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if self.masked_hist_match:
hist_mask_a *= prd_face_mask_a
hist_match_1 = prd_face_bgr * hist_mask_a + (
1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + (
1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = image_utils.color_hist_match(
hist_match_1, hist_match_2, self.hist_match_threshold)
if self.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size, out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
if debug:
debugs += [out_img.copy()]
if self.mode == 'overlay':
pass
if 'seamless' in self.mode:
try:
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8),
(img_bgr * 255).astype(np.uint8),
(img_face_seamless_mask_aaa * 255).astype(
np.uint8), (maskx, masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
if debug:
debugs += [out_img.copy()]
out_img = np.clip(
img_bgr * (1 - img_mask_blurry_aaa) +
(out_img * img_mask_blurry_aaa), 0, 1.0)
if self.mode == 'seamless-hist-match':
out_face_bgr = cv2.warpAffine(
out_img, face_mat,
(self.output_size, self.output_size))
new_out_face_bgr = image_utils.color_hist_match(
out_face_bgr, dst_face_bgr, self.hist_match_threshold)
new_out = cv2.warpAffine(
new_out_face_bgr, face_mat, img_size, img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_mask_blurry_aaa) +
(new_out * img_mask_blurry_aaa), 0, 1.0)
if self.final_image_color_degrade_power != 0:
if debug:
debugs += [out_img.copy()]
out_img_reduced = image_utils.reduce_colors(out_img, 256)
if self.final_image_color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = self.final_image_color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
if self.alpha:
out_img = np.concatenate([
out_img,
np.expand_dims(img_mask_blurry_aaa[:, :, 0], -1)
], -1)
if self.over_res != 1:
out_img = cv2.resize(out_img, (img_bgr.shape[1] // self.over_res,
img_bgr.shape[0] // self.over_res))
out_img = np.clip(out_img, 0.0, 1.0)
if debug:
debugs += [out_img.copy()]
return debugs if debug else out_img
def process(samples,
sample_process_options,
output_sample_types,
debug,
ct_sample=None):
SPST = SampleProcessor.SampleType
SPCT = SampleProcessor.ChannelType
SPFMT = SampleProcessor.FaceMaskType
sample_rnd_seed = np.random.randint(0x80000000)
outputs = []
for sample in samples:
sample_bgr = sample.load_bgr()
ct_sample_bgr = None
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range)
outputs_sample = []
for opts in output_sample_types:
sample_type = opts.get('sample_type', SPST.NONE)
channel_type = opts.get('channel_type', SPCT.NONE)
resolution = opts.get('resolution', 0)
warp = opts.get('warp', False)
transform = opts.get('transform', False)
motion_blur = opts.get('motion_blur', None)
gaussian_blur = opts.get('gaussian_blur', None)
normalize_tanh = opts.get('normalize_tanh', False)
ct_mode = opts.get('ct_mode', 'None')
data_format = opts.get('data_format', 'NHWC')
if sample_type == SPST.FACE_IMAGE or sample_type == SPST.FACE_MASK:
if not is_face_sample:
raise ValueError(
"face_samples should be provided for sample_type FACE_*"
)
if is_face_sample:
face_type = opts.get('face_type', None)
face_mask_type = opts.get('face_mask_type', SPFMT.NONE)
if face_type is None:
raise ValueError(
"face_type must be defined for face samples")
if face_type > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
% (sample.filename, sample.face_type, target_ft))
if sample_type == SPST.FACE_IMAGE or sample_type == SPST.FACE_MASK:
if sample_type == SPST.FACE_MASK:
if face_mask_type == SPFMT.ALL_HULL or \
face_mask_type == SPFMT.EYES_HULL or \
face_mask_type == SPFMT.ALL_EYES_HULL:
if face_mask_type == SPFMT.ALL_HULL or \
face_mask_type == SPFMT.ALL_EYES_HULL:
if sample.eyebrows_expand_mod is not None:
all_mask = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.
eyebrows_expand_mod)
else:
all_mask = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape, sample.landmarks)
all_mask = np.clip(all_mask, 0, 1)
if face_mask_type == SPFMT.EYES_HULL or \
face_mask_type == SPFMT.ALL_EYES_HULL:
eyes_mask = LandmarksProcessor.get_image_eye_mask(
sample_bgr.shape, sample.landmarks)
eyes_mask = np.clip(eyes_mask, 0, 1)
if face_mask_type == SPFMT.ALL_HULL:
img = all_mask
elif face_mask_type == SPFMT.EYES_HULL:
img = eyes_mask
elif face_mask_type == SPFMT.ALL_EYES_HULL:
img = all_mask + eyes_mask
elif face_mask_type == SPFMT.STRUCT:
if sample.eyebrows_expand_mod is not None:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.
eyebrows_expand_mod)
else:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape, sample.landmarks)
if sample.ie_polys is not None:
sample.ie_polys.overlay_mask(img)
if sample.face_type == FaceType.MARK_ONLY:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0], face_type)
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_LINEAR)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=False,
cv2_inter=cv2.INTER_LINEAR)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_LINEAR)[..., None]
else:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, face_type)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=False,
cv2_inter=cv2.INTER_LINEAR)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_CONSTANT,
flags=cv2.INTER_LINEAR)[..., None]
if channel_type == SPCT.G:
out_sample = img.astype(np.float32)
else:
raise ValueError(
"only channel_type.G supported for the mask")
elif sample_type == SPST.FACE_IMAGE:
img = sample_bgr
if motion_blur is not None:
chance, mb_max_size = motion_blur
chance = np.clip(chance, 0, 100)
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
mblur_rnd_chance = l_rnd_state.randint(100)
mblur_rnd_kernel = l_rnd_state.randint(
mb_max_size) + 1
mblur_rnd_deg = l_rnd_state.randint(360)
if mblur_rnd_chance < chance:
img = imagelib.LinearMotionBlur(
img, mblur_rnd_kernel, mblur_rnd_deg)
if gaussian_blur is not None:
chance, kernel_max_size = gaussian_blur
chance = np.clip(chance, 0, 100)
l_rnd_state = np.random.RandomState(
sample_rnd_seed + 1)
gblur_rnd_chance = l_rnd_state.randint(100)
gblur_rnd_kernel = l_rnd_state.randint(
kernel_max_size) * 2 + 1
if gblur_rnd_chance < chance:
img = cv2.GaussianBlur(
img, (gblur_rnd_kernel, ) * 2, 0)
if sample.face_type == FaceType.MARK_ONLY:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0], face_type)
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=True)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
else:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, face_type)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=True)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_REPLICATE,
flags=cv2.INTER_CUBIC)
img = np.clip(img.astype(np.float32), 0, 1)
# Apply random color transfer
if ct_mode is not None and ct_sample is not None:
if ct_sample_bgr is None:
ct_sample_bgr = ct_sample.load_bgr()
img = imagelib.color_transfer(
ct_mode, img,
cv2.resize(ct_sample_bgr,
(resolution, resolution),
cv2.INTER_LINEAR))
# Transform from BGR to desired channel_type
if channel_type == SPCT.BGR:
out_sample = img
elif channel_type == SPCT.BGR_SHUFFLE:
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
out_sample = np.take(img,
l_rnd_state.permutation(
img.shape[-1]),
axis=-1)
elif channel_type == SPCT.BGR_RANDOM_HSV_SHIFT:
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
h, s, v = cv2.split(hsv)
h = (h + l_rnd_state.randint(360)) % 360
s = np.clip(s + l_rnd_state.random() - 0.5, 0, 1)
v = np.clip(v + l_rnd_state.random() - 0.5, 0, 1)
hsv = cv2.merge([h, s, v])
out_sample = np.clip(
cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0, 1)
elif channel_type == SPCT.BGR_RANDOM_RGB_LEVELS:
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
np_rnd = l_rnd_state.rand
inBlack = np.array([
np_rnd() * 0.25,
np_rnd() * 0.25,
np_rnd() * 0.25
],
dtype=np.float32)
inWhite = np.array([
1.0 - np_rnd() * 0.25, 1.0 - np_rnd() * 0.25,
1.0 - np_rnd() * 0.25
],
dtype=np.float32)
inGamma = np.array([
0.5 + np_rnd(), 0.5 + np_rnd(), 0.5 + np_rnd()
],
dtype=np.float32)
outBlack = np.array([0.0, 0.0, 0.0],
dtype=np.float32)
outWhite = np.array([1.0, 1.0, 1.0],
dtype=np.float32)
out_sample = np.clip(
(img - inBlack) / (inWhite - inBlack), 0, 1)
out_sample = (out_sample**(1 / inGamma)) * (
outWhite - outBlack) + outBlack
out_sample = np.clip(out_sample, 0, 1)
elif channel_type == SPCT.G:
out_sample = cv2.cvtColor(img,
cv2.COLOR_BGR2GRAY)[...,
None]
elif channel_type == SPCT.GGG:
out_sample = np.repeat(
np.expand_dims(
cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1)
# Final transformations
if not debug:
if normalize_tanh:
out_sample = np.clip(out_sample * 2.0 - 1.0, -1.0,
1.0)
if data_format == "NCHW":
out_sample = np.transpose(out_sample, (2, 0, 1))
#else:
# img = imagelib.warp_by_params (params, img, warp, transform, can_flip=True, border_replicate=True)
# img = cv2.resize( img, (resolution,resolution), cv2.INTER_CUBIC )
elif sample_type == SPST.LANDMARKS_ARRAY:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
out_sample = l
elif sample_type == SPST.PITCH_YAW_ROLL or sample_type == SPST.PITCH_YAW_ROLL_SIGMOID:
pitch_yaw_roll = sample.get_pitch_yaw_roll()
if params['flip']:
yaw = -yaw
if sample_type == SPST.PITCH_YAW_ROLL_SIGMOID:
pitch = np.clip((pitch / math.pi) / 2.0 + 0.5, 0, 1)
yaw = np.clip((yaw / math.pi) / 2.0 + 0.5, 0, 1)
roll = np.clip((roll / math.pi) / 2.0 + 0.5, 0, 1)
out_sample = (pitch, yaw, roll)
else:
raise ValueError('expected sample_type')
outputs_sample.append(out_sample)
outputs += [outputs_sample]
return outputs
def MergeMaskedFace(predictor_func, predictor_input_shape, cfg, frame_info,
img_bgr_uint8, img_bgr, img_face_landmarks):
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
if cfg.mode == 'original':
return img_bgr, img_face_mask_a
out_img = img_bgr.copy()
out_merging_mask_a = None
input_size = predictor_input_shape[0]
mask_subres_size = input_size * 4
output_size = input_size
if cfg.super_resolution_power != 0:
output_size *= 4
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
if mask_subres_size == output_size:
face_mask_output_mat = face_output_mat
else:
face_mask_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
mask_subres_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_bgr = np.clip(dst_face_bgr, 0, 1)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size))
predicted = predictor_func(predictor_input_bgr)
if isinstance(predicted, tuple):
#merger return bgr,mask
prd_face_bgr = np.clip(predicted[0], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0)
predictor_masked = True
else:
#merger return bgr only, using dst mask
prd_face_bgr = np.clip(predicted, 0, 1.0)
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(input_size, input_size))
predictor_masked = False
if cfg.super_resolution_power != 0:
prd_face_bgr_enhanced = cfg.superres_func(prd_face_bgr)
mod = cfg.super_resolution_power / 100.0
prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * (
1.0 - mod) + prd_face_bgr_enhanced * mod
prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
if cfg.super_resolution_power != 0:
if predictor_masked:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 2: #dst
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
elif cfg.mask_mode >= 3 and cfg.mask_mode <= 8:
if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6:
prd_face_fanseg_bgr = cv2.resize(prd_face_bgr,
(cfg.fanseg_input_size, ) * 2)
prd_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, prd_face_fanseg_bgr)
FAN_prd_face_mask_a_0 = cv2.resize(prd_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode >= 4 and cfg.mask_mode <= 7:
full_face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=FaceType.FULL)
dst_face_fanseg_bgr = cv2.warpAffine(img_bgr,
full_face_fanseg_mat,
(cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, dst_face_fanseg_bgr)
if cfg.face_type == FaceType.FULL:
FAN_dst_face_mask_a_0 = cv2.resize(dst_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=cfg.face_type)
fanseg_rect_corner_pts = np.array(
[[0, 0], [cfg.fanseg_input_size - 1, 0],
[0, cfg.fanseg_input_size - 1]],
dtype=np.float32)
a = LandmarksProcessor.transform_points(fanseg_rect_corner_pts,
face_fanseg_mat,
invert=True)
b = LandmarksProcessor.transform_points(
a, full_face_fanseg_mat)
m = cv2.getAffineTransform(b, fanseg_rect_corner_pts)
FAN_dst_face_mask_a_0 = cv2.warpAffine(
dst_face_fanseg_mask,
m, (cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
FAN_dst_face_mask_a_0 = cv2.resize(FAN_dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 3: #FAN-prd
prd_face_mask_a_0 = FAN_prd_face_mask_a_0
elif cfg.mask_mode == 4: #FAN-dst
prd_face_mask_a_0 = FAN_dst_face_mask_a_0
elif cfg.mask_mode == 5:
prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 6:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 7:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0
prd_face_mask_a_0[prd_face_mask_a_0 < (1.0 /
255.0)] = 0.0 # get rid of noise
# resize to mask_subres_size
if prd_face_mask_a_0.shape[0] != mask_subres_size:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(mask_subres_size, mask_subres_size),
cv2.INTER_CUBIC)
# process mask in local predicted space
if 'raw' not in cfg.mode:
# add zero pad
prd_face_mask_a_0 = np.pad(prd_face_mask_a_0, input_size)
ero = cfg.erode_mask_modifier
blur = cfg.blur_mask_modifier
if ero > 0:
prd_face_mask_a_0 = cv2.erode(prd_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
prd_face_mask_a_0 = cv2.dilate(prd_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
# clip eroded/dilated mask in actual predict area
# pad with half blur size in order to accuratelly fade to zero at the boundary
clip_size = input_size + blur // 2
prd_face_mask_a_0[:clip_size, :] = 0
prd_face_mask_a_0[-clip_size:, :] = 0
prd_face_mask_a_0[:, :clip_size] = 0
prd_face_mask_a_0[:, -clip_size:] = 0
if blur > 0:
blur = blur + (1 - blur % 2)
prd_face_mask_a_0 = cv2.GaussianBlur(prd_face_mask_a_0,
(blur, blur), 0)
prd_face_mask_a_0 = prd_face_mask_a_0[input_size:-input_size,
input_size:-input_size]
prd_face_mask_a_0 = np.clip(prd_face_mask_a_0, 0, 1)
img_face_mask_a = cv2.warpAffine(prd_face_mask_a_0,
face_mask_output_mat,
img_size,
np.zeros(img_bgr.shape[0:2],
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)[..., None]
img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0)
img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise
if prd_face_mask_a_0.shape[0] != output_size:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
prd_face_mask_a = prd_face_mask_a_0[..., None]
prd_face_mask_area_a = prd_face_mask_a.copy()
prd_face_mask_area_a[prd_face_mask_area_a > 0] = 1.0
if 'raw' in cfg.mode:
if cfg.mode == 'raw-rgb':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_merging_mask_a = img_face_mask_a
out_img = np.clip(out_img, 0.0, 1.0)
else:
#averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
ar = []
for i in range(1, 10):
maxregion = np.argwhere(img_face_mask_a > i / 10.0)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
ar += [[lenx, leny]]
if len(ar) > 0:
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1: #rct
prd_face_bgr = imagelib.reinhard_color_transfer(
np.clip(prd_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8),
np.clip(dst_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8),
)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
prd_face_bgr = imagelib.color_transfer_sot(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
prd_face_bgr = imagelib.color_transfer_mix(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
if cfg.mode == 'hist-match':
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= prd_face_mask_area_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2,
cfg.hist_match_threshold).astype(dtype=np.float32)
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_a = None
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_a.copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[img_face_seamless_mask_a <= i /
10.0] = 0.0
break
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
if 'seamless' in cfg.mode:
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
out_img = img_bgr * (1 - img_face_mask_a) + (out_img *
img_face_mask_a)
out_face_bgr = cv2.warpAffine(out_img,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
out_face_bgr = imagelib.reinhard_color_transfer(
np.clip(out_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8),
np.clip(dst_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8))
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
out_face_bgr = imagelib.color_transfer_sot(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
out_face_bgr = imagelib.color_transfer_mix(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr, cfg.hist_match_threshold)
cfg_mp = cfg.motion_blur_power / 100.0
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_power != 0:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.blursharpen_amount != 0:
out_face_bgr = cfg.blursharpen_func(out_face_bgr,
cfg.sharpen_mode, 3,
cfg.blursharpen_amount)
if cfg.image_denoise_power != 0:
n = cfg.image_denoise_power
while n > 0:
img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
if int(n / 100) != 0:
img_bgr = img_bgr_denoised
else:
pass_power = (n % 100) / 100.0
img_bgr = img_bgr * (
1.0 - pass_power) + img_bgr_denoised * pass_power
n = max(n - 10, 0)
if cfg.bicubic_degrade_power != 0:
p = 1.0 - cfg.bicubic_degrade_power / 101.0
img_bgr_downscaled = cv2.resize(
img_bgr, (int(img_size[0] * p), int(img_size[1] * p)),
cv2.INTER_CUBIC)
img_bgr = cv2.resize(img_bgr_downscaled, img_size,
cv2.INTER_CUBIC)
new_out = cv2.warpAffine(out_face_bgr, face_mat, img_size,
img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_face_mask_a) + (new_out * img_face_mask_a),
0, 1.0)
if cfg.color_degrade_power != 0:
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
out_merging_mask_a = img_face_mask_a
return out_img, out_merging_mask_a
def convert_face (self, img_bgr, img_face_landmarks, debug):
if debug:
debugs = [img_bgr.copy()]
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask (img_bgr, img_face_landmarks)
face_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, self.output_size, face_type=self.face_type)
dst_face_bgr = cv2.warpAffine( img_bgr , face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4 )
dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4 )
predictor_input_bgr = cv2.resize (dst_face_bgr, (self.predictor_input_size,self.predictor_input_size))
predictor_input_mask_a_0 = cv2.resize (dst_face_mask_a_0, (self.predictor_input_size,self.predictor_input_size))
predictor_input_mask_a = np.expand_dims (predictor_input_mask_a_0, -1)
predicted_bgra = self.predictor ( np.concatenate( (predictor_input_bgr, predictor_input_mask_a), -1) )
prd_face_bgr = np.clip (predicted_bgra[:,:,0:3], 0, 1.0 )
prd_face_mask_a_0 = np.clip (predicted_bgra[:,:,3], 0.0, 1.0)
prd_face_mask_a_0[ prd_face_mask_a_0 < 0.001 ] = 0.0
prd_face_mask_a = np.expand_dims (prd_face_mask_a_0, axis=-1)
prd_face_mask_aaa = np.repeat (prd_face_mask_a, (3,), axis=-1)
img_prd_face_mask_aaa = cv2.warpAffine( prd_face_mask_aaa, face_mat, img_size, np.zeros(img_bgr.shape, dtype=float), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4 )
img_prd_face_mask_aaa = np.clip (img_prd_face_mask_aaa, 0.0, 1.0)
img_face_mask_aaa = img_prd_face_mask_aaa
if debug:
debugs += [img_face_mask_aaa.copy()]
img_face_mask_aaa [ img_face_mask_aaa <= 0.1 ] = 0.0
img_face_mask_flatten_aaa = img_face_mask_aaa.copy()
img_face_mask_flatten_aaa[img_face_mask_flatten_aaa > 0.9] = 1.0
maxregion = np.argwhere(img_face_mask_flatten_aaa==1.0)
out_img = img_bgr.copy()
if maxregion.size != 0:
miny,minx = maxregion.min(axis=0)[:2]
maxy,maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
masky = int(minx+(lenx//2))
maskx = int(miny+(leny//2))
lowest_len = min (lenx, leny)
if debug:
print ("lowest_len = %f" % (lowest_len) )
ero = int( lowest_len * ( 0.126 - lowest_len * 0.00004551365 ) * 0.01*self.erode_mask_modifier )
blur = int( lowest_len * 0.10 * 0.01*self.blur_mask_modifier )
if debug:
print ("ero = %d, blur = %d" % (ero, blur) )
img_mask_blurry_aaa = img_face_mask_aaa
if self.erode_mask:
if ero > 0:
img_mask_blurry_aaa = cv2.erode(img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 )
elif ero < 0:
img_mask_blurry_aaa = cv2.dilate(img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(-ero,-ero)), iterations = 1 )
if self.blur_mask and blur > 0:
img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa, (blur, blur) )
img_mask_blurry_aaa = np.clip( img_mask_blurry_aaa, 0, 1.0 )
if self.clip_border_mask_per > 0:
prd_border_rect_mask_a = np.ones ( prd_face_mask_a.shape, dtype=prd_face_mask_a.dtype)
prd_border_size = int ( prd_border_rect_mask_a.shape[1] * self.clip_border_mask_per )
prd_border_rect_mask_a[0:prd_border_size,:,:] = 0
prd_border_rect_mask_a[-prd_border_size:,:,:] = 0
prd_border_rect_mask_a[:,0:prd_border_size,:] = 0
prd_border_rect_mask_a[:,-prd_border_size:,:] = 0
prd_border_rect_mask_a = np.expand_dims(cv2.blur(prd_border_rect_mask_a, (prd_border_size, prd_border_size) ),-1)
if self.mode == 'hist-match-bw':
prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY)
prd_face_bgr = np.repeat( np.expand_dims (prd_face_bgr, -1), (3,), -1 )
if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
if debug:
debugs += [ cv2.warpAffine( prd_face_bgr, face_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ) ]
hist_mask_a = np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=prd_face_bgr.dtype)
if self.masked_hist_match:
hist_mask_a *= prd_face_mask_a
new_prd_face_bgr = image_utils.color_hist_match(prd_face_bgr*hist_mask_a, dst_face_bgr*hist_mask_a )
prd_face_bgr = new_prd_face_bgr
if self.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(np.float32)
out_img = cv2.warpAffine( prd_face_bgr, face_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT )
if debug:
debugs += [out_img.copy()]
debugs += [img_mask_blurry_aaa.copy()]
if self.mode == 'seamless' or self.mode == 'seamless-hist-match':
out_img = np.clip( img_bgr*(1-img_face_mask_aaa) + (out_img*img_face_mask_aaa) , 0, 1.0 )
if debug:
debugs += [out_img.copy()]
out_img = cv2.seamlessClone( (out_img*255).astype(np.uint8), (img_bgr*255).astype(np.uint8), (img_face_mask_flatten_aaa*255).astype(np.uint8), (masky,maskx) , cv2.NORMAL_CLONE )
out_img = out_img.astype(np.float32) / 255.0
if debug:
debugs += [out_img.copy()]
if self.clip_border_mask_per > 0:
img_prd_border_rect_mask_a = cv2.warpAffine( prd_border_rect_mask_a, face_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT )
img_prd_border_rect_mask_a = np.expand_dims (img_prd_border_rect_mask_a, -1)
out_img = out_img * img_prd_border_rect_mask_a + img_bgr * (1.0 - img_prd_border_rect_mask_a)
img_mask_blurry_aaa *= img_prd_border_rect_mask_a
out_img = np.clip( img_bgr*(1-img_mask_blurry_aaa) + (out_img*img_mask_blurry_aaa) , 0, 1.0 )
if self.mode == 'seamless-hist-match':
out_face_bgr = cv2.warpAffine( out_img, face_mat, (self.output_size, self.output_size) )
new_out_face_bgr = image_utils.color_hist_match(out_face_bgr, dst_face_bgr )
new_out = cv2.warpAffine( new_out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT )
out_img = np.clip( img_bgr*(1-img_mask_blurry_aaa) + (new_out*img_mask_blurry_aaa) , 0, 1.0 )
if self.transfercolor: #making transfer color from original DST image to fake
from skimage import io, color
lab_clr = color.rgb2lab(img_bgr) #original DST, converting RGB to LAB color space
lab_bw = color.rgb2lab(out_img) #fake, converting RGB to LAB color space
tmp_channel, a_channel, b_channel = cv2.split(lab_clr) #taking color channel A and B from original dst image
l_channel, tmp2_channel, tmp3_channel = cv2.split(lab_bw) #taking lightness channel L from merged fake
img_LAB = cv2.merge((l_channel,a_channel, b_channel)) #merging light and color
out_img = color.lab2rgb(img_LAB) #converting LAB to RGB
if self.alpha:
new_image = out_img.copy()
new_image = (new_image*255).astype(np.uint8) #convert image to int
b_channel, g_channel, r_channel = cv2.split(new_image) #splitting RGB
alpha_channel = img_mask_blurry_aaa.copy() #making copy of alpha channel
alpha_channel = (alpha_channel*255).astype(np.uint8)
alpha_channel, tmp2, tmp3 = cv2.split(alpha_channel) #splitting alpha to three channels, they all same in original alpha channel, we need just one
out_img = cv2.merge((b_channel,g_channel, r_channel, alpha_channel)) #mergin RGB with alpha
out_img = out_img.astype(np.float32) / 255.0
if debug:
debugs += [out_img.copy()]
return debugs if debug else out_img
def process(sample, sample_process_options, output_sample_types, debug):
SPTF = SampleProcessor.TypeFlags
sample_bgr = sample.load_bgr()
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
close_sample = sample.close_target_list[np.random.randint(
0, len(sample.close_target_list)
)] if sample.close_target_list is not None else None
close_sample_bgr = close_sample.load_bgr(
) if close_sample is not None else None
if debug and close_sample_bgr is not None:
LandmarksProcessor.draw_landmarks(close_sample_bgr,
close_sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range)
images = [[None] * 3 for _ in range(30)]
sample_rnd_seed = np.random.randint(0x80000000)
outputs = []
for sample_type in output_sample_types:
f = sample_type[0]
size = sample_type[1]
random_sub_size = 0 if len(sample_type) < 3 else min(
sample_type[2], size)
if f & SPTF.SOURCE != 0:
img_type = 0
elif f & SPTF.WARPED != 0:
img_type = 1
elif f & SPTF.WARPED_TRANSFORMED != 0:
img_type = 2
elif f & SPTF.TRANSFORMED != 0:
img_type = 3
elif f & SPTF.LANDMARKS_ARRAY != 0:
img_type = 4
else:
raise ValueError('expected SampleTypeFlags type')
if f & SPTF.RANDOM_CLOSE != 0:
img_type += 10
elif f & SPTF.MORPH_TO_RANDOM_CLOSE != 0:
img_type += 20
face_mask_type = 0
if f & SPTF.FACE_MASK_FULL != 0:
face_mask_type = 1
elif f & SPTF.FACE_MASK_EYES != 0:
face_mask_type = 2
target_face_type = -1
if f & SPTF.FACE_TYPE_HALF != 0:
target_face_type = FaceType.HALF
elif f & SPTF.FACE_TYPE_FULL != 0:
target_face_type = FaceType.FULL
elif f & SPTF.FACE_TYPE_HEAD != 0:
target_face_type = FaceType.HEAD
elif f & SPTF.FACE_TYPE_AVATAR != 0:
target_face_type = FaceType.AVATAR
apply_motion_blur = f & SPTF.OPT_APPLY_MOTION_BLUR != 0
if img_type == 4:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
img = l
else:
if images[img_type][face_mask_type] is None:
if img_type >= 10 and img_type <= 19: #RANDOM_CLOSE
img_type -= 10
img = close_sample_bgr
cur_sample = close_sample
elif img_type >= 20 and img_type <= 29: #MORPH_TO_RANDOM_CLOSE
img_type -= 20
res = sample.shape[0]
s_landmarks = sample.landmarks.copy()
d_landmarks = close_sample.landmarks.copy()
idxs = list(range(len(s_landmarks)))
#remove landmarks near boundaries
for i in idxs[:]:
s_l = s_landmarks[i]
d_l = d_landmarks[i]
if s_l[0] < 5 or s_l[1] < 5 or s_l[0] >= res-5 or s_l[1] >= res-5 or \
d_l[0] < 5 or d_l[1] < 5 or d_l[0] >= res-5 or d_l[1] >= res-5:
idxs.remove(i)
#remove landmarks that close to each other in 5 dist
for landmarks in [s_landmarks, d_landmarks]:
for i in idxs[:]:
s_l = landmarks[i]
for j in idxs[:]:
if i == j:
continue
s_l_2 = landmarks[j]
diff_l = np.abs(s_l - s_l_2)
if np.sqrt(diff_l.dot(diff_l)) < 5:
idxs.remove(i)
break
s_landmarks = s_landmarks[idxs]
d_landmarks = d_landmarks[idxs]
s_landmarks = np.concatenate([
s_landmarks,
[[0, 0], [res // 2, 0], [res - 1, 0],
[0, res // 2], [res - 1, res // 2], [0, res - 1],
[res // 2, res - 1], [res - 1, res - 1]]
])
d_landmarks = np.concatenate([
d_landmarks,
[[0, 0], [res // 2, 0], [res - 1, 0],
[0, res // 2], [res - 1, res // 2], [0, res - 1],
[res // 2, res - 1], [res - 1, res - 1]]
])
img = imagelib.morph_by_points(sample_bgr, s_landmarks,
d_landmarks)
cur_sample = close_sample
else:
img = sample_bgr
cur_sample = sample
if is_face_sample:
if apply_motion_blur and sample_process_options.motion_blur is not None:
chance, mb_range = sample_process_options.motion_blur
if np.random.randint(100) < chance:
dim = mb_range[np.random.randint(
len(mb_range))]
img = imagelib.LinearMotionBlur(
img, dim, np.random.randint(180))
if face_mask_type == 1:
mask = cur_sample.load_fanseg_mask(
) #using fanseg_mask if exist
if mask is None:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape, cur_sample.landmarks)
if cur_sample.ie_polys is not None:
cur_sample.ie_polys.overlay_mask(mask)
img = np.concatenate((img, mask), -1)
elif face_mask_type == 2:
mask = LandmarksProcessor.get_image_eye_mask(
img.shape, cur_sample.landmarks)
mask = np.expand_dims(
cv2.blur(mask, (w // 32, w // 32)), -1)
mask[mask > 0.0] = 1.0
img = np.concatenate((img, mask), -1)
images[img_type][face_mask_type] = imagelib.warp_by_params(
params, img, (img_type == 1 or img_type == 2),
(img_type == 2 or img_type == 3), img_type != 0,
face_mask_type == 0)
img = images[img_type][face_mask_type]
if is_face_sample and target_face_type != -1:
if target_face_type > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
% (sample.filename, sample.face_type,
target_face_type))
img = cv2.warpAffine(img,
LandmarksProcessor.get_transform_mat(
sample.landmarks, size,
target_face_type), (size, size),
flags=cv2.INTER_CUBIC)
else:
img = cv2.resize(img, (size, size), cv2.INTER_CUBIC)
if random_sub_size != 0:
sub_size = size - random_sub_size
rnd_state = np.random.RandomState(sample_rnd_seed +
random_sub_size)
start_x = rnd_state.randint(sub_size + 1)
start_y = rnd_state.randint(sub_size + 1)
img = img[start_y:start_y + sub_size,
start_x:start_x + sub_size, :]
img_bgr = img[..., 0:3]
img_mask = img[..., 3:4]
if f & SPTF.MODE_BGR != 0:
img = img_bgr
elif f & SPTF.MODE_BGR_SHUFFLE != 0:
rnd_state = np.random.RandomState(sample_rnd_seed)
img_bgr = np.take(img_bgr,
rnd_state.permutation(img_bgr.shape[-1]),
axis=-1)
img = np.concatenate((img_bgr, img_mask), -1)
elif f & SPTF.MODE_G != 0:
img = np.concatenate((np.expand_dims(
cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY),
-1), img_mask), -1)
elif f & SPTF.MODE_GGG != 0:
img = np.concatenate((np.repeat(
np.expand_dims(
cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1), img_mask), -1)
elif is_face_sample and f & SPTF.MODE_M != 0:
if face_mask_type == 0:
raise ValueError('no face_mask_type defined')
img = img_mask
else:
raise ValueError('expected SampleTypeFlags mode')
if not debug:
if sample_process_options.normalize_tanh:
img = np.clip(img * 2.0 - 1.0, -1.0, 1.0)
else:
img = np.clip(img, 0.0, 1.0)
outputs.append(img)
if debug:
result = []
for output in outputs:
if output.shape[2] < 4:
result += [
output,
]
elif output.shape[2] == 4:
result += [
output[..., 0:3] * output[..., 3:4],
]
return result
else:
return outputs
def process(sample,
sample_process_options,
output_sample_types,
debug,
ct_sample=None):
SPTF = SampleProcessor.Types
sample_bgr = sample.load_bgr()
ct_sample_bgr = None
ct_sample_mask = None
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range)
cached_images = collections.defaultdict(dict)
sample_rnd_seed = np.random.randint(0x80000000)
SPTF_FACETYPE_TO_FACETYPE = {
SPTF.FACE_TYPE_HALF: FaceType.HALF,
SPTF.FACE_TYPE_FULL: FaceType.FULL,
SPTF.FACE_TYPE_HEAD: FaceType.HEAD,
SPTF.FACE_TYPE_FULL_NO_ALIGN: FaceType.FULL_NO_ALIGN
}
outputs = []
for opts in output_sample_types:
resolution = opts.get('resolution', 0)
types = opts.get('types', [])
random_sub_res = opts.get('random_sub_res', 0)
normalize_std_dev = opts.get('normalize_std_dev', False)
normalize_vgg = opts.get('normalize_vgg', False)
motion_blur = opts.get('motion_blur', None)
apply_ct = opts.get('apply_ct', False)
normalize_tanh = opts.get('normalize_tanh', False)
img_type = SPTF.NONE
target_face_type = SPTF.NONE
face_mask_type = SPTF.NONE
mode_type = SPTF.NONE
for t in types:
if t >= SPTF.IMG_TYPE_BEGIN and t < SPTF.IMG_TYPE_END:
img_type = t
elif t >= SPTF.FACE_TYPE_BEGIN and t < SPTF.FACE_TYPE_END:
target_face_type = t
elif t >= SPTF.MODE_BEGIN and t < SPTF.MODE_END:
mode_type = t
if img_type == SPTF.NONE:
raise ValueError('expected IMG_ type')
if img_type == SPTF.IMG_LANDMARKS_ARRAY:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
img = l
elif img_type == SPTF.IMG_PITCH_YAW_ROLL or img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch_yaw_roll = sample.pitch_yaw_roll
if pitch_yaw_roll is not None:
pitch, yaw, roll = pitch_yaw_roll
else:
pitch, yaw, roll = LandmarksProcessor.estimate_pitch_yaw_roll(
sample.landmarks)
if params['flip']:
yaw = -yaw
if img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch = (pitch + 1.0) / 2.0
yaw = (yaw + 1.0) / 2.0
roll = (roll + 1.0) / 2.0
img = (pitch, yaw, roll)
else:
if mode_type == SPTF.NONE:
raise ValueError('expected MODE_ type')
def do_transform(img, mask):
warp = (img_type == SPTF.IMG_WARPED
or img_type == SPTF.IMG_WARPED_TRANSFORMED)
transform = (img_type == SPTF.IMG_WARPED_TRANSFORMED
or img_type == SPTF.IMG_TRANSFORMED)
flip = img_type != SPTF.IMG_WARPED
img = imagelib.warp_by_params(params, img, warp, transform,
flip, True)
if mask is not None:
mask = imagelib.warp_by_params(params, mask, warp,
transform, flip, False)
if len(mask.shape) == 2:
mask = mask[..., np.newaxis]
img = np.concatenate((img, mask), -1)
return img
img = cached_images.get(img_type, None)
if img is None:
img = sample_bgr
mask = None
cur_sample = sample
if is_face_sample:
if motion_blur is not None:
chance, mb_range = motion_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
mb_range = [3, 5, 7,
9][:np.clip(mb_range, 0, 3) + 1]
dim = mb_range[np.random.randint(
len(mb_range))]
img = imagelib.LinearMotionBlur(
img, dim, np.random.randint(180))
mask = cur_sample.load_fanseg_mask(
) #using fanseg_mask if exist
if mask is None:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape, cur_sample.landmarks)
if cur_sample.ie_polys is not None:
cur_sample.ie_polys.overlay_mask(mask)
if sample.face_type == FaceType.MARK_ONLY:
if mask is not None:
img = np.concatenate((img, mask), -1)
else:
img = do_transform(img, mask)
cached_images[img_type] = img
if is_face_sample and target_face_type != SPTF.NONE:
ft = SPTF_FACETYPE_TO_FACETYPE[target_face_type]
if ft > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
% (sample.filename, sample.face_type, ft))
if sample.face_type == FaceType.MARK_ONLY:
img = cv2.warpAffine(
img,
LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0],
ft), (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
mask = img[..., 3:4] if img.shape[2] > 3 else None
img = img[..., 0:3]
img = do_transform(img, mask)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
else:
img = cv2.warpAffine(
img,
LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution,
ft), (resolution, resolution),
flags=cv2.INTER_CUBIC)
else:
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
if random_sub_res != 0:
sub_size = resolution - random_sub_res
rnd_state = np.random.RandomState(sample_rnd_seed +
random_sub_res)
start_x = rnd_state.randint(sub_size + 1)
start_y = rnd_state.randint(sub_size + 1)
img = img[start_y:start_y + sub_size,
start_x:start_x + sub_size, :]
img = np.clip(img, 0, 1)
img_bgr = img[..., 0:3]
img_mask = img[..., 3:4]
if apply_ct and ct_sample is not None:
if ct_sample_bgr is None:
ct_sample_bgr = ct_sample.load_bgr()
ct_sample_bgr_resized = cv2.resize(
ct_sample_bgr, (resolution, resolution),
cv2.INTER_LINEAR)
img_bgr = imagelib.linear_color_transfer(
img_bgr, ct_sample_bgr_resized)
img_bgr = np.clip(img_bgr, 0.0, 1.0)
if normalize_std_dev:
img_bgr = (img_bgr - img_bgr.mean((0, 1))) / img_bgr.std(
(0, 1))
elif normalize_vgg:
img_bgr = np.clip(img_bgr * 255, 0, 255)
img_bgr[:, :, 0] -= 103.939
img_bgr[:, :, 1] -= 116.779
img_bgr[:, :, 2] -= 123.68
if mode_type == SPTF.MODE_BGR:
img = img_bgr
elif mode_type == SPTF.MODE_BGR_SHUFFLE:
rnd_state = np.random.RandomState(sample_rnd_seed)
img = np.take(img_bgr,
rnd_state.permutation(img_bgr.shape[-1]),
axis=-1)
elif mode_type == SPTF.MODE_G:
img = np.concatenate((np.expand_dims(
cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY),
-1), img_mask), -1)
elif mode_type == SPTF.MODE_GGG:
img = np.concatenate((np.repeat(
np.expand_dims(
cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1), img_mask), -1)
elif mode_type == SPTF.MODE_M and is_face_sample:
img = img_mask
if not debug:
if normalize_tanh:
img = np.clip(img * 2.0 - 1.0, -1.0, 1.0)
else:
img = np.clip(img, 0.0, 1.0)
outputs.append(img)
if debug:
result = []
for output in outputs:
if output.shape[2] < 4:
result += [
output,
]
elif output.shape[2] == 4:
result += [
output[..., 0:3] * output[..., 3:4],
]
return result
else:
return outputs
def ConvertMaskedFace(cfg, frame_info, img_bgr_uint8, img_bgr,
img_face_landmarks):
#if debug:
# debugs = [img_bgr.copy()]
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
if cfg.mode == 'original':
if cfg.export_mask_alpha:
img_bgr = np.concatenate([img_bgr, img_face_mask_a], -1)
return img_bgr, img_face_mask_a
out_img = img_bgr.copy()
out_merging_mask = None
output_size = cfg.predictor_input_shape[0]
if cfg.super_resolution_mode != 0:
output_size *= 2
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
predictor_input_bgr = cv2.resize(dst_face_bgr,
cfg.predictor_input_shape[0:2])
if cfg.predictor_masked:
prd_face_bgr, prd_face_mask_a_0 = cfg.predictor_func(
predictor_input_bgr)
prd_face_bgr = np.clip(prd_face_bgr, 0, 1.0)
prd_face_mask_a_0 = np.clip(prd_face_mask_a_0, 0.0, 1.0)
else:
predicted = cfg.predictor_func(predictor_input_bgr)
prd_face_bgr = np.clip(predicted, 0, 1.0)
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
cfg.predictor_input_shape[0:2])
if cfg.super_resolution_mode:
#if debug:
# tmp = cv2.resize (prd_face_bgr, (output_size,output_size), cv2.INTER_CUBIC)
# debugs += [ np.clip( cv2.warpAffine( tmp, face_output_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
prd_face_bgr = cfg.superres_func(cfg.super_resolution_mode,
prd_face_bgr)
#if debug:
# debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
if cfg.predictor_masked:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 2: #dst
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
elif cfg.mask_mode >= 3 and cfg.mask_mode <= 7:
if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6:
prd_face_fanseg_bgr = cv2.resize(prd_face_bgr,
(cfg.fanseg_input_size, ) * 2)
prd_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, prd_face_fanseg_bgr)
FAN_prd_face_mask_a_0 = cv2.resize(prd_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode >= 4 or cfg.mask_mode <= 7:
full_face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=FaceType.FULL)
dst_face_fanseg_bgr = cv2.warpAffine(img_bgr,
full_face_fanseg_mat,
(cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, dst_face_fanseg_bgr)
if cfg.face_type == FaceType.FULL:
FAN_dst_face_mask_a_0 = cv2.resize(dst_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
elif cfg.face_type == FaceType.HALF:
half_face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=FaceType.HALF)
fanseg_rect_corner_pts = np.array(
[[0, 0], [cfg.fanseg_input_size - 1, 0],
[0, cfg.fanseg_input_size - 1]],
dtype=np.float32)
a = LandmarksProcessor.transform_points(fanseg_rect_corner_pts,
half_face_fanseg_mat,
invert=True)
b = LandmarksProcessor.transform_points(
a, full_face_fanseg_mat)
m = cv2.getAffineTransform(b, fanseg_rect_corner_pts)
FAN_dst_face_mask_a_0 = cv2.warpAffine(
dst_face_fanseg_mask,
m, (cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
FAN_dst_face_mask_a_0 = cv2.resize(FAN_dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
raise ValueError("cfg.face_type unsupported")
if cfg.mask_mode == 3: #FAN-prd
prd_face_mask_a_0 = FAN_prd_face_mask_a_0
elif cfg.mask_mode == 4: #FAN-dst
prd_face_mask_a_0 = FAN_dst_face_mask_a_0
elif cfg.mask_mode == 5:
prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 6:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 7:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0
prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0
prd_face_mask_a = prd_face_mask_a_0[..., np.newaxis]
prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1)
img_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa,
face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0.0, 1.0)
img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 #get rid of noise
#if debug:
# debugs += [img_face_mask_aaa.copy()]
if 'raw' in cfg.mode:
face_corner_pts = np.array(
[[0, 0], [output_size - 1, 0], [output_size - 1, output_size - 1],
[0, output_size - 1]],
dtype=np.float32)
square_mask = np.zeros(img_bgr.shape, dtype=np.float32)
cv2.fillConvexPoly(square_mask, \
LandmarksProcessor.transform_points (face_corner_pts, face_output_mat, invert=True ).astype(np.int), \
(1,1,1) )
if cfg.mode == 'raw-rgb':
out_merging_mask = square_mask
if cfg.mode == 'raw-rgb' or cfg.mode == 'raw-rgb-mask':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
if cfg.mode == 'raw-rgb-mask':
out_img = np.concatenate(
[out_img,
np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1)
out_merging_mask = square_mask
elif cfg.mode == 'raw-mask-only':
out_img = img_face_mask_aaa
out_merging_mask = img_face_mask_aaa
elif cfg.mode == 'raw-predicted-only':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_merging_mask = square_mask
out_img = np.clip(out_img, 0.0, 1.0)
else:
#averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
ar = []
for i in range(1, 10):
maxregion = np.argwhere(img_face_mask_aaa > i / 10.0)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
ar += [[lenx, leny]]
if len(ar) > 0:
lenx, leny = np.mean(ar, axis=0)
lowest_len = min(lenx, leny)
#if debug:
# io.log_info ("lenx/leny:(%d/%d) " % (lenx, leny ) )
# io.log_info ("lowest_len = %f" % (lowest_len) )
if cfg.erode_mask_modifier != 0:
ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) *
0.01 * cfg.erode_mask_modifier)
#if debug:
# io.log_info ("erode_size = %d" % (ero) )
if ero > 0:
img_face_mask_aaa = cv2.erode(img_face_mask_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(ero, ero)),
iterations=1)
elif ero < 0:
img_face_mask_aaa = cv2.dilate(img_face_mask_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
if cfg.clip_hborder_mask_per > 0: #clip hborder before blur
prd_hborder_rect_mask_a = np.ones(prd_face_mask_a.shape,
dtype=np.float32)
prd_border_size = int(prd_hborder_rect_mask_a.shape[1] *
cfg.clip_hborder_mask_per)
prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0
prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0
prd_hborder_rect_mask_a[-prd_border_size:, :, :] = 0
prd_hborder_rect_mask_a = np.expand_dims(
cv2.blur(prd_hborder_rect_mask_a,
(prd_border_size, prd_border_size)), -1)
img_prd_hborder_rect_mask_a = cv2.warpAffine(
prd_hborder_rect_mask_a, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
img_prd_hborder_rect_mask_a = np.expand_dims(
img_prd_hborder_rect_mask_a, -1)
img_face_mask_aaa *= img_prd_hborder_rect_mask_a
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0, 1.0)
#if debug:
# debugs += [img_face_mask_aaa.copy()]
if cfg.blur_mask_modifier > 0:
blur = int(lowest_len * 0.10 * 0.01 * cfg.blur_mask_modifier)
#if debug:
# io.log_info ("blur_size = %d" % (blur) )
if blur > 0:
img_face_mask_aaa = cv2.blur(img_face_mask_aaa,
(blur, blur))
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0, 1.0)
#if debug:
# debugs += [img_face_mask_aaa.copy()]
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
#if debug:
# debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
prd_face_bgr = imagelib.reinhard_color_transfer(
np.clip((prd_face_bgr * 255).astype(np.uint8), 0, 255),
np.clip((dst_face_bgr * 255).astype(np.uint8), 0, 255),
source_mask=prd_face_mask_a,
target_mask=prd_face_mask_a)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
#if debug:
# debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
elif cfg.color_transfer_mode == 2:
#if debug:
# debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
#if debug:
# debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
if cfg.mode == 'hist-match-bw':
prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY)
prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1),
(3, ), -1)
if cfg.mode == 'hist-match' or cfg.mode == 'hist-match-bw':
#if debug:
# debugs += [ cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ) ]
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= prd_face_mask_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2, cfg.hist_match_threshold)
#if cfg.masked_hist_match:
# prd_face_bgr -= white
if cfg.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
#if debug:
# debugs += [out_img.copy()]
if cfg.mode == 'overlay':
pass
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_a = None
img_face_mask_a = img_face_mask_aaa[..., 0:1]
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_aaa[...,
0:1].copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[img_face_seamless_mask_a <= i /
10.0] = 0.0
break
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
#if debug:
# debugs += [out_img.copy()]
out_img = img_bgr * (1 - img_face_mask_aaa) + (out_img *
img_face_mask_aaa)
out_face_bgr = cv2.warpAffine(out_img, face_mat,
(output_size, output_size))
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
#if debug:
# debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
face_mask_aaa = cv2.warpAffine(img_face_mask_aaa, face_mat,
(output_size, output_size))
out_face_bgr = imagelib.reinhard_color_transfer(
np.clip((out_face_bgr * 255).astype(np.uint8), 0, 255),
np.clip((dst_face_bgr * 255).astype(np.uint8), 0, 255),
source_mask=face_mask_aaa,
target_mask=face_mask_aaa)
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
#if debug:
# debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
elif cfg.color_transfer_mode == 2:
#if debug:
# debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
#if debug:
# debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ]
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr, cfg.hist_match_threshold)
cfg_mp = cfg.motion_blur_power / 100.0
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_mode:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.sharpen_mode != 0 and cfg.sharpen_amount != 0:
out_face_bgr = cfg.sharpen_func(out_face_bgr, cfg.sharpen_mode,
3, cfg.sharpen_amount)
new_out = cv2.warpAffine(out_face_bgr, face_mat, img_size,
img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_face_mask_aaa) +
(new_out * img_face_mask_aaa), 0, 1.0)
if cfg.color_degrade_power != 0:
#if debug:
# debugs += [out_img.copy()]
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
if cfg.export_mask_alpha:
out_img = np.concatenate(
[out_img, img_face_mask_aaa[:, :, 0:1]], -1)
out_merging_mask = img_face_mask_aaa
#if debug:
# debugs += [out_img.copy()]
return out_img, out_merging_mask
def convert_face(self, img_bgr, img_face_landmarks, debug):
if debug:
debugs = [img_bgr.copy()]
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
face_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, self.output_size, face_type=self.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
self.output_size,
face_type=self.face_type,
scale=self.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat,
(self.output_size, self.output_size),
flags=cv2.INTER_LANCZOS4)
dst_face_mask_a_0 = cv2.warpAffine(
img_face_mask_a,
face_mat, (self.output_size, self.output_size),
flags=cv2.INTER_LANCZOS4)
predictor_input_bgr = cv2.resize(
dst_face_bgr,
(self.predictor_input_size, self.predictor_input_size))
predictor_input_mask_a_0 = cv2.resize(
dst_face_mask_a_0,
(self.predictor_input_size, self.predictor_input_size))
predictor_input_mask_a = np.expand_dims(predictor_input_mask_a_0, -1)
predicted_bgra = self.predictor(
np.concatenate((predictor_input_bgr, predictor_input_mask_a), -1))
prd_face_bgr = np.clip(predicted_bgra[:, :, 0:3], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted_bgra[:, :, 3], 0.0, 1.0)
if not self.use_predicted_mask:
prd_face_mask_a_0 = predictor_input_mask_a_0
prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0
prd_face_mask_a = np.expand_dims(prd_face_mask_a_0, axis=-1)
prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1)
img_prd_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa,
face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4)
img_prd_face_mask_aaa = np.clip(img_prd_face_mask_aaa, 0.0, 1.0)
img_face_mask_aaa = img_prd_face_mask_aaa
if debug:
debugs += [img_face_mask_aaa.copy()]
img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0
img_face_mask_flatten_aaa = img_face_mask_aaa.copy()
img_face_mask_flatten_aaa[img_face_mask_flatten_aaa > 0.9] = 1.0
maxregion = np.argwhere(img_face_mask_flatten_aaa == 1.0)
out_img = img_bgr.copy()
if self.mode == 'raw':
if self.raw_mode == 'rgb' or self.raw_mode == 'rgb-mask':
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size, out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
if self.raw_mode == 'rgb-mask':
out_img = np.concatenate(
[out_img,
np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1)
if self.raw_mode == 'mask-only':
out_img = img_face_mask_aaa
if self.raw_mode == 'predicted-only':
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.zeros(out_img.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
else:
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
if debug:
print(
"maxregion.size: %d, minx:%d, maxx:%d miny:%d, maxy:%d"
% (maxregion.size, minx, maxx, miny, maxy))
lenx = maxx - minx
leny = maxy - miny
if lenx >= 4 and leny >= 4:
masky = int(minx + (lenx // 2))
maskx = int(miny + (leny // 2))
lowest_len = min(lenx, leny)
if debug:
print("lowest_len = %f" % (lowest_len))
img_mask_blurry_aaa = img_face_mask_aaa
if self.erode_mask_modifier != 0:
ero = int(lowest_len *
(0.126 - lowest_len * 0.00004551365) * 0.01 *
self.erode_mask_modifier)
if debug:
print("erode_size = %d" % (ero))
if ero > 0:
img_mask_blurry_aaa = cv2.erode(
img_mask_blurry_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
img_mask_blurry_aaa = cv2.dilate(
img_mask_blurry_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (-ero, -ero)),
iterations=1)
if self.seamless_erode_mask_modifier != 0:
ero = int(lowest_len *
(0.126 - lowest_len * 0.00004551365) * 0.01 *
self.seamless_erode_mask_modifier)
if debug:
print("seamless_erode_size = %d" % (ero))
if ero > 0:
img_face_mask_flatten_aaa = cv2.erode(
img_face_mask_flatten_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
img_face_mask_flatten_aaa = cv2.dilate(
img_face_mask_flatten_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (-ero, -ero)),
iterations=1)
if self.clip_hborder_mask_per > 0: #clip hborder before blur
prd_hborder_rect_mask_a = np.ones(
prd_face_mask_a.shape, dtype=np.float32)
prd_border_size = int(
prd_hborder_rect_mask_a.shape[1] *
self.clip_hborder_mask_per)
prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0
prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0
prd_hborder_rect_mask_a = np.expand_dims(
cv2.blur(prd_hborder_rect_mask_a,
(prd_border_size, prd_border_size)), -1)
img_prd_hborder_rect_mask_a = cv2.warpAffine(
prd_hborder_rect_mask_a, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4)
img_prd_hborder_rect_mask_a = np.expand_dims(
img_prd_hborder_rect_mask_a, -1)
img_mask_blurry_aaa *= img_prd_hborder_rect_mask_a
img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0,
1.0)
if debug:
debugs += [img_mask_blurry_aaa.copy()]
if self.blur_mask_modifier > 0:
blur = int(lowest_len * 0.10 * 0.01 *
self.blur_mask_modifier)
if debug:
print("blur_size = %d" % (blur))
if blur > 0:
img_mask_blurry_aaa = cv2.blur(
img_mask_blurry_aaa, (blur, blur))
img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0)
if debug:
debugs += [img_mask_blurry_aaa.copy()]
if self.color_transfer_mode is not None:
if self.color_transfer_mode == 'rct':
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
prd_face_bgr = image_utils.reinhard_color_transfer(
np.clip((prd_face_bgr * 255).astype(np.uint8),
0, 255),
np.clip((dst_face_bgr * 255).astype(np.uint8),
0, 255),
source_mask=prd_face_mask_a,
target_mask=prd_face_mask_a)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0,
1.0)
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
elif self.color_transfer_mode == 'lct':
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
prd_face_bgr = image_utils.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
if debug:
debugs += [
np.clip(
cv2.warpAffine(
prd_face_bgr, face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
cv2.WARP_INVERSE_MAP
| cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT), 0, 1.0)
]
if self.mode == 'hist-match-bw':
prd_face_bgr = cv2.cvtColor(prd_face_bgr,
cv2.COLOR_BGR2GRAY)
prd_face_bgr = np.repeat(
np.expand_dims(prd_face_bgr, -1), (3, ), -1)
if self.mode == 'hist-match' or self.mode == 'hist-match-bw':
if debug:
debugs += [
cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
]
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if self.masked_hist_match:
hist_mask_a *= prd_face_mask_a
hist_match_1 = prd_face_bgr * hist_mask_a + (
1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + (
1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = image_utils.color_hist_match(
hist_match_1, hist_match_2,
self.hist_match_threshold)
if self.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size, out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
if debug:
debugs += [out_img.copy()]
if self.mode == 'overlay':
pass
if self.mode == 'seamless' or self.mode == 'seamless-hist-match':
out_img = np.clip(
img_bgr * (1 - img_face_mask_aaa) +
(out_img * img_face_mask_aaa), 0, 1.0)
if debug:
debugs += [out_img.copy()]
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8),
(img_bgr * 255).astype(np.uint8),
(img_face_mask_flatten_aaa * 255).astype(np.uint8),
(masky, maskx), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
if debug:
debugs += [out_img.copy()]
out_img = np.clip(
img_bgr * (1 - img_mask_blurry_aaa) +
(out_img * img_mask_blurry_aaa), 0, 1.0)
if self.mode == 'seamless-hist-match':
out_face_bgr = cv2.warpAffine(
out_img, face_mat,
(self.output_size, self.output_size))
new_out_face_bgr = image_utils.color_hist_match(
out_face_bgr, dst_face_bgr,
self.hist_match_threshold)
new_out = cv2.warpAffine(
new_out_face_bgr, face_mat, img_size,
img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_mask_blurry_aaa) +
(new_out * img_mask_blurry_aaa), 0, 1.0)
if self.final_image_color_degrade_power != 0:
if debug:
debugs += [out_img.copy()]
out_img_reduced = image_utils.reduce_colors(
out_img, 256)
if self.final_image_color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = self.final_image_color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
if self.alpha:
out_img = np.concatenate([
out_img,
np.expand_dims(img_mask_blurry_aaa[:, :, 0], -1)
], -1)
out_img = np.clip(out_img, 0.0, 1.0)
if debug:
debugs += [out_img.copy()]
return debugs if debug else out_img
