def MergeMaskedFace(predictor_func, predictor_input_shape, face_enhancer_func,
xseg_256_extract_func, cfg, frame_info, img_bgr_uint8,
img_bgr, img_face_landmarks):
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
input_size = predictor_input_shape[0]
mask_subres_size = input_size * 4
output_size = input_size
if cfg.super_resolution_power != 0:
output_size *= 4
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
if mask_subres_size == output_size:
face_mask_output_mat = face_output_mat
else:
face_mask_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
mask_subres_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_bgr = np.clip(dst_face_bgr, 0, 1)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size))
predicted = predictor_func(predictor_input_bgr)
prd_face_bgr = np.clip(predicted[0], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0)
prd_face_dst_mask_a_0 = np.clip(predicted[2], 0, 1.0)
if cfg.super_resolution_power != 0:
prd_face_bgr_enhanced = face_enhancer_func(prd_face_bgr,
is_tanh=True,
preserve_size=False)
mod = cfg.super_resolution_power / 100.0
prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * (
1.0 - mod) + prd_face_bgr_enhanced * mod
prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
if cfg.super_resolution_power != 0:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
prd_face_dst_mask_a_0 = cv2.resize(prd_face_dst_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
if cfg.mask_mode == 1: #dst
wrk_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
elif cfg.mask_mode == 2: #learned-prd
wrk_face_mask_a_0 = prd_face_mask_a_0
elif cfg.mask_mode == 3: #learned-dst
wrk_face_mask_a_0 = prd_face_dst_mask_a_0
elif cfg.mask_mode == 4: #learned-prd*learned-dst
wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0
elif cfg.mask_mode == 5: #learned-prd+learned-dst
wrk_face_mask_a_0 = np.clip(prd_face_mask_a_0 + prd_face_dst_mask_a_0,
0, 1)
elif cfg.mask_mode >= 6 and cfg.mask_mode <= 9: #XSeg modes
if cfg.mask_mode == 6 or cfg.mask_mode == 8 or cfg.mask_mode == 9:
# obtain XSeg-prd
prd_face_xseg_bgr = cv2.resize(prd_face_bgr,
(xseg_input_size, ) * 2,
interpolation=cv2.INTER_CUBIC)
prd_face_xseg_mask = xseg_256_extract_func(prd_face_xseg_bgr)
X_prd_face_mask_a_0 = cv2.resize(prd_face_xseg_mask,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
if cfg.mask_mode >= 7 and cfg.mask_mode <= 9:
# obtain XSeg-dst
xseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, xseg_input_size, face_type=cfg.face_type)
dst_face_xseg_bgr = cv2.warpAffine(img_bgr,
xseg_mat,
(xseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_xseg_mask = xseg_256_extract_func(dst_face_xseg_bgr)
X_dst_face_mask_a_0 = cv2.resize(dst_face_xseg_mask,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
if cfg.mask_mode == 6: #'XSeg-prd'
wrk_face_mask_a_0 = X_prd_face_mask_a_0
elif cfg.mask_mode == 7: #'XSeg-dst'
wrk_face_mask_a_0 = X_dst_face_mask_a_0
elif cfg.mask_mode == 8: #'XSeg-prd*XSeg-dst'
wrk_face_mask_a_0 = X_prd_face_mask_a_0 * X_dst_face_mask_a_0
elif cfg.mask_mode == 9: #learned-prd*learned-dst*XSeg-prd*XSeg-dst
wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0 * X_prd_face_mask_a_0 * X_dst_face_mask_a_0
wrk_face_mask_a_0[wrk_face_mask_a_0 < (1.0 /
255.0)] = 0.0 # get rid of noise
# resize to mask_subres_size
if wrk_face_mask_a_0.shape[0] != mask_subres_size:
wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0,
(mask_subres_size, mask_subres_size),
interpolation=cv2.INTER_CUBIC)
# process mask in local predicted space
if 'raw' not in cfg.mode:
# add zero pad
wrk_face_mask_a_0 = np.pad(wrk_face_mask_a_0, input_size)
ero = cfg.erode_mask_modifier
blur = cfg.blur_mask_modifier
if ero > 0:
wrk_face_mask_a_0 = cv2.erode(wrk_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
wrk_face_mask_a_0 = cv2.dilate(wrk_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
# clip eroded/dilated mask in actual predict area
# pad with half blur size in order to accuratelly fade to zero at the boundary
clip_size = input_size + blur // 2
wrk_face_mask_a_0[:clip_size, :] = 0
wrk_face_mask_a_0[-clip_size:, :] = 0
wrk_face_mask_a_0[:, :clip_size] = 0
wrk_face_mask_a_0[:, -clip_size:] = 0
if blur > 0:
blur = blur + (1 - blur % 2)
wrk_face_mask_a_0 = cv2.GaussianBlur(wrk_face_mask_a_0,
(blur, blur), 0)
wrk_face_mask_a_0 = wrk_face_mask_a_0[input_size:-input_size,
input_size:-input_size]
wrk_face_mask_a_0 = np.clip(wrk_face_mask_a_0, 0, 1)
img_face_mask_a = cv2.warpAffine(wrk_face_mask_a_0,
face_mask_output_mat,
img_size,
np.zeros(img_bgr.shape[0:2],
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)[..., None]
img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0)
img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise
if wrk_face_mask_a_0.shape[0] != output_size:
wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0,
(output_size, output_size),
interpolation=cv2.INTER_CUBIC)
wrk_face_mask_a = wrk_face_mask_a_0[..., None]
out_img = None
out_merging_mask_a = None
if cfg.mode == 'original':
return img_bgr, img_face_mask_a
elif 'raw' in cfg.mode:
if cfg.mode == 'raw-rgb':
out_img_face = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img_face_mask = cv2.warpAffine(
np.ones_like(prd_face_bgr), face_output_mat, img_size,
np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img = img_bgr * (
1 - out_img_face_mask) + out_img_face * out_img_face_mask
out_merging_mask_a = img_face_mask_a
elif cfg.mode == 'raw-predict':
out_img = prd_face_bgr
out_merging_mask_a = wrk_face_mask_a
else:
raise ValueError(f"undefined raw type {cfg.mode}")
out_img = np.clip(out_img, 0.0, 1.0)
else:
# Process if the mask meets minimum size
maxregion = np.argwhere(img_face_mask_a >= 0.1)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
wrk_face_mask_area_a = wrk_face_mask_a.copy()
wrk_face_mask_area_a[wrk_face_mask_area_a > 0] = 1.0
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1: #rct
prd_face_bgr = imagelib.reinhard_color_transfer(
np.clip(prd_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
np.clip(dst_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
prd_face_bgr = imagelib.color_transfer_sot(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a,
steps=10,
batch_size=30)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
prd_face_bgr = imagelib.color_transfer_mix(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
if cfg.mode == 'hist-match':
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= wrk_face_mask_area_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2,
cfg.hist_match_threshold).astype(dtype=np.float32)
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_a = None
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_a.copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[
img_face_seamless_mask_a <= i / 10.0] = 0.0
break
out_img = cv2.warpAffine(
prd_face_bgr, face_output_mat, img_size,
np.empty_like(img_bgr),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img = np.clip(out_img, 0.0, 1.0)
if 'seamless' in cfg.mode:
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
cfg_mp = cfg.motion_blur_power / 100.0
out_img = img_bgr * (1 - img_face_mask_a) + (out_img *
img_face_mask_a)
if ('seamless' in cfg.mode and cfg.color_transfer_mode != 0) or \
cfg.mode == 'seamless-hist-match' or \
cfg_mp != 0 or \
cfg.blursharpen_amount != 0 or \
cfg.image_denoise_power != 0 or \
cfg.bicubic_degrade_power != 0:
out_face_bgr = cv2.warpAffine(out_img,
face_mat,
(output_size, output_size),
flags=cv2.INTER_CUBIC)
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
out_face_bgr = imagelib.reinhard_color_transfer(
np.clip(
out_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
np.clip(
dst_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8))
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0,
1.0)
elif cfg.color_transfer_mode == 2: #lct
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
out_face_bgr = imagelib.color_transfer_sot(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a,
steps=10,
batch_size=30)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
out_face_bgr = imagelib.color_transfer_mix(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr,
cfg.hist_match_threshold)
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_power != 0:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.blursharpen_amount != 0:
out_face_bgr = imagelib.blursharpen(
out_face_bgr, cfg.sharpen_mode, 3,
cfg.blursharpen_amount)
if cfg.image_denoise_power != 0:
n = cfg.image_denoise_power
while n > 0:
img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
if int(n / 100) != 0:
img_bgr = img_bgr_denoised
else:
pass_power = (n % 100) / 100.0
img_bgr = img_bgr * (
1.0 -
pass_power) + img_bgr_denoised * pass_power
n = max(n - 10, 0)
if cfg.bicubic_degrade_power != 0:
p = 1.0 - cfg.bicubic_degrade_power / 101.0
img_bgr_downscaled = cv2.resize(
img_bgr,
(int(img_size[0] * p), int(img_size[1] * p)),
interpolation=cv2.INTER_CUBIC)
img_bgr = cv2.resize(img_bgr_downscaled,
img_size,
interpolation=cv2.INTER_CUBIC)
new_out = cv2.warpAffine(
out_face_bgr, face_mat, img_size,
np.empty_like(img_bgr),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
out_img = np.clip(
img_bgr * (1 - img_face_mask_a) +
(new_out * img_face_mask_a), 0, 1.0)
if cfg.color_degrade_power != 0:
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
out_merging_mask_a = img_face_mask_a
if out_img is None:
out_img = img_bgr.copy()
return out_img, out_merging_mask_a
def process(samples,
sample_process_options,
output_sample_types,
debug,
ct_sample=None):
SPST = SampleProcessor.SampleType
SPCT = SampleProcessor.ChannelType
SPFMT = SampleProcessor.FaceMaskType
sample_rnd_seed = np.random.randint(0x80000000)
outputs = []
for sample in samples:
sample_bgr = sample.load_bgr()
ct_sample_bgr = None
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range)
outputs_sample = []
for opts in output_sample_types:
sample_type = opts.get('sample_type', SPST.NONE)
channel_type = opts.get('channel_type', SPCT.NONE)
resolution = opts.get('resolution', 0)
warp = opts.get('warp', False)
transform = opts.get('transform', False)
motion_blur = opts.get('motion_blur', None)
gaussian_blur = opts.get('gaussian_blur', None)
normalize_tanh = opts.get('normalize_tanh', False)
ct_mode = opts.get('ct_mode', 'None')
data_format = opts.get('data_format', 'NHWC')
if sample_type == SPST.FACE_IMAGE or sample_type == SPST.FACE_MASK:
if not is_face_sample:
raise ValueError(
"face_samples should be provided for sample_type FACE_*"
)
if is_face_sample:
face_type = opts.get('face_type', None)
face_mask_type = opts.get('face_mask_type', SPFMT.NONE)
if face_type is None:
raise ValueError(
"face_type must be defined for face samples")
if face_type > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
% (sample.filename, sample.face_type, target_ft))
if sample_type == SPST.FACE_IMAGE or sample_type == SPST.FACE_MASK:
if sample_type == SPST.FACE_MASK:
if face_mask_type == SPFMT.ALL_HULL or \
face_mask_type == SPFMT.EYES_HULL or \
face_mask_type == SPFMT.ALL_EYES_HULL:
if face_mask_type == SPFMT.ALL_HULL or \
face_mask_type == SPFMT.ALL_EYES_HULL:
if sample.eyebrows_expand_mod is not None:
all_mask = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.
eyebrows_expand_mod)
else:
all_mask = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape, sample.landmarks)
all_mask = np.clip(all_mask, 0, 1)
if face_mask_type == SPFMT.EYES_HULL or \
face_mask_type == SPFMT.ALL_EYES_HULL:
eyes_mask = LandmarksProcessor.get_image_eye_mask(
sample_bgr.shape, sample.landmarks)
eyes_mask = np.clip(eyes_mask, 0, 1)
if face_mask_type == SPFMT.ALL_HULL:
img = all_mask
elif face_mask_type == SPFMT.EYES_HULL:
img = eyes_mask
elif face_mask_type == SPFMT.ALL_EYES_HULL:
img = all_mask + eyes_mask
elif face_mask_type == SPFMT.STRUCT:
if sample.eyebrows_expand_mod is not None:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.
eyebrows_expand_mod)
else:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape, sample.landmarks)
if sample.ie_polys is not None:
sample.ie_polys.overlay_mask(img)
if sample.face_type == FaceType.MARK_ONLY:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0], face_type)
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_LINEAR)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=False,
cv2_inter=cv2.INTER_LINEAR)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_LINEAR)[..., None]
else:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, face_type)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=False,
cv2_inter=cv2.INTER_LINEAR)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_CONSTANT,
flags=cv2.INTER_LINEAR)[..., None]
if channel_type == SPCT.G:
out_sample = img.astype(np.float32)
else:
raise ValueError(
"only channel_type.G supported for the mask")
elif sample_type == SPST.FACE_IMAGE:
img = sample_bgr
if motion_blur is not None:
chance, mb_max_size = motion_blur
chance = np.clip(chance, 0, 100)
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
mblur_rnd_chance = l_rnd_state.randint(100)
mblur_rnd_kernel = l_rnd_state.randint(
mb_max_size) + 1
mblur_rnd_deg = l_rnd_state.randint(360)
if mblur_rnd_chance < chance:
img = imagelib.LinearMotionBlur(
img, mblur_rnd_kernel, mblur_rnd_deg)
if gaussian_blur is not None:
chance, kernel_max_size = gaussian_blur
chance = np.clip(chance, 0, 100)
l_rnd_state = np.random.RandomState(
sample_rnd_seed + 1)
gblur_rnd_chance = l_rnd_state.randint(100)
gblur_rnd_kernel = l_rnd_state.randint(
kernel_max_size) * 2 + 1
if gblur_rnd_chance < chance:
img = cv2.GaussianBlur(
img, (gblur_rnd_kernel, ) * 2, 0)
if sample.face_type == FaceType.MARK_ONLY:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0], face_type)
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=True)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
else:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, face_type)
img = imagelib.warp_by_params(
params,
img,
warp,
transform,
can_flip=True,
border_replicate=True)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_REPLICATE,
flags=cv2.INTER_CUBIC)
img = np.clip(img.astype(np.float32), 0, 1)
# Apply random color transfer
if ct_mode is not None and ct_sample is not None:
if ct_sample_bgr is None:
ct_sample_bgr = ct_sample.load_bgr()
img = imagelib.color_transfer(
ct_mode, img,
cv2.resize(ct_sample_bgr,
(resolution, resolution),
cv2.INTER_LINEAR))
# Transform from BGR to desired channel_type
if channel_type == SPCT.BGR:
out_sample = img
elif channel_type == SPCT.BGR_SHUFFLE:
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
out_sample = np.take(img,
l_rnd_state.permutation(
img.shape[-1]),
axis=-1)
elif channel_type == SPCT.BGR_RANDOM_HSV_SHIFT:
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
h, s, v = cv2.split(hsv)
h = (h + l_rnd_state.randint(360)) % 360
s = np.clip(s + l_rnd_state.random() - 0.5, 0, 1)
v = np.clip(v + l_rnd_state.random() - 0.5, 0, 1)
hsv = cv2.merge([h, s, v])
out_sample = np.clip(
cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0, 1)
elif channel_type == SPCT.BGR_RANDOM_RGB_LEVELS:
l_rnd_state = np.random.RandomState(
sample_rnd_seed)
np_rnd = l_rnd_state.rand
inBlack = np.array([
np_rnd() * 0.25,
np_rnd() * 0.25,
np_rnd() * 0.25
],
dtype=np.float32)
inWhite = np.array([
1.0 - np_rnd() * 0.25, 1.0 - np_rnd() * 0.25,
1.0 - np_rnd() * 0.25
],
dtype=np.float32)
inGamma = np.array([
0.5 + np_rnd(), 0.5 + np_rnd(), 0.5 + np_rnd()
],
dtype=np.float32)
outBlack = np.array([0.0, 0.0, 0.0],
dtype=np.float32)
outWhite = np.array([1.0, 1.0, 1.0],
dtype=np.float32)
out_sample = np.clip(
(img - inBlack) / (inWhite - inBlack), 0, 1)
out_sample = (out_sample**(1 / inGamma)) * (
outWhite - outBlack) + outBlack
out_sample = np.clip(out_sample, 0, 1)
elif channel_type == SPCT.G:
out_sample = cv2.cvtColor(img,
cv2.COLOR_BGR2GRAY)[...,
None]
elif channel_type == SPCT.GGG:
out_sample = np.repeat(
np.expand_dims(
cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1)
# Final transformations
if not debug:
if normalize_tanh:
out_sample = np.clip(out_sample * 2.0 - 1.0, -1.0,
1.0)
if data_format == "NCHW":
out_sample = np.transpose(out_sample, (2, 0, 1))
#else:
# img = imagelib.warp_by_params (params, img, warp, transform, can_flip=True, border_replicate=True)
# img = cv2.resize( img, (resolution,resolution), cv2.INTER_CUBIC )
elif sample_type == SPST.LANDMARKS_ARRAY:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
out_sample = l
elif sample_type == SPST.PITCH_YAW_ROLL or sample_type == SPST.PITCH_YAW_ROLL_SIGMOID:
pitch_yaw_roll = sample.get_pitch_yaw_roll()
if params['flip']:
yaw = -yaw
if sample_type == SPST.PITCH_YAW_ROLL_SIGMOID:
pitch = np.clip((pitch / math.pi) / 2.0 + 0.5, 0, 1)
yaw = np.clip((yaw / math.pi) / 2.0 + 0.5, 0, 1)
roll = np.clip((roll / math.pi) / 2.0 + 0.5, 0, 1)
out_sample = (pitch, yaw, roll)
else:
raise ValueError('expected sample_type')
outputs_sample.append(out_sample)
outputs += [outputs_sample]
return outputs
def process(samples,
sample_process_options,
output_sample_types,
debug,
ct_sample=None):
SPTF = SampleProcessor.Types
sample_rnd_seed = np.random.randint(0x80000000)
outputs = []
for sample in samples:
sample_bgr = sample.load_bgr()
ct_sample_bgr = None
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range,
rnd_seed=sample_rnd_seed)
outputs_sample = []
for opts in output_sample_types:
resolution = opts.get('resolution', 0)
types = opts.get('types', [])
motion_blur = opts.get('motion_blur', None)
gaussian_blur = opts.get('gaussian_blur', None)
ct_mode = opts.get('ct_mode', 'None')
normalize_tanh = opts.get('normalize_tanh', False)
data_format = opts.get('data_format', 'NHWC')
img_type = SPTF.NONE
target_face_type = SPTF.NONE
mode_type = SPTF.NONE
for t in types:
if t >= SPTF.IMG_TYPE_BEGIN and t < SPTF.IMG_TYPE_END:
img_type = t
elif t >= SPTF.FACE_TYPE_BEGIN and t < SPTF.FACE_TYPE_END:
target_face_type = t
elif t >= SPTF.MODE_BEGIN and t < SPTF.MODE_END:
mode_type = t
if mode_type == SPTF.MODE_FACE_MASK_HULL and not is_face_sample:
raise ValueError(
"MODE_FACE_MASK_HULL applicable only for face samples")
if mode_type == SPTF.MODE_FACE_MASK_STRUCT and not is_face_sample:
raise ValueError(
"MODE_FACE_MASK_STRUCT applicable only for face samples"
)
if is_face_sample:
if target_face_type == SPTF.NONE:
raise ValueError(
"target face type must be defined for face samples"
)
can_warp = (img_type == SPTF.IMG_WARPED
or img_type == SPTF.IMG_WARPED_TRANSFORMED)
can_transform = (img_type == SPTF.IMG_WARPED_TRANSFORMED
or img_type == SPTF.IMG_TRANSFORMED)
if img_type == SPTF.NONE:
raise ValueError('expected IMG_ type')
if img_type == SPTF.IMG_LANDMARKS_ARRAY:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
out_sample = l
elif img_type == SPTF.IMG_PITCH_YAW_ROLL or img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch_yaw_roll = sample.get_pitch_yaw_roll()
if params['flip']:
yaw = -yaw
if img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch = np.clip((pitch / math.pi) / 2.0 + 0.5, 0, 1)
yaw = np.clip((yaw / math.pi) / 2.0 + 0.5, 0, 1)
roll = np.clip((roll / math.pi) / 2.0 + 0.5, 0, 1)
out_sample = (pitch, yaw, roll)
else:
if mode_type == SPTF.NONE:
raise ValueError('expected MODE_ type')
if mode_type == SPTF.MODE_FACE_MASK_HULL:
if sample.eyebrows_expand_mod is not None:
img = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.eyebrows_expand_mod)
else:
img = LandmarksProcessor.get_image_hull_mask(
sample_bgr.shape, sample.landmarks)
if sample.ie_polys is not None:
sample.ie_polys.overlay_mask(img)
elif mode_type == SPTF.MODE_FACE_MASK_STRUCT:
if sample.eyebrows_expand_mod is not None:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape,
sample.landmarks,
eyebrows_expand_mod=sample.eyebrows_expand_mod)
else:
img = LandmarksProcessor.get_face_struct_mask(
sample_bgr.shape, sample.landmarks)
else:
img = sample_bgr
if motion_blur is not None:
chance, mb_max_size = motion_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
img = imagelib.LinearMotionBlur(
img,
np.random.randint(mb_max_size) + 1,
np.random.randint(360))
if gaussian_blur is not None:
chance, kernel_max_size = gaussian_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
img = cv2.GaussianBlur(
img,
(np.random.randint(kernel_max_size) * 2 +
1, ) * 2, 0)
if is_face_sample:
target_ft = SampleProcessor.SPTF_FACETYPE_TO_FACETYPE[
target_face_type]
if target_ft > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
%
(sample.filename, sample.face_type, target_ft))
if sample.face_type == FaceType.MARK_ONLY:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0], target_ft)
if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT:
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=False)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)[..., None]
else:
img = cv2.warpAffine(
img,
mat, (sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=True)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
else:
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, target_ft)
if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT:
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=False)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_CONSTANT,
flags=cv2.INTER_CUBIC)[..., None]
else:
img = imagelib.warp_by_params(
params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=True)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=cv2.BORDER_REPLICATE,
flags=cv2.INTER_CUBIC)
else:
img = imagelib.warp_by_params(params,
img,
can_warp,
can_transform,
can_flip=True,
border_replicate=True)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT:
out_sample = np.clip(img.astype(np.float32), 0, 1)
else:
img = np.clip(img.astype(np.float32), 0, 1)
if ct_mode is not None and ct_sample is not None:
if ct_sample_bgr is None:
ct_sample_bgr = ct_sample.load_bgr()
img = imagelib.color_transfer(
ct_mode, img,
cv2.resize(ct_sample_bgr,
(resolution, resolution),
cv2.INTER_LINEAR))
if mode_type == SPTF.MODE_BGR:
out_sample = img
elif mode_type == SPTF.MODE_BGR_SHUFFLE:
rnd_state = np.random.RandomState(sample_rnd_seed)
out_sample = np.take(img,
rnd_state.permutation(
img.shape[-1]),
axis=-1)
elif mode_type == SPTF.MODE_BGR_RANDOM_HSV_SHIFT:
rnd_state = np.random.RandomState(sample_rnd_seed)
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
h, s, v = cv2.split(hsv)
h = (h + rnd_state.randint(360)) % 360
s = np.clip(s + rnd_state.random() - 0.5, 0, 1)
v = np.clip(v + rnd_state.random() - 0.5, 0, 1)
hsv = cv2.merge([h, s, v])
out_sample = np.clip(
cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0, 1)
elif mode_type == SPTF.MODE_G:
out_sample = cv2.cvtColor(img,
cv2.COLOR_BGR2GRAY)[...,
None]
elif mode_type == SPTF.MODE_GGG:
out_sample = np.repeat(
np.expand_dims(
cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1)
if not debug:
if normalize_tanh:
out_sample = np.clip(out_sample * 2.0 - 1.0, -1.0,
1.0)
if data_format == "NCHW":
out_sample = np.transpose(out_sample, (2, 0, 1))
outputs_sample.append(out_sample)
outputs += [outputs_sample]
return outputs
def MergeMaskedFace(predictor_func, predictor_input_shape, cfg, frame_info,
img_bgr_uint8, img_bgr, img_face_landmarks):
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
if cfg.mode == 'original':
return img_bgr, img_face_mask_a
out_img = img_bgr.copy()
out_merging_mask_a = None
input_size = predictor_input_shape[0]
mask_subres_size = input_size * 4
output_size = input_size
if cfg.super_resolution_power != 0:
output_size *= 4
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
if mask_subres_size == output_size:
face_mask_output_mat = face_output_mat
else:
face_mask_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
mask_subres_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_bgr = np.clip(dst_face_bgr, 0, 1)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size))
predicted = predictor_func(predictor_input_bgr)
if isinstance(predicted, tuple):
#merger return bgr,mask
prd_face_bgr = np.clip(predicted[0], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0)
predictor_masked = True
else:
#merger return bgr only, using dst mask
prd_face_bgr = np.clip(predicted, 0, 1.0)
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(input_size, input_size))
predictor_masked = False
if cfg.super_resolution_power != 0:
prd_face_bgr_enhanced = cfg.superres_func(prd_face_bgr)
mod = cfg.super_resolution_power / 100.0
prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * (
1.0 - mod) + prd_face_bgr_enhanced * mod
prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
if cfg.super_resolution_power != 0:
if predictor_masked:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 2: #dst
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
elif cfg.mask_mode >= 3 and cfg.mask_mode <= 8:
if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6:
prd_face_fanseg_bgr = cv2.resize(prd_face_bgr,
(cfg.fanseg_input_size, ) * 2)
prd_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, prd_face_fanseg_bgr)
FAN_prd_face_mask_a_0 = cv2.resize(prd_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode >= 4 and cfg.mask_mode <= 7:
full_face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=FaceType.FULL)
dst_face_fanseg_bgr = cv2.warpAffine(img_bgr,
full_face_fanseg_mat,
(cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, dst_face_fanseg_bgr)
if cfg.face_type == FaceType.FULL:
FAN_dst_face_mask_a_0 = cv2.resize(dst_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=cfg.face_type)
fanseg_rect_corner_pts = np.array(
[[0, 0], [cfg.fanseg_input_size - 1, 0],
[0, cfg.fanseg_input_size - 1]],
dtype=np.float32)
a = LandmarksProcessor.transform_points(fanseg_rect_corner_pts,
face_fanseg_mat,
invert=True)
b = LandmarksProcessor.transform_points(
a, full_face_fanseg_mat)
m = cv2.getAffineTransform(b, fanseg_rect_corner_pts)
FAN_dst_face_mask_a_0 = cv2.warpAffine(
dst_face_fanseg_mask,
m, (cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
FAN_dst_face_mask_a_0 = cv2.resize(FAN_dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 3: #FAN-prd
prd_face_mask_a_0 = FAN_prd_face_mask_a_0
elif cfg.mask_mode == 4: #FAN-dst
prd_face_mask_a_0 = FAN_dst_face_mask_a_0
elif cfg.mask_mode == 5:
prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 6:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 7:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0
prd_face_mask_a_0[prd_face_mask_a_0 < (1.0 /
255.0)] = 0.0 # get rid of noise
# resize to mask_subres_size
if prd_face_mask_a_0.shape[0] != mask_subres_size:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(mask_subres_size, mask_subres_size),
cv2.INTER_CUBIC)
# process mask in local predicted space
if 'raw' not in cfg.mode:
# add zero pad
prd_face_mask_a_0 = np.pad(prd_face_mask_a_0, input_size)
ero = cfg.erode_mask_modifier
blur = cfg.blur_mask_modifier
if ero > 0:
prd_face_mask_a_0 = cv2.erode(prd_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
prd_face_mask_a_0 = cv2.dilate(prd_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
# clip eroded/dilated mask in actual predict area
# pad with half blur size in order to accuratelly fade to zero at the boundary
clip_size = input_size + blur // 2
prd_face_mask_a_0[:clip_size, :] = 0
prd_face_mask_a_0[-clip_size:, :] = 0
prd_face_mask_a_0[:, :clip_size] = 0
prd_face_mask_a_0[:, -clip_size:] = 0
if blur > 0:
blur = blur + (1 - blur % 2)
prd_face_mask_a_0 = cv2.GaussianBlur(prd_face_mask_a_0,
(blur, blur), 0)
prd_face_mask_a_0 = prd_face_mask_a_0[input_size:-input_size,
input_size:-input_size]
prd_face_mask_a_0 = np.clip(prd_face_mask_a_0, 0, 1)
img_face_mask_a = cv2.warpAffine(prd_face_mask_a_0,
face_mask_output_mat,
img_size,
np.zeros(img_bgr.shape[0:2],
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)[..., None]
img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0)
img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise
if prd_face_mask_a_0.shape[0] != output_size:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
prd_face_mask_a = prd_face_mask_a_0[..., None]
prd_face_mask_area_a = prd_face_mask_a.copy()
prd_face_mask_area_a[prd_face_mask_area_a > 0] = 1.0
if 'raw' in cfg.mode:
if cfg.mode == 'raw-rgb':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_merging_mask_a = img_face_mask_a
out_img = np.clip(out_img, 0.0, 1.0)
else:
#averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
ar = []
for i in range(1, 10):
maxregion = np.argwhere(img_face_mask_a > i / 10.0)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
ar += [[lenx, leny]]
if len(ar) > 0:
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1: #rct
prd_face_bgr = imagelib.reinhard_color_transfer(
np.clip(prd_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8),
np.clip(dst_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8),
)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
prd_face_bgr = imagelib.color_transfer_sot(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
prd_face_bgr = imagelib.color_transfer_mix(
prd_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
if cfg.mode == 'hist-match':
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= prd_face_mask_area_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2,
cfg.hist_match_threshold).astype(dtype=np.float32)
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_a = None
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_a.copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[img_face_seamless_mask_a <= i /
10.0] = 0.0
break
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
if 'seamless' in cfg.mode:
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
out_img = img_bgr * (1 - img_face_mask_a) + (out_img *
img_face_mask_a)
out_face_bgr = cv2.warpAffine(out_img,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
out_face_bgr = imagelib.reinhard_color_transfer(
np.clip(out_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8),
np.clip(dst_face_bgr * prd_face_mask_area_a * 255, 0,
255).astype(np.uint8))
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
out_face_bgr = imagelib.color_transfer_sot(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
out_face_bgr = imagelib.color_transfer_mix(
out_face_bgr * prd_face_mask_area_a,
dst_face_bgr * prd_face_mask_area_a)
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr, cfg.hist_match_threshold)
cfg_mp = cfg.motion_blur_power / 100.0
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_power != 0:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.blursharpen_amount != 0:
out_face_bgr = cfg.blursharpen_func(out_face_bgr,
cfg.sharpen_mode, 3,
cfg.blursharpen_amount)
if cfg.image_denoise_power != 0:
n = cfg.image_denoise_power
while n > 0:
img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
if int(n / 100) != 0:
img_bgr = img_bgr_denoised
else:
pass_power = (n % 100) / 100.0
img_bgr = img_bgr * (
1.0 - pass_power) + img_bgr_denoised * pass_power
n = max(n - 10, 0)
if cfg.bicubic_degrade_power != 0:
p = 1.0 - cfg.bicubic_degrade_power / 101.0
img_bgr_downscaled = cv2.resize(
img_bgr, (int(img_size[0] * p), int(img_size[1] * p)),
cv2.INTER_CUBIC)
img_bgr = cv2.resize(img_bgr_downscaled, img_size,
cv2.INTER_CUBIC)
new_out = cv2.warpAffine(out_face_bgr, face_mat, img_size,
img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_face_mask_a) + (new_out * img_face_mask_a),
0, 1.0)
if cfg.color_degrade_power != 0:
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
out_merging_mask_a = img_face_mask_a
return out_img, out_merging_mask_a
def MergeMaskedFace(predictor_func, predictor_input_shape, cfg, frame_info,
img_bgr_uint8, img_bgr, img_face_landmarks):
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
if cfg.mode == 'original':
if cfg.export_mask_alpha:
img_bgr = np.concatenate([img_bgr, img_face_mask_a], -1)
return img_bgr, img_face_mask_a
out_img = img_bgr.copy()
out_merging_mask = None
output_size = predictor_input_shape[0]
if cfg.super_resolution_mode != 0:
output_size *= 4
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_bgr = np.clip(dst_face_bgr, 0, 1)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
predictor_input_bgr = cv2.resize(dst_face_bgr, predictor_input_shape[0:2])
predicted = predictor_func(predictor_input_bgr)
if isinstance(predicted, tuple):
#merger return bgr,mask
prd_face_bgr = np.clip(predicted[0], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0)
predictor_masked = True
else:
#merger return bgr only, using dst mask
prd_face_bgr = np.clip(predicted, 0, 1.0)
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
predictor_input_shape[0:2])
predictor_masked = False
if cfg.super_resolution_mode:
prd_face_bgr = cfg.superres_func(cfg.super_resolution_mode,
prd_face_bgr)
prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
if predictor_masked:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 2: #dst
prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
elif cfg.mask_mode >= 3 and cfg.mask_mode <= 8:
if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6:
prd_face_fanseg_bgr = cv2.resize(prd_face_bgr,
(cfg.fanseg_input_size, ) * 2)
prd_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, prd_face_fanseg_bgr)
FAN_prd_face_mask_a_0 = cv2.resize(prd_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode >= 4 and cfg.mask_mode <= 7:
full_face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=FaceType.FULL)
dst_face_fanseg_bgr = cv2.warpAffine(img_bgr,
full_face_fanseg_mat,
(cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_fanseg_mask = cfg.fanseg_extract_func(
FaceType.FULL, dst_face_fanseg_bgr)
if cfg.face_type == FaceType.FULL:
FAN_dst_face_mask_a_0 = cv2.resize(dst_face_fanseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
else:
face_fanseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
cfg.fanseg_input_size,
face_type=cfg.face_type)
fanseg_rect_corner_pts = np.array(
[[0, 0], [cfg.fanseg_input_size - 1, 0],
[0, cfg.fanseg_input_size - 1]],
dtype=np.float32)
a = LandmarksProcessor.transform_points(fanseg_rect_corner_pts,
face_fanseg_mat,
invert=True)
b = LandmarksProcessor.transform_points(
a, full_face_fanseg_mat)
m = cv2.getAffineTransform(b, fanseg_rect_corner_pts)
FAN_dst_face_mask_a_0 = cv2.warpAffine(
dst_face_fanseg_mask,
m, (cfg.fanseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
FAN_dst_face_mask_a_0 = cv2.resize(FAN_dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 3: #FAN-prd
prd_face_mask_a_0 = FAN_prd_face_mask_a_0
elif cfg.mask_mode == 4: #FAN-dst
prd_face_mask_a_0 = FAN_dst_face_mask_a_0
elif cfg.mask_mode == 5:
prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 6:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0
elif cfg.mask_mode == 7:
prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0
#elif cfg.mask_mode == 8: #FANCHQ-dst
# prd_face_mask_a_0 = FANCHQ_dst_face_mask_a_0
prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0
prd_face_mask_a = prd_face_mask_a_0[..., np.newaxis]
prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1)
img_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa,
face_output_mat,
img_size,
np.zeros(img_bgr.shape,
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0.0, 1.0)
img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 #get rid of noise
if 'raw' in cfg.mode:
face_corner_pts = np.array(
[[0, 0], [output_size - 1, 0], [output_size - 1, output_size - 1],
[0, output_size - 1]],
dtype=np.float32)
square_mask = np.zeros(img_bgr.shape, dtype=np.float32)
cv2.fillConvexPoly(square_mask, \
LandmarksProcessor.transform_points (face_corner_pts, face_output_mat, invert=True ).astype(np.int), \
(1,1,1) )
if cfg.mode == 'raw-rgb':
out_merging_mask = square_mask
if cfg.mode == 'raw-rgb' or cfg.mode == 'raw-rgb-mask':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
if cfg.mode == 'raw-rgb-mask':
out_img = np.concatenate(
[out_img,
np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1)
out_merging_mask = square_mask
elif cfg.mode == 'raw-mask-only':
out_img = img_face_mask_aaa
out_merging_mask = img_face_mask_aaa
elif cfg.mode == 'raw-predicted-only':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_merging_mask = square_mask
out_img = np.clip(out_img, 0.0, 1.0)
else:
#averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
ar = []
for i in range(1, 10):
maxregion = np.argwhere(img_face_mask_aaa > i / 10.0)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
ar += [[lenx, leny]]
if len(ar) > 0:
lenx, leny = np.mean(ar, axis=0)
lowest_len = min(lenx, leny)
if cfg.erode_mask_modifier != 0:
ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) *
0.01 * cfg.erode_mask_modifier)
if ero > 0:
img_face_mask_aaa = cv2.erode(img_face_mask_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(ero, ero)),
iterations=1)
elif ero < 0:
img_face_mask_aaa = cv2.dilate(img_face_mask_aaa,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
if cfg.clip_hborder_mask_per > 0: #clip hborder before blur
prd_hborder_rect_mask_a = np.ones(prd_face_mask_a.shape,
dtype=np.float32)
prd_border_size = int(prd_hborder_rect_mask_a.shape[1] *
cfg.clip_hborder_mask_per)
prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0
prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0
prd_hborder_rect_mask_a[-prd_border_size:, :, :] = 0
prd_hborder_rect_mask_a = np.expand_dims(
cv2.blur(prd_hborder_rect_mask_a,
(prd_border_size, prd_border_size)), -1)
img_prd_hborder_rect_mask_a = cv2.warpAffine(
prd_hborder_rect_mask_a, face_output_mat, img_size,
np.zeros(img_bgr.shape, dtype=np.float32),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)
img_prd_hborder_rect_mask_a = np.expand_dims(
img_prd_hborder_rect_mask_a, -1)
img_face_mask_aaa *= img_prd_hborder_rect_mask_a
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0, 1.0)
if cfg.blur_mask_modifier > 0:
blur = int(lowest_len * 0.10 * 0.01 * cfg.blur_mask_modifier)
if blur > 0:
img_face_mask_aaa = cv2.blur(img_face_mask_aaa,
(blur, blur))
img_face_mask_aaa = np.clip(img_face_mask_aaa, 0, 1.0)
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1: #rct
prd_face_bgr = imagelib.reinhard_color_transfer(
(prd_face_bgr * 255).astype(np.uint8),
(dst_face_bgr * 255).astype(np.uint8),
source_mask=prd_face_mask_a,
target_mask=prd_face_mask_a)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 3: #mkl
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
elif cfg.color_transfer_mode == 5: #idt
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
elif cfg.color_transfer_mode == 7: #sot-m
prd_face_bgr = imagelib.color_transfer_sot(
prd_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
prd_face_bgr = imagelib.color_transfer_mix(
prd_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
if cfg.mode == 'hist-match-bw':
prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY)
prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1),
(3, ), -1)
if cfg.mode == 'hist-match' or cfg.mode == 'hist-match-bw':
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= prd_face_mask_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2,
cfg.hist_match_threshold).astype(dtype=np.float32)
if cfg.mode == 'hist-match-bw':
prd_face_bgr = prd_face_bgr.astype(dtype=np.float32)
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_mask_a = img_face_mask_aaa[..., 0:1]
img_face_seamless_mask_a = None
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_a.copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[img_face_seamless_mask_a <= i /
10.0] = 0.0
break
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
if 'seamless' in cfg.mode:
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
out_img = img_bgr * (1 - img_face_mask_aaa) + (out_img *
img_face_mask_aaa)
out_face_bgr = cv2.warpAffine(out_img, face_mat,
(output_size, output_size))
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
face_mask_aaa = cv2.warpAffine(img_face_mask_aaa, face_mat,
(output_size, output_size))
out_face_bgr = imagelib.reinhard_color_transfer(
(out_face_bgr * 255).astype(np.uint8),
(dst_face_bgr * 255).astype(np.uint8),
source_mask=face_mask_aaa,
target_mask=face_mask_aaa)
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 3: #mkl
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
elif cfg.color_transfer_mode == 5: #idt
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
elif cfg.color_transfer_mode == 7: #sot-m
out_face_bgr = imagelib.color_transfer_sot(
out_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
out_face_bgr = imagelib.color_transfer_mix(
out_face_bgr * prd_face_mask_a,
dst_face_bgr * prd_face_mask_a)
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr, cfg.hist_match_threshold)
cfg_mp = cfg.motion_blur_power / 100.0
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_mode:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.blursharpen_amount != 0:
out_face_bgr = cfg.blursharpen_func(out_face_bgr,
cfg.sharpen_mode, 3,
cfg.blursharpen_amount)
if cfg.image_denoise_power != 0:
n = cfg.image_denoise_power
while n > 0:
img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
if int(n / 100) != 0:
img_bgr = img_bgr_denoised
else:
pass_power = (n % 100) / 100.0
img_bgr = img_bgr * (
1.0 - pass_power) + img_bgr_denoised * pass_power
n = max(n - 10, 0)
if cfg.bicubic_degrade_power != 0:
p = 1.0 - cfg.bicubic_degrade_power / 101.0
img_bgr_downscaled = cv2.resize(
img_bgr, (int(img_size[0] * p), int(img_size[1] * p)),
cv2.INTER_CUBIC)
img_bgr = cv2.resize(img_bgr_downscaled, img_size,
cv2.INTER_CUBIC)
new_out = cv2.warpAffine(out_face_bgr, face_mat, img_size,
img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_face_mask_aaa) +
(new_out * img_face_mask_aaa), 0, 1.0)
if cfg.color_degrade_power != 0:
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
out_merging_mask = img_face_mask_aaa
return out_img, out_merging_mask[..., 0:1]
def batch_func(self, param):
images_path, masks_path, mask_file_id_hash, data_format = param
file_ids = list(mask_file_id_hash.keys())
shuffle_file_ids = []
resolution = 256
random_flip = True
rotation_range = [-15, 15]
scale_range = [-0.10, 0.95]
tx_range = [-0.3, 0.3]
ty_range = [-0.3, 0.3]
random_bilinear_resize = (25, 75)
motion_blur = (25, 5)
gaussian_blur = (25, 5)
bs = self.batch_size
while True:
batches = None
n_batch = 0
while n_batch < bs:
try:
if len(shuffle_file_ids) == 0:
shuffle_file_ids = file_ids.copy()
np.random.shuffle(shuffle_file_ids)
file_id = shuffle_file_ids.pop()
masks = mask_file_id_hash[file_id]
image_path = images_path / f'{file_id}.jpg'
skin_path = masks.get(MaskType.skin, None)
hair_path = masks.get(MaskType.hair, None)
hat_path = masks.get(MaskType.hat, None)
#neck_path = masks.get(MaskType.neck, None)
img = cv2_imread(image_path).astype(np.float32) / 255.0
mask = cv2_imread(masks_path / skin_path)[..., 0:1].astype(
np.float32) / 255.0
if hair_path is not None:
hair_path = masks_path / hair_path
if hair_path.exists():
hair = cv2_imread(hair_path)[..., 0:1].astype(
np.float32) / 255.0
mask *= (1 - hair)
if hat_path is not None:
hat_path = masks_path / hat_path
if hat_path.exists():
hat = cv2_imread(hat_path)[..., 0:1].astype(
np.float32) / 255.0
mask *= (1 - hat)
#if neck_path is not None:
# neck_path = masks_path / neck_path
# if neck_path.exists():
# neck = cv2_imread(neck_path)[...,0:1].astype(np.float32) / 255.0
# mask = np.clip(mask+neck, 0, 1)
warp_params = imagelib.gen_warp_params(
resolution,
random_flip,
rotation_range=rotation_range,
scale_range=scale_range,
tx_range=tx_range,
ty_range=ty_range)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_LANCZOS4)
h, s, v = cv2.split(cv2.cvtColor(img, cv2.COLOR_BGR2HSV))
h = (h + np.random.randint(360)) % 360
s = np.clip(s + np.random.random() - 0.5, 0, 1)
v = np.clip(v + np.random.random() / 2 - 0.25, 0, 1)
img = np.clip(
cv2.cvtColor(cv2.merge([h, s, v]), cv2.COLOR_HSV2BGR),
0, 1)
if motion_blur is not None:
chance, mb_max_size = motion_blur
chance = np.clip(chance, 0, 100)
mblur_rnd_chance = np.random.randint(100)
mblur_rnd_kernel = np.random.randint(mb_max_size) + 1
mblur_rnd_deg = np.random.randint(360)
if mblur_rnd_chance < chance:
img = imagelib.LinearMotionBlur(
img, mblur_rnd_kernel, mblur_rnd_deg)
img = imagelib.warp_by_params(warp_params,
img,
can_warp=True,
can_transform=True,
can_flip=True,
border_replicate=False,
cv2_inter=cv2.INTER_LANCZOS4)
if gaussian_blur is not None:
chance, kernel_max_size = gaussian_blur
chance = np.clip(chance, 0, 100)
gblur_rnd_chance = np.random.randint(100)
gblur_rnd_kernel = np.random.randint(
kernel_max_size) * 2 + 1
if gblur_rnd_chance < chance:
img = cv2.GaussianBlur(img,
(gblur_rnd_kernel, ) * 2, 0)
if random_bilinear_resize is not None:
chance, max_size_per = random_bilinear_resize
chance = np.clip(chance, 0, 100)
pick_chance = np.random.randint(100)
resize_to = resolution - int(
np.random.rand() * int(resolution *
(max_size_per / 100.0)))
img = cv2.resize(img, (resize_to, resize_to),
cv2.INTER_LINEAR)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_LINEAR)
mask = cv2.resize(mask, (resolution, resolution),
cv2.INTER_LANCZOS4)[..., None]
mask = imagelib.warp_by_params(
warp_params,
mask,
can_warp=True,
can_transform=True,
can_flip=True,
border_replicate=False,
cv2_inter=cv2.INTER_LANCZOS4)
mask[mask < 0.5] = 0.0
mask[mask >= 0.5] = 1.0
mask = np.clip(mask, 0, 1)
if data_format == "NCHW":
img = np.transpose(img, (2, 0, 1))
mask = np.transpose(mask, (2, 0, 1))
if batches is None:
batches = [[], []]
batches[0].append(img)
batches[1].append(mask)
n_batch += 1
except:
io.log_err(traceback.format_exc())
yield [np.array(batch) for batch in batches]
def process(samples,
sample_process_options,
output_sample_types,
debug,
ct_sample=None):
SPTF = SampleProcessor.Types
sample_rnd_seed = np.random.randint(0x80000000)
outputs = []
for sample in samples:
sample_bgr = sample.load_bgr()
ct_sample_bgr = None
ct_sample_mask = None
h, w, c = sample_bgr.shape
is_face_sample = sample.landmarks is not None
if debug and is_face_sample:
LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks,
(0, 1, 0))
params = imagelib.gen_warp_params(
sample_bgr,
sample_process_options.random_flip,
rotation_range=sample_process_options.rotation_range,
scale_range=sample_process_options.scale_range,
tx_range=sample_process_options.tx_range,
ty_range=sample_process_options.ty_range,
rnd_seed=sample_rnd_seed)
outputs_sample = []
for opts in output_sample_types:
resolution = opts.get('resolution', 0)
types = opts.get('types', [])
border_replicate = opts.get('border_replicate', True)
random_sub_res = opts.get('random_sub_res', 0)
normalize_std_dev = opts.get('normalize_std_dev', False)
normalize_vgg = opts.get('normalize_vgg', False)
motion_blur = opts.get('motion_blur', None)
gaussian_blur = opts.get('gaussian_blur', None)
ct_mode = opts.get('ct_mode', 'None')
normalize_tanh = opts.get('normalize_tanh', False)
data_format = opts.get('data_format', 'NHWC')
img_type = SPTF.NONE
target_face_type = SPTF.NONE
face_mask_type = SPTF.NONE
mode_type = SPTF.NONE
for t in types:
if t >= SPTF.IMG_TYPE_BEGIN and t < SPTF.IMG_TYPE_END:
img_type = t
elif t >= SPTF.FACE_TYPE_BEGIN and t < SPTF.FACE_TYPE_END:
target_face_type = t
elif t >= SPTF.MODE_BEGIN and t < SPTF.MODE_END:
mode_type = t
if img_type == SPTF.NONE:
raise ValueError('expected IMG_ type')
if img_type == SPTF.IMG_LANDMARKS_ARRAY:
l = sample.landmarks
l = np.concatenate([
np.expand_dims(l[:, 0] / w, -1),
np.expand_dims(l[:, 1] / h, -1)
], -1)
l = np.clip(l, 0.0, 1.0)
img = l
elif img_type == SPTF.IMG_PITCH_YAW_ROLL or img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch_yaw_roll = sample.get_pitch_yaw_roll()
if params['flip']:
yaw = -yaw
if img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID:
pitch = np.clip((pitch / math.pi) / 2.0 + 1.0, 0, 1)
yaw = np.clip((yaw / math.pi) / 2.0 + 1.0, 0, 1)
roll = np.clip((roll / math.pi) / 2.0 + 1.0, 0, 1)
img = (pitch, yaw, roll)
else:
if mode_type == SPTF.NONE:
raise ValueError('expected MODE_ type')
def do_transform(img, mask):
warp = (img_type == SPTF.IMG_WARPED
or img_type == SPTF.IMG_WARPED_TRANSFORMED)
transform = (img_type == SPTF.IMG_WARPED_TRANSFORMED
or img_type == SPTF.IMG_TRANSFORMED)
flip = img_type != SPTF.IMG_WARPED
img = imagelib.warp_by_params(params, img, warp,
transform, flip,
border_replicate)
if mask is not None:
mask = imagelib.warp_by_params(
params, mask, warp, transform, flip, False)
if len(mask.shape) == 2:
mask = mask[..., np.newaxis]
return img, mask
img = sample_bgr
### Prepare a mask
mask = None
if is_face_sample:
if sample.eyebrows_expand_mod is not None:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape,
sample.landmarks,
eyebrows_expand_mod=sample.eyebrows_expand_mod)
else:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape, sample.landmarks)
if sample.ie_polys is not None:
sample.ie_polys.overlay_mask(mask)
##################
if motion_blur is not None:
chance, mb_max_size = motion_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
img = imagelib.LinearMotionBlur(
img,
np.random.randint(mb_max_size) + 1,
np.random.randint(360))
if gaussian_blur is not None:
chance, kernel_max_size = gaussian_blur
chance = np.clip(chance, 0, 100)
if np.random.randint(100) < chance:
img = cv2.GaussianBlur(
img,
(np.random.randint(kernel_max_size) * 2 + 1, )
* 2, 0)
if is_face_sample and target_face_type != SPTF.NONE:
target_ft = SampleProcessor.SPTF_FACETYPE_TO_FACETYPE[
target_face_type]
if target_ft > sample.face_type:
raise Exception(
'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.'
%
(sample.filename, sample.face_type, target_ft))
if sample.face_type == FaceType.MARK_ONLY:
#first warp to target facetype
img = cv2.warpAffine(
img,
LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0],
target_ft),
(sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
mask = cv2.warpAffine(
mask,
LandmarksProcessor.get_transform_mat(
sample.landmarks, sample.shape[0],
target_ft),
(sample.shape[0], sample.shape[0]),
flags=cv2.INTER_CUBIC)
#then apply transforms
img, mask = do_transform(img, mask)
img = np.concatenate((img, mask), -1)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
else:
img, mask = do_transform(img, mask)
mat = LandmarksProcessor.get_transform_mat(
sample.landmarks, resolution, target_ft)
img = cv2.warpAffine(
img,
mat, (resolution, resolution),
borderMode=(cv2.BORDER_REPLICATE
if border_replicate else
cv2.BORDER_CONSTANT),
flags=cv2.INTER_CUBIC)
mask = cv2.warpAffine(
mask,
mat, (resolution, resolution),
borderMode=cv2.BORDER_CONSTANT,
flags=cv2.INTER_CUBIC)
img = np.concatenate((img, mask[..., None]), -1)
else:
img, mask = do_transform(img, mask)
img = np.concatenate((img, mask), -1)
img = cv2.resize(img, (resolution, resolution),
cv2.INTER_CUBIC)
if random_sub_res != 0:
sub_size = resolution - random_sub_res
rnd_state = np.random.RandomState(sample_rnd_seed +
random_sub_res)
start_x = rnd_state.randint(sub_size + 1)
start_y = rnd_state.randint(sub_size + 1)
img = img[start_y:start_y + sub_size,
start_x:start_x + sub_size, :]
img = np.clip(img, 0, 1).astype(np.float32)
img_bgr = img[..., 0:3]
img_mask = img[..., 3:4]
if ct_mode is not None and ct_sample is not None:
if ct_sample_bgr is None:
ct_sample_bgr = ct_sample.load_bgr()
ct_sample_bgr_resized = cv2.resize(
ct_sample_bgr, (resolution, resolution),
cv2.INTER_LINEAR)
if ct_mode == 'lct':
img_bgr = imagelib.linear_color_transfer(
img_bgr, ct_sample_bgr_resized)
img_bgr = np.clip(img_bgr, 0.0, 1.0)
elif ct_mode == 'rct':
img_bgr = imagelib.reinhard_color_transfer(
np.clip((img_bgr * 255).astype(np.uint8), 0,
255),
np.clip((ct_sample_bgr_resized * 255).astype(
np.uint8), 0, 255))
img_bgr = np.clip(
img_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif ct_mode == 'mkl':
img_bgr = imagelib.color_transfer_mkl(
img_bgr, ct_sample_bgr_resized)
elif ct_mode == 'idt':
img_bgr = imagelib.color_transfer_idt(
img_bgr, ct_sample_bgr_resized)
elif ct_mode == 'sot':
img_bgr = imagelib.color_transfer_sot(
img_bgr, ct_sample_bgr_resized)
img_bgr = np.clip(img_bgr, 0.0, 1.0)
if normalize_std_dev:
img_bgr = (img_bgr - img_bgr.mean(
(0, 1))) / img_bgr.std((0, 1))
elif normalize_vgg:
img_bgr = np.clip(img_bgr * 255, 0, 255)
img_bgr[:, :, 0] -= 103.939
img_bgr[:, :, 1] -= 116.779
img_bgr[:, :, 2] -= 123.68
if mode_type == SPTF.MODE_BGR:
img = img_bgr
elif mode_type == SPTF.MODE_BGR_SHUFFLE:
rnd_state = np.random.RandomState(sample_rnd_seed)
img = np.take(img_bgr,
rnd_state.permutation(img_bgr.shape[-1]),
axis=-1)
elif mode_type == SPTF.MODE_BGR_RANDOM_HSV_SHIFT:
rnd_state = np.random.RandomState(sample_rnd_seed)
hsv = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2HSV)
h, s, v = cv2.split(hsv)
h = (h + rnd_state.randint(360)) % 360
s = np.clip(s + rnd_state.random() - 0.5, 0, 1)
v = np.clip(v + rnd_state.random() - 0.5, 0, 1)
hsv = cv2.merge([h, s, v])
img = np.clip(cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0,
1)
elif mode_type == SPTF.MODE_G:
img = cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY)[...,
None]
elif mode_type == SPTF.MODE_GGG:
img = np.repeat(
np.expand_dims(
cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1),
(3, ), -1)
elif mode_type == SPTF.MODE_M and is_face_sample:
img = img_mask
if not debug:
if normalize_tanh:
img = np.clip(img * 2.0 - 1.0, -1.0, 1.0)
else:
img = np.clip(img, 0.0, 1.0)
if data_format == "NCHW":
img = np.transpose(img, (2, 0, 1))
outputs_sample.append(img)
outputs += [outputs_sample]
return outputs
def MergeMaskedFace(predictor_func, predictor_input_shape, face_enhancer_func,
xseg_256_extract_func, cfg, frame_info, img_bgr_uint8,
img_bgr, img_face_landmarks):
img_size = img_bgr.shape[1], img_bgr.shape[0]
img_face_mask_a = LandmarksProcessor.get_image_hull_mask(
img_bgr.shape, img_face_landmarks)
out_img = img_bgr.copy()
out_merging_mask_a = None
input_size = predictor_input_shape[0]
mask_subres_size = input_size * 4
output_size = input_size
if cfg.super_resolution_power != 0:
output_size *= 4
face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks,
output_size,
face_type=cfg.face_type)
face_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
output_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
if mask_subres_size == output_size:
face_mask_output_mat = face_output_mat
else:
face_mask_output_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks,
mask_subres_size,
face_type=cfg.face_type,
scale=1.0 + 0.01 * cfg.output_face_scale)
dst_face_bgr = cv2.warpAffine(img_bgr,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_bgr = np.clip(dst_face_bgr, 0, 1)
dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a,
face_mat, (output_size, output_size),
flags=cv2.INTER_CUBIC)
dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1)
predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size))
predicted = predictor_func(predictor_input_bgr)
prd_face_bgr = np.clip(predicted[0], 0, 1.0)
prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0)
prd_face_dst_mask_a_0 = np.clip(predicted[2], 0, 1.0)
if cfg.super_resolution_power != 0:
prd_face_bgr_enhanced = face_enhancer_func(prd_face_bgr,
is_tanh=True,
preserve_size=False)
mod = cfg.super_resolution_power / 100.0
prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * (
1.0 - mod) + prd_face_bgr_enhanced * mod
prd_face_bgr = np.clip(prd_face_bgr, 0, 1)
if cfg.super_resolution_power != 0:
prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
prd_face_dst_mask_a_0 = cv2.resize(prd_face_dst_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 1: #dst
wrk_face_mask_a_0 = cv2.resize(dst_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
elif cfg.mask_mode == 2: #learned-prd
wrk_face_mask_a_0 = prd_face_mask_a_0
elif cfg.mask_mode == 3: #learned-dst
wrk_face_mask_a_0 = prd_face_dst_mask_a_0
elif cfg.mask_mode == 4: #learned-prd*learned-dst
wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0
elif cfg.mask_mode == 5: #learned-prd+learned-dst
wrk_face_mask_a_0 = np.clip(prd_face_mask_a_0 + prd_face_dst_mask_a_0,
0, 1)
elif cfg.mask_mode >= 6 and cfg.mask_mode <= 9: #XSeg modes
if cfg.mask_mode == 6 or cfg.mask_mode == 8 or cfg.mask_mode == 9:
# obtain XSeg-prd
prd_face_xseg_bgr = cv2.resize(prd_face_bgr,
(xseg_input_size, ) * 2,
cv2.INTER_CUBIC)
prd_face_xseg_mask = xseg_256_extract_func(prd_face_xseg_bgr)
X_prd_face_mask_a_0 = cv2.resize(prd_face_xseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode >= 7 and cfg.mask_mode <= 9:
# obtain XSeg-dst
xseg_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, xseg_input_size, face_type=cfg.face_type)
dst_face_xseg_bgr = cv2.warpAffine(img_bgr,
xseg_mat,
(xseg_input_size, ) * 2,
flags=cv2.INTER_CUBIC)
dst_face_xseg_mask = xseg_256_extract_func(dst_face_xseg_bgr)
X_dst_face_mask_a_0 = cv2.resize(dst_face_xseg_mask,
(output_size, output_size),
cv2.INTER_CUBIC)
if cfg.mask_mode == 6: #'XSeg-prd'
wrk_face_mask_a_0 = X_prd_face_mask_a_0
elif cfg.mask_mode == 7: #'XSeg-dst'
wrk_face_mask_a_0 = X_dst_face_mask_a_0
elif cfg.mask_mode == 8: #'XSeg-prd*XSeg-dst'
wrk_face_mask_a_0 = X_prd_face_mask_a_0 * X_dst_face_mask_a_0
elif cfg.mask_mode == 9: #learned-prd*learned-dst*XSeg-prd*XSeg-dst
wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0 * X_prd_face_mask_a_0 * X_dst_face_mask_a_0
wrk_face_mask_a_0[wrk_face_mask_a_0 < (1.0 /
255.0)] = 0.0 # get rid of noise
# resize to mask_subres_size
if wrk_face_mask_a_0.shape[0] != mask_subres_size:
wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0,
(mask_subres_size, mask_subres_size),
cv2.INTER_CUBIC)
# process mask in local predicted space
if 'raw' not in cfg.mode:
# add zero pad
wrk_face_mask_a_0 = np.pad(wrk_face_mask_a_0, input_size)
ero = cfg.erode_mask_modifier
blur = cfg.blur_mask_modifier
if ero > 0:
wrk_face_mask_a_0 = cv2.erode(wrk_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (ero, ero)),
iterations=1)
elif ero < 0:
wrk_face_mask_a_0 = cv2.dilate(wrk_face_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE,
(-ero, -ero)),
iterations=1)
# clip eroded/dilated mask in actual predict area
# pad with half blur size in order to accuratelly fade to zero at the boundary
clip_size = input_size + blur // 2
wrk_face_mask_a_0[:clip_size, :] = 0
wrk_face_mask_a_0[-clip_size:, :] = 0
wrk_face_mask_a_0[:, :clip_size] = 0
wrk_face_mask_a_0[:, -clip_size:] = 0
if blur > 0:
blur = blur + (1 - blur % 2)
wrk_face_mask_a_0 = cv2.GaussianBlur(wrk_face_mask_a_0,
(blur, blur), 0)
wrk_face_mask_a_0 = wrk_face_mask_a_0[input_size:-input_size,
input_size:-input_size]
wrk_face_mask_a_0 = np.clip(wrk_face_mask_a_0, 0, 1)
img_face_mask_a = cv2.warpAffine(wrk_face_mask_a_0,
face_mask_output_mat,
img_size,
np.zeros(img_bgr.shape[0:2],
dtype=np.float32),
flags=cv2.WARP_INVERSE_MAP
| cv2.INTER_CUBIC)[..., None]
img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0)
img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise
if wrk_face_mask_a_0.shape[0] != output_size:
wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0,
(output_size, output_size),
cv2.INTER_CUBIC)
wrk_face_mask_a = wrk_face_mask_a_0[..., None]
wrk_face_mask_area_a = wrk_face_mask_a.copy()
wrk_face_mask_area_a[wrk_face_mask_area_a > 0] = 1.0
if cfg.mode == 'original':
return img_bgr, img_face_mask_a
elif 'raw' in cfg.mode:
if cfg.mode == 'raw-rgb':
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_merging_mask_a = img_face_mask_a
elif cfg.mode == 'raw-predict':
out_img = prd_face_bgr
out_merging_mask_a = wrk_face_mask_a
out_img = np.clip(out_img, 0.0, 1.0)
else:
#averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask
ar = []
for i in range(1, 10):
maxregion = np.argwhere(img_face_mask_a > i / 10.0)
if maxregion.size != 0:
miny, minx = maxregion.min(axis=0)[:2]
maxy, maxx = maxregion.max(axis=0)[:2]
lenx = maxx - minx
leny = maxy - miny
if min(lenx, leny) >= 4:
ar += [[lenx, leny]]
if len(ar) > 0:
if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1: #rct
prd_face_bgr = imagelib.reinhard_color_transfer(
np.clip(prd_face_bgr * wrk_face_mask_area_a * 255, 0,
255).astype(np.uint8),
np.clip(dst_face_bgr * wrk_face_mask_area_a * 255, 0,
255).astype(np.uint8),
)
prd_face_bgr = np.clip(
prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
prd_face_bgr = imagelib.linear_color_transfer(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
prd_face_bgr = imagelib.color_transfer_mkl(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
prd_face_bgr = imagelib.color_transfer_idt(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
prd_face_bgr = imagelib.color_transfer_sot(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a,
steps=10,
batch_size=30)
prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
prd_face_bgr = imagelib.color_transfer_mix(
prd_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
if cfg.mode == 'hist-match':
hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ),
dtype=np.float32)
if cfg.masked_hist_match:
hist_mask_a *= wrk_face_mask_area_a
white = (1.0 - hist_mask_a) * np.ones(
prd_face_bgr.shape[:2] + (1, ), dtype=np.float32)
hist_match_1 = prd_face_bgr * hist_mask_a + white
hist_match_1[hist_match_1 > 1.0] = 1.0
hist_match_2 = dst_face_bgr * hist_mask_a + white
hist_match_2[hist_match_1 > 1.0] = 1.0
prd_face_bgr = imagelib.color_hist_match(
hist_match_1, hist_match_2,
cfg.hist_match_threshold).astype(dtype=np.float32)
if 'seamless' in cfg.mode:
#mask used for cv2.seamlessClone
img_face_seamless_mask_a = None
for i in range(1, 10):
a = img_face_mask_a > i / 10.0
if len(np.argwhere(a)) == 0:
continue
img_face_seamless_mask_a = img_face_mask_a.copy()
img_face_seamless_mask_a[a] = 1.0
img_face_seamless_mask_a[img_face_seamless_mask_a <= i /
10.0] = 0.0
break
out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size,
out_img,
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(out_img, 0.0, 1.0)
if 'seamless' in cfg.mode:
try:
#calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering)
l, t, w, h = cv2.boundingRect(
(img_face_seamless_mask_a * 255).astype(np.uint8))
s_maskx, s_masky = int(l + w / 2), int(t + h / 2)
out_img = cv2.seamlessClone(
(out_img * 255).astype(np.uint8), img_bgr_uint8,
(img_face_seamless_mask_a * 255).astype(np.uint8),
(s_maskx, s_masky), cv2.NORMAL_CLONE)
out_img = out_img.astype(dtype=np.float32) / 255.0
except Exception as e:
#seamlessClone may fail in some cases
e_str = traceback.format_exc()
if 'MemoryError' in e_str:
raise Exception(
"Seamless fail: " + e_str
) #reraise MemoryError in order to reprocess this data by other processes
else:
print("Seamless fail: " + e_str)
cfg_mp = 0.3 #todo cfg.motion_blur_power / 100.0
###
shrink_res = output_size #512
shrink_prd_face_dst_mask_a_0 = cv2.resize(prd_face_dst_mask_a_0,
(shrink_res, shrink_res),
cv2.INTER_CUBIC)
shrink_blur_size = (shrink_res // 32) + 1
shrink_blur_size += (1 - shrink_blur_size % 2)
# Feather the mask
shrink_prd_face_dst_mask_a_0 = cv2.GaussianBlur(
shrink_prd_face_dst_mask_a_0,
(shrink_blur_size, shrink_blur_size), 0)
shrink_prd_face_dst_mask_a_0[
shrink_prd_face_dst_mask_a_0 < 0.5] = 0.0
shrink_prd_face_dst_mask_a_0[
shrink_prd_face_dst_mask_a_0 >= 0.5] = 1.0
cnts = cv2.findContours(
shrink_prd_face_dst_mask_a_0.astype(np.uint8), cv2.RETR_LIST,
cv2.CHAIN_APPROX_TC89_KCOS)
# Get the largest found contour
cnt = sorted(cnts[0], key=cv2.contourArea,
reverse=True)[0].squeeze()
l, t = cnt.min(0)
r, b = cnt.max(0)
min_dist_to_edge = min(l, t, r, b)
center = np.mean(cnt, 0)
cnt2 = cnt.copy().astype(np.float32)
cnt2_c = center - cnt2
cnt2_len = npla.norm(cnt2_c, axis=1, keepdims=True)
cnt2_vec = cnt2_c / cnt2_len
# Anchor perimeter
pts_count = shrink_res // 2
h = shrink_res
w = shrink_res
perim_pts = np.concatenate(
(np.concatenate([
np.arange(0, w + w / pts_count, w / pts_count)[..., None],
np.array([[0]] * (pts_count + 1))
],
axis=-1),
np.concatenate([
np.arange(0, w + w / pts_count, w / pts_count)[..., None],
np.array([[h]] * (pts_count + 1))
],
axis=-1),
np.concatenate([
np.array([[0]] * (pts_count + 1)),
np.arange(0, h + h / pts_count, h / pts_count)[..., None]
],
axis=-1),
np.concatenate([
np.array([[w]] * (pts_count + 1)),
np.arange(0, h + h / pts_count, h / pts_count)[..., None]
],
axis=-1)), 0).astype(np.int32)
cnt2 += cnt2_vec * cnt2_len * cfg_mp #todo
cnt2 = cnt2.astype(np.int32)
cnt2 = np.concatenate((cnt2, perim_pts), 0)
cnt = np.concatenate((cnt, perim_pts), 0)
shrink_face_mat = LandmarksProcessor.get_transform_mat(
img_face_landmarks, shrink_res,
face_type=cfg.face_type) #todo check face type
shrink_dst_face_bgr = cv2.warpAffine(img_bgr,
shrink_face_mat,
(shrink_res, shrink_res),
flags=cv2.INTER_CUBIC)
shrinked_dst_face_bgr = mls_rigid_deformation_inv(
shrink_dst_face_bgr, cnt, cnt2)
new_img_bgr = cv2.warpAffine(
shrinked_dst_face_bgr, shrink_face_mat, img_size,
img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
shrink_ero_size = int(min_dist_to_edge * 0.9) #todo
shrink_ero_size += (1 - shrink_ero_size % 2)
shrink_blur_size = int(shrink_ero_size * 1.5)
shrink_blur_size += (1 - shrink_blur_size % 2)
if shrink_ero_size != 0:
shrink_prd_face_dst_mask_a_0_before = shrink_prd_face_dst_mask_a_0.copy(
)
shrink_prd_face_dst_mask_a_0 = cv2.dilate(
shrink_prd_face_dst_mask_a_0,
cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, (shrink_ero_size, shrink_ero_size)),
iterations=1)
shrink_prd_face_dst_mask_a_0 = cv2.GaussianBlur(
shrink_prd_face_dst_mask_a_0,
(shrink_blur_size, shrink_blur_size), 0)
#while True:
# cv2.imshow("", (shrink_prd_face_dst_mask_a_0_before*255).astype(np.uint8) )
# cv2.waitKey(0)
# cv2.imshow("", (shrink_prd_face_dst_mask_a_0*255).astype(np.uint8) )
# cv2.waitKey(0)
shrink_img_mask = cv2.warpAffine(
shrink_prd_face_dst_mask_a_0, shrink_face_mat, img_size,
img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
shrink_img_mask_a = shrink_img_mask[..., None]
new_img_bgr = img_bgr * (1 - shrink_img_mask_a) + (
new_img_bgr * shrink_img_mask_a)
#cv2.imshow("", (shrink_dst_face_bgr*255).astype(np.uint8) )
#cv2.waitKey(0)
#cv2.imshow("", (shrinked_dst_face_bgr*255).astype(np.uint8) )
#cv2.waitKey(0)
while True:
cv2.imshow("", (img_bgr * 255).astype(np.uint8))
cv2.waitKey(0)
cv2.imshow("", (new_img_bgr * 255).astype(np.uint8))
cv2.waitKey(0)
#cv2.imshow("", (shrink_img_mask*255).astype(np.uint8) )
#cv2.waitKey(0)
###
out_img = img_bgr * (1 - img_face_mask_a) + (out_img *
img_face_mask_a)
if ('seamless' in cfg.mode and cfg.color_transfer_mode != 0) or \
cfg.mode == 'seamless-hist-match' or \
cfg_mp != 0 or \
cfg.blursharpen_amount != 0 or \
cfg.image_denoise_power != 0 or \
cfg.bicubic_degrade_power != 0:
out_face_bgr = cv2.warpAffine(out_img,
face_mat,
(output_size, output_size),
flags=cv2.INTER_CUBIC)
if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0:
if cfg.color_transfer_mode == 1:
out_face_bgr = imagelib.reinhard_color_transfer(
np.clip(out_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8),
np.clip(dst_face_bgr * wrk_face_mask_area_a * 255,
0, 255).astype(np.uint8))
out_face_bgr = np.clip(
out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0)
elif cfg.color_transfer_mode == 2: #lct
out_face_bgr = imagelib.linear_color_transfer(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 3: #mkl
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 4: #mkl-m
out_face_bgr = imagelib.color_transfer_mkl(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 5: #idt
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr, dst_face_bgr)
elif cfg.color_transfer_mode == 6: #idt-m
out_face_bgr = imagelib.color_transfer_idt(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
elif cfg.color_transfer_mode == 7: #sot-m
out_face_bgr = imagelib.color_transfer_sot(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a,
steps=10,
batch_size=30)
out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0)
elif cfg.color_transfer_mode == 8: #mix-m
out_face_bgr = imagelib.color_transfer_mix(
out_face_bgr * wrk_face_mask_area_a,
dst_face_bgr * wrk_face_mask_area_a)
if cfg.mode == 'seamless-hist-match':
out_face_bgr = imagelib.color_hist_match(
out_face_bgr, dst_face_bgr, cfg.hist_match_threshold)
if cfg_mp != 0:
k_size = int(frame_info.motion_power * cfg_mp)
if k_size >= 1:
k_size = np.clip(k_size + 1, 2, 50)
if cfg.super_resolution_power != 0:
k_size *= 2
out_face_bgr = imagelib.LinearMotionBlur(
out_face_bgr, k_size, frame_info.motion_deg)
if cfg.blursharpen_amount != 0:
out_face_bgr = imagelib.blursharpen(
out_face_bgr, cfg.sharpen_mode, 3,
cfg.blursharpen_amount)
if cfg.image_denoise_power != 0:
n = cfg.image_denoise_power
while n > 0:
img_bgr_denoised = cv2.medianBlur(img_bgr, 5)
if int(n / 100) != 0:
img_bgr = img_bgr_denoised
else:
pass_power = (n % 100) / 100.0
img_bgr = img_bgr * (
1.0 -
pass_power) + img_bgr_denoised * pass_power
n = max(n - 10, 0)
if cfg.bicubic_degrade_power != 0:
p = 1.0 - cfg.bicubic_degrade_power / 101.0
img_bgr_downscaled = cv2.resize(
img_bgr, (int(img_size[0] * p), int(img_size[1] * p)),
cv2.INTER_CUBIC)
img_bgr = cv2.resize(img_bgr_downscaled, img_size,
cv2.INTER_CUBIC)
new_out = cv2.warpAffine(
out_face_bgr, face_mat, img_size, img_bgr.copy(),
cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC,
cv2.BORDER_TRANSPARENT)
out_img = np.clip(
img_bgr * (1 - img_face_mask_a) +
(new_out * img_face_mask_a), 0, 1.0)
if cfg.color_degrade_power != 0:
out_img_reduced = imagelib.reduce_colors(out_img, 256)
if cfg.color_degrade_power == 100:
out_img = out_img_reduced
else:
alpha = cfg.color_degrade_power / 100.0
out_img = (out_img * (1.0 - alpha) +
out_img_reduced * alpha)
out_merging_mask_a = img_face_mask_a
return out_img, out_merging_mask_a
