def get_full_face_mask(): if sample.eyebrows_expand_mod is not None: full_face_mask = LandmarksProcessor.get_image_hull_mask( sample_bgr.shape, sample_landmarks, eyebrows_expand_mod=sample.eyebrows_expand_mod) else: full_face_mask = LandmarksProcessor.get_image_hull_mask( sample_bgr.shape, sample_landmarks) return np.clip(full_face_mask, 0, 1)
def onClientProcessData(self, data): filepath = Path(data[0]) try: if filepath.suffix != '.png': raise Exception( "%s is not a png file required for sort_final" % (filepath.name)) dflpng = DFLPNG.load(str(filepath), print_on_no_embedded_data=True) if dflpng is None: raise Exception("") bgr = cv2.imread(str(filepath)) if bgr is None: raise Exception("Unable to load %s" % (filepath.name)) gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) gray_masked = (gray * LandmarksProcessor.get_image_hull_mask( bgr.shape, dflpng.get_landmarks())[:, :, 0]).astype(np.uint8) sharpness = estimate_sharpness(gray_masked) hist = cv2.calcHist([gray], [0], None, [256], [0, 256]) except Exception as e: print(e) return [1, [str(filepath)]] return [0, [str(filepath), sharpness, hist, dflpng.get_yaw_value()]]
def sort_by_hist_dissim(input_path): print("Sorting by histogram dissimilarity...") img_list = [] for filename_path in tqdm(Path_utils.get_image_paths(input_path), desc="Loading"): image = cv2.imread(filename_path) dflpng = DFLPNG.load(str(filename_path)) if dflpng is not None: face_mask = LandmarksProcessor.get_image_hull_mask( image, dflpng.get_landmarks()) image = (image * face_mask).astype(np.uint8) img_list.append([ filename_path, cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None, [256], [0, 256]), 0 ]) img_list = HistDissimSubprocessor(img_list).process() print("Sorting...") img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True) return img_list
def process_data(self, data): filepath = Path(data[0]) try: if filepath.suffix == '.png': dflimg = DFLPNG.load( str(filepath) ) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load( str(filepath) ) else: dflimg = None if dflimg is None: self.log_err("%s is not a dfl image file" % (filepath.name)) return [ 1, [str(filepath)] ] bgr = cv2_imread(str(filepath)) if bgr is None: raise Exception ("Unable to load %s" % (filepath.name) ) gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) gray_masked = ( gray * LandmarksProcessor.get_image_hull_mask (bgr.shape, dflimg.get_landmarks() )[:,:,0] ).astype(np.uint8) sharpness = estimate_sharpness(gray_masked) pitch, yaw = LandmarksProcessor.estimate_pitch_yaw ( dflimg.get_landmarks() ) hist = cv2.calcHist([gray], [0], None, [256], [0, 256]) except Exception as e: self.log_err (e) return [ 1, [str(filepath)] ] return [ 0, [str(filepath), sharpness, hist, yaw ] ]
def sort_by_hist_dissim(input_path): io.log_info("Sorting by histogram dissimilarity...") img_list = [] trash_img_list = [] for filepath in io.progress_bar_generator( pathex.get_image_paths(input_path), "Loading"): filepath = Path(filepath) dflimg = DFLIMG.load(filepath) image = cv2_imread(str(filepath)) if dflimg is not None: face_mask = LandmarksProcessor.get_image_hull_mask( image.shape, dflimg.get_landmarks()) image = (image * face_mask).astype(np.uint8) img_list.append([ str(filepath), cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None, [256], [0, 256]), 0 ]) img_list = HistDissimSubprocessor(img_list).run() io.log_info("Sorting...") img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True) return img_list, trash_img_list
def sort_by_hist_dissim(input_path): io.log_info ("Sorting by histogram dissimilarity...") img_list = [] trash_img_list = [] for filepath in io.progress_bar_generator( Path_utils.get_image_paths(input_path), "Loading"): filepath = Path(filepath) if filepath.suffix == '.png': dflimg = DFLPNG.load( str(filepath) ) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load ( str(filepath) ) else: dflimg = None if dflimg is None: io.log_err ("%s is not a dfl image file" % (filepath.name) ) trash_img_list.append ([str(filepath)]) continue image = cv2_imread(str(filepath)) face_mask = LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks()) image = (image*face_mask).astype(np.uint8) img_list.append ([str(filepath), cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None, [256], [0, 256]), 0 ]) img_list = HistDissimSubprocessor(img_list).run() io.log_info ("Sorting...") img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True) return img_list, trash_img_list
def onProcessSample(self, sample, debug): image = sample.load_image() if debug: LandmarksProcessor.draw_landmarks(image, sample.landmarks, (0, 1, 0)) hull_mask = LandmarksProcessor.get_image_hull_mask( image, sample.landmarks) s_image = np.concatenate((image, hull_mask), axis=2) warped_img, target_img = HalfFaceTrainingDataGenerator.warp( s_image, sample.landmarks, self.warped_size, self.target_size) if self.random_flip and np.random.randint(2) == 1: warped_img = warped_img[:, ::-1, :] target_img = target_img[:, ::-1, :] if debug: return (self.target_size[1], image, target_img, warped_img[..., 0:3], warped_img[..., 0:3] * np.expand_dims(warped_img[..., 3], 2)) else: return (warped_img, target_img)
def sort_by_hist_dissim(input_path): print("Sorting by histogram dissimilarity...") img_list = [] for filepath in tqdm(Path_utils.get_image_paths(input_path), desc="Loading", ascii=True): filepath = Path(filepath) if filepath.suffix == '.png': dflimg = DFLPNG.load(str(filepath), print_on_no_embedded_data=True) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load(str(filepath), print_on_no_embedded_data=True) else: print("%s is not a dfl image file" % (filepath.name)) continue image = cv2.imread(str(filepath)) face_mask = LandmarksProcessor.get_image_hull_mask( image.shape, dflimg.get_landmarks()) image = (image * face_mask).astype(np.uint8) img_list.append([ str(filepath), cv2.calcHist([cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)], [0], None, [256], [0, 256]), 0 ]) img_list = HistDissimSubprocessor(img_list).process() print("Sorting...") img_list = sorted(img_list, key=operator.itemgetter(2), reverse=True) return img_list
def cli_convert_face (self, img_bgr, img_face_landmarks, debug, avaperator_face_bgr=None, **kwargs): if debug: debugs = [img_bgr.copy()] img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask (img_bgr.shape, img_face_landmarks) img_face_mask_aaa = np.repeat(img_face_mask_a, 3, -1) output_size = self.predictor_input_size face_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, output_size, face_type=FaceType.FULL) dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC ) predictor_input_dst_face_mask_a_0 = cv2.resize (dst_face_mask_a_0, (self.predictor_input_size,self.predictor_input_size), cv2.INTER_CUBIC ) prd_inp_dst_face_mask_a = predictor_input_dst_face_mask_a_0[...,np.newaxis] prd_inp_avaperator_face_bgr = cv2.resize (avaperator_face_bgr, (self.predictor_input_size,self.predictor_input_size), cv2.INTER_CUBIC ) prd_face_bgr = self.predictor_func ( prd_inp_avaperator_face_bgr, prd_inp_dst_face_mask_a ) out_img = img_bgr.copy() out_img = cv2.warpAffine( prd_face_bgr, face_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ) out_img = np.clip(out_img, 0.0, 1.0) if debug: debugs += [out_img.copy()] out_img = np.clip( img_bgr*(1-img_face_mask_aaa) + (out_img*img_face_mask_aaa) , 0, 1.0 ) if debug: debugs += [out_img.copy()] return debugs if debug else out_img
def process_data(self, data): filepath = Path(data[0]) try: dflimg = DFLIMG.load(filepath) if dflimg is None or not dflimg.has_data(): self.log_err(f"{filepath.name} is not a dfl image file") return [1, [str(filepath)]] bgr = cv2_imread(str(filepath)) if bgr is None: raise Exception("Unable to load %s" % (filepath.name)) gray = cv2.cvtColor(bgr, cv2.COLOR_BGR2GRAY) if self.faster: source_rect = dflimg.get_source_rect() sharpness = mathlib.polygon_area( np.array(source_rect[[0, 2, 2, 0]]).astype(np.float32), np.array(source_rect[[1, 1, 3, 3]]).astype(np.float32)) else: face_mask = LandmarksProcessor.get_image_hull_mask( gray.shape, dflimg.get_landmarks()) sharpness = estimate_sharpness( (gray[..., None] * face_mask).astype(np.uint8)) pitch, yaw, roll = LandmarksProcessor.estimate_pitch_yaw_roll( dflimg.get_landmarks(), size=dflimg.get_shape()[1]) hist = cv2.calcHist([gray], [0], None, [256], [0, 256]) except Exception as e: self.log_err(e) return [1, [str(filepath)]] return [0, [str(filepath), sharpness, hist, yaw, pitch]]
def apply_celebamaskhq(input_dir ): input_path = Path(input_dir) img_path = input_path / 'aligned' mask_path = input_path / 'mask' if not img_path.exists(): raise ValueError(f'{str(img_path)} directory not found. Please ensure it exists.') CelebAMASKHQSubprocessor(pathex.get_image_paths(img_path), pathex.get_image_paths(mask_path, subdirs=True) ).run() return paths_to_extract = [] for filename in io.progress_bar_generator(pathex.get_image_paths(img_path), desc="Processing"): filepath = Path(filename) dflimg = DFLIMG.load(filepath) if dflimg is not None: paths_to_extract.append (filepath) image_to_face_mat = dflimg.get_image_to_face_mat() src_filename = dflimg.get_source_filename() #img = cv2_imread(filename) h,w,c = dflimg.get_shape() fanseg_mask = LandmarksProcessor.get_image_hull_mask( (h,w,c), dflimg.get_landmarks() ) idx_name = '%.5d' % int(src_filename.split('.')[0]) idx_files = [ x for x in masks_files if idx_name in x ] skin_files = [ x for x in idx_files if 'skin' in x ] eye_glass_files = [ x for x in idx_files if 'eye_g' in x ] for files, is_invert in [ (skin_files,False), (eye_glass_files,True) ]: if len(files) > 0: mask = cv2_imread(files[0]) mask = mask[...,0] mask[mask == 255] = 1 mask = mask.astype(np.float32) mask = cv2.resize(mask, (1024,1024) ) mask = cv2.warpAffine(mask, image_to_face_mat, (w, h), cv2.INTER_LANCZOS4) if not is_invert: fanseg_mask *= mask[...,None] else: fanseg_mask *= (1-mask[...,None]) #cv2.imshow("", (fanseg_mask*255).astype(np.uint8) ) #cv2.waitKey(0) dflimg.embed_and_set (filename, fanseg_mask=fanseg_mask)
def get_full_face_mask(): xseg_mask = sample.get_xseg_mask() if xseg_mask is not None: if xseg_mask.shape[0] != h or xseg_mask.shape[1] != w: xseg_mask = cv2.resize(xseg_mask, (w,h), interpolation=cv2.INTER_CUBIC) xseg_mask = imagelib.normalize_channels(xseg_mask, 1) return np.clip(xseg_mask, 0, 1) else: full_face_mask = LandmarksProcessor.get_image_hull_mask (sample_bgr.shape, sample_landmarks, eyebrows_expand_mod=sample.eyebrows_expand_mod ) return np.clip(full_face_mask, 0, 1)
def onClientProcessData(self, data): filename_path = Path( data[0] ) dflpng = DFLPNG.load( str(filename_path), print_on_no_embedded_data=True ) if dflpng is not None: image = cv2.imread( str(filename_path) ) image = ( image * \ LandmarksProcessor.get_image_hull_mask (image, dflpng.get_landmarks()) \ ).astype(np.uint8) return [ str(filename_path), estimate_sharpness( image ) ] else: return [ str(filename_path), 0 ]
def process_data(self, data): filepath = Path( data[0] ) dflimg = DFLIMG.load (filepath) if dflimg is None or not dflimg.has_data(): self.log_err (f"{filepath.name} is not a dfl image file") return [ str(filepath), 0 ] else: image = cv2_imread( str(filepath) ) face_mask = LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks()) image = (image*face_mask).astype(np.uint8) return [ str(filepath), estimate_sharpness(image) ]
def onClientProcessData(self, data): filepath = Path(data[0]) if filepath.suffix == '.png': dflimg = DFLPNG.load(str(filepath), print_on_no_embedded_data=True) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load(str(filepath), print_on_no_embedded_data=True) else: print("%s is not a dfl image file" % (filepath.name)) dflimg = None if dflimg is not None: image = cv2.imread(str(filepath)) image = ( image * \ LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks()) \ ).astype(np.uint8) return [str(filepath), estimate_sharpness(image)] else: return [str(filepath), 0]
def process_data(self, data): filepath = Path( data[0] ) if filepath.suffix == '.png': dflimg = DFLPNG.load( str(filepath) ) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load ( str(filepath) ) else: dflimg = None if dflimg is not None: image = cv2_imread( str(filepath) ) image = ( image * \ LandmarksProcessor.get_image_hull_mask (image.shape, dflimg.get_landmarks()) \ ).astype(np.uint8) return [ str(filepath), estimate_sharpness( image ) ] else: self.log_err ("%s is not a dfl image file" % (filepath.name) ) return [ str(filepath), 0 ]
def process_data(self, data): filename = data[0] filepath = Path(filename) if filepath.suffix == '.png': dflimg = DFLPNG.load(str(filepath)) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load(str(filepath)) else: dflimg = None image_to_face_mat = dflimg.get_image_to_face_mat() src_filename = dflimg.get_source_filename() img = cv2_imread(filename) h, w, c = img.shape fanseg_mask = LandmarksProcessor.get_image_hull_mask( img.shape, dflimg.get_landmarks()) idx_name = '%.5d' % int(src_filename.split('.')[0]) idx_files = [x for x in self.masks_files_paths if idx_name in x] skin_files = [x for x in idx_files if 'skin' in x] eye_glass_files = [x for x in idx_files if 'eye_g' in x] for files, is_invert in [(skin_files, False), (eye_glass_files, True)]: if len(files) > 0: mask = cv2_imread(files[0]) mask = mask[..., 0] mask[mask == 255] = 1 mask = mask.astype(np.float32) mask = cv2.resize(mask, (1024, 1024)) mask = cv2.warpAffine(mask, image_to_face_mat, (w, h), cv2.INTER_LANCZOS4) if not is_invert: fanseg_mask *= mask[..., None] else: fanseg_mask *= (1 - mask[..., None]) dflimg.embed_and_set(filename, fanseg_mask=fanseg_mask) return 1
def process_data(self, data): filepath = Path(data[0]) dflimg = DFLIMG.load(filepath) if dflimg is None or not dflimg.has_data(): self.log_err(f"{filepath.name} is not a dfl image file") return [str(filepath), 0] else: image = cv2_imread(str(filepath)) face_mask = LandmarksProcessor.get_image_hull_mask( image.shape, dflimg.get_landmarks()) image = (image * face_mask).astype(np.uint8) if self.estimate_motion_blur: value = cv2.Laplacian(image, cv2.CV_64F, ksize=11).var() else: value = estimate_sharpness(image) return [str(filepath), value]
def convert_face(self, img_bgr, img_face_landmarks, debug): if debug: debugs = [img_bgr.copy()] img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask( img_bgr, img_face_landmarks) face_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, self.output_size, face_type=self.face_type) face_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, self.output_size, face_type=self.face_type, scale=self.output_face_scale) dst_face_bgr = cv2.warpAffine(img_bgr, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4) dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4) predictor_input_bgr = cv2.resize( dst_face_bgr, (self.predictor_input_size, self.predictor_input_size)) predictor_input_mask_a_0 = cv2.resize( dst_face_mask_a_0, (self.predictor_input_size, self.predictor_input_size)) predictor_input_mask_a = np.expand_dims(predictor_input_mask_a_0, -1) predicted_bgra = self.predictor( np.concatenate((predictor_input_bgr, predictor_input_mask_a), -1)) prd_face_bgr = np.clip(predicted_bgra[:, :, 0:3], 0, 1.0) prd_face_mask_a_0 = np.clip(predicted_bgra[:, :, 3], 0.0, 1.0) prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0 prd_face_mask_a = np.expand_dims(prd_face_mask_a_0, axis=-1) prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1) img_prd_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=float), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4) img_prd_face_mask_aaa = np.clip(img_prd_face_mask_aaa, 0.0, 1.0) img_face_mask_aaa = img_prd_face_mask_aaa if debug: debugs += [img_face_mask_aaa.copy()] img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 img_face_mask_flatten_aaa = img_face_mask_aaa.copy() img_face_mask_flatten_aaa[img_face_mask_flatten_aaa > 0.9] = 1.0 maxregion = np.argwhere(img_face_mask_flatten_aaa == 1.0) out_img = img_bgr.copy() if maxregion.size != 0: miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] if debug: print("maxregion.size: %d, minx:%d, maxx:%d miny:%d, maxy:%d" % (maxregion.size, minx, maxx, miny, maxy)) lenx = maxx - minx leny = maxy - miny if lenx >= 4 and leny >= 4: masky = int(minx + (lenx // 2)) maskx = int(miny + (leny // 2)) lowest_len = min(lenx, leny) if debug: print("lowest_len = %f" % (lowest_len)) ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) * 0.01 * self.erode_mask_modifier) blur = int(lowest_len * 0.10 * 0.01 * self.blur_mask_modifier) if debug: print("erode_size = %d, blur_size = %d" % (ero, blur)) img_mask_blurry_aaa = img_face_mask_aaa if self.erode_mask: if ero > 0: img_mask_blurry_aaa = cv2.erode( img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: img_mask_blurry_aaa = cv2.dilate( img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) if self.blur_mask and blur > 0: img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa, (blur, blur)) img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0) if self.clip_border_mask_per > 0: prd_border_rect_mask_a = np.ones( prd_face_mask_a.shape, dtype=prd_face_mask_a.dtype) prd_border_size = int(prd_border_rect_mask_a.shape[1] * self.clip_border_mask_per) prd_border_rect_mask_a[0:prd_border_size, :, :] = 0 prd_border_rect_mask_a[-prd_border_size:, :, :] = 0 prd_border_rect_mask_a[:, 0:prd_border_size, :] = 0 prd_border_rect_mask_a[:, -prd_border_size:, :] = 0 prd_border_rect_mask_a = np.expand_dims( cv2.blur(prd_border_rect_mask_a, (prd_border_size, prd_border_size)), -1) if self.mode == 'hist-match-bw': prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY) prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1), (3, ), -1) if self.mode == 'hist-match' or self.mode == 'hist-match-bw': if debug: debugs += [ cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) ] hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=prd_face_bgr.dtype) if self.masked_hist_match: hist_mask_a *= prd_face_mask_a hist_match_1 = prd_face_bgr * hist_mask_a + ( 1.0 - hist_mask_a) * np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=prd_face_bgr.dtype) hist_match_1[hist_match_1 > 1.0] = 1.0 hist_match_2 = dst_face_bgr * hist_mask_a + ( 1.0 - hist_mask_a) * np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=prd_face_bgr.dtype) hist_match_2[hist_match_1 > 1.0] = 1.0 new_prd_face_bgr = image_utils.color_hist_match( hist_match_1, hist_match_2) prd_face_bgr = new_prd_face_bgr if self.mode == 'hist-match-bw': prd_face_bgr = prd_face_bgr.astype(np.float32) out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) if debug: debugs += [out_img.copy()] debugs += [img_mask_blurry_aaa.copy()] if self.mode == 'seamless' or self.mode == 'seamless-hist-match': out_img = np.clip( img_bgr * (1 - img_face_mask_aaa) + (out_img * img_face_mask_aaa), 0, 1.0) if debug: debugs += [out_img.copy()] out_img = cv2.seamlessClone( (out_img * 255).astype(np.uint8), (img_bgr * 255).astype(np.uint8), (img_face_mask_flatten_aaa * 255).astype(np.uint8), (masky, maskx), cv2.NORMAL_CLONE) out_img = out_img.astype(np.float32) / 255.0 if debug: debugs += [out_img.copy()] if self.clip_border_mask_per > 0: img_prd_border_rect_mask_a = cv2.warpAffine( prd_border_rect_mask_a, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) img_prd_border_rect_mask_a = np.expand_dims( img_prd_border_rect_mask_a, -1) out_img = out_img * img_prd_border_rect_mask_a + img_bgr * ( 1.0 - img_prd_border_rect_mask_a) img_mask_blurry_aaa *= img_prd_border_rect_mask_a out_img = np.clip( img_bgr * (1 - img_mask_blurry_aaa) + (out_img * img_mask_blurry_aaa), 0, 1.0) if self.mode == 'seamless-hist-match': out_face_bgr = cv2.warpAffine( out_img, face_mat, (self.output_size, self.output_size)) new_out_face_bgr = image_utils.color_hist_match( out_face_bgr, dst_face_bgr) new_out = cv2.warpAffine( new_out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) out_img = np.clip( img_bgr * (1 - img_mask_blurry_aaa) + (new_out * img_mask_blurry_aaa), 0, 1.0) if self.transfercolor: if self.TFLabConverter is None: self.TFLabConverter = image_utils.TFLabConverter() img_lab_l, img_lab_a, img_lab_b = np.split( self.TFLabConverter.bgr2lab(img_bgr), 3, axis=-1) out_img_lab_l, out_img_lab_a, out_img_lab_b = np.split( self.TFLabConverter.bgr2lab(out_img), 3, axis=-1) out_img = self.TFLabConverter.lab2bgr( np.concatenate([out_img_lab_l, img_lab_a, img_lab_b], axis=-1)) if self.final_image_color_degrade_power != 0: if debug: debugs += [out_img.copy()] out_img_reduced = image_utils.reduce_colors(out_img, 256) if self.final_image_color_degrade_power == 100: out_img = out_img_reduced else: alpha = self.final_image_color_degrade_power / 100.0 out_img = (out_img * (1.0 - alpha) + out_img_reduced * alpha) if self.alpha: new_image = out_img.copy() new_image = (new_image * 255).astype( np.uint8) #convert image to int b_channel, g_channel, r_channel = cv2.split( new_image) #splitting RGB alpha_channel = img_mask_blurry_aaa.copy( ) #making copy of alpha channel alpha_channel = (alpha_channel * 255).astype(np.uint8) alpha_channel, tmp2, tmp3 = cv2.split( alpha_channel ) #splitting alpha to three channels, they all same in original alpha channel, we need just one out_img = cv2.merge( (b_channel, g_channel, r_channel, alpha_channel)) #mergin RGB with alpha out_img = out_img.astype(np.float32) / 255.0 if debug: debugs += [out_img.copy()] return debugs if debug else out_img
def MergeMaskedFace(predictor_func, predictor_input_shape, face_enhancer_func, xseg_256_extract_func, cfg, frame_info, img_bgr_uint8, img_bgr, img_face_landmarks): img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask( img_bgr.shape, img_face_landmarks) input_size = predictor_input_shape[0] mask_subres_size = input_size * 4 output_size = input_size if cfg.super_resolution_power != 0: output_size *= 4 face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks, output_size, face_type=cfg.face_type) face_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, output_size, face_type=cfg.face_type, scale=1.0 + 0.01 * cfg.output_face_scale) if mask_subres_size == output_size: face_mask_output_mat = face_output_mat else: face_mask_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, mask_subres_size, face_type=cfg.face_type, scale=1.0 + 0.01 * cfg.output_face_scale) dst_face_bgr = cv2.warpAffine(img_bgr, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) dst_face_bgr = np.clip(dst_face_bgr, 0, 1) dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1) predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size)) predicted = predictor_func(predictor_input_bgr) prd_face_bgr = np.clip(predicted[0], 0, 1.0) prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0) prd_face_dst_mask_a_0 = np.clip(predicted[2], 0, 1.0) if cfg.super_resolution_power != 0: prd_face_bgr_enhanced = face_enhancer_func(prd_face_bgr, is_tanh=True, preserve_size=False) mod = cfg.super_resolution_power / 100.0 prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * ( 1.0 - mod) + prd_face_bgr_enhanced * mod prd_face_bgr = np.clip(prd_face_bgr, 0, 1) if cfg.super_resolution_power != 0: prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0, (output_size, output_size), interpolation=cv2.INTER_CUBIC) prd_face_dst_mask_a_0 = cv2.resize(prd_face_dst_mask_a_0, (output_size, output_size), interpolation=cv2.INTER_CUBIC) if cfg.mask_mode == 1: #dst wrk_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, (output_size, output_size), interpolation=cv2.INTER_CUBIC) elif cfg.mask_mode == 2: #learned-prd wrk_face_mask_a_0 = prd_face_mask_a_0 elif cfg.mask_mode == 3: #learned-dst wrk_face_mask_a_0 = prd_face_dst_mask_a_0 elif cfg.mask_mode == 4: #learned-prd*learned-dst wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0 elif cfg.mask_mode == 5: #learned-prd+learned-dst wrk_face_mask_a_0 = np.clip(prd_face_mask_a_0 + prd_face_dst_mask_a_0, 0, 1) elif cfg.mask_mode >= 6 and cfg.mask_mode <= 9: #XSeg modes if cfg.mask_mode == 6 or cfg.mask_mode == 8 or cfg.mask_mode == 9: # obtain XSeg-prd prd_face_xseg_bgr = cv2.resize(prd_face_bgr, (xseg_input_size, ) * 2, interpolation=cv2.INTER_CUBIC) prd_face_xseg_mask = xseg_256_extract_func(prd_face_xseg_bgr) X_prd_face_mask_a_0 = cv2.resize(prd_face_xseg_mask, (output_size, output_size), interpolation=cv2.INTER_CUBIC) if cfg.mask_mode >= 7 and cfg.mask_mode <= 9: # obtain XSeg-dst xseg_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, xseg_input_size, face_type=cfg.face_type) dst_face_xseg_bgr = cv2.warpAffine(img_bgr, xseg_mat, (xseg_input_size, ) * 2, flags=cv2.INTER_CUBIC) dst_face_xseg_mask = xseg_256_extract_func(dst_face_xseg_bgr) X_dst_face_mask_a_0 = cv2.resize(dst_face_xseg_mask, (output_size, output_size), interpolation=cv2.INTER_CUBIC) if cfg.mask_mode == 6: #'XSeg-prd' wrk_face_mask_a_0 = X_prd_face_mask_a_0 elif cfg.mask_mode == 7: #'XSeg-dst' wrk_face_mask_a_0 = X_dst_face_mask_a_0 elif cfg.mask_mode == 8: #'XSeg-prd*XSeg-dst' wrk_face_mask_a_0 = X_prd_face_mask_a_0 * X_dst_face_mask_a_0 elif cfg.mask_mode == 9: #learned-prd*learned-dst*XSeg-prd*XSeg-dst wrk_face_mask_a_0 = prd_face_mask_a_0 * prd_face_dst_mask_a_0 * X_prd_face_mask_a_0 * X_dst_face_mask_a_0 wrk_face_mask_a_0[wrk_face_mask_a_0 < (1.0 / 255.0)] = 0.0 # get rid of noise # resize to mask_subres_size if wrk_face_mask_a_0.shape[0] != mask_subres_size: wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0, (mask_subres_size, mask_subres_size), interpolation=cv2.INTER_CUBIC) # process mask in local predicted space if 'raw' not in cfg.mode: # add zero pad wrk_face_mask_a_0 = np.pad(wrk_face_mask_a_0, input_size) ero = cfg.erode_mask_modifier blur = cfg.blur_mask_modifier if ero > 0: wrk_face_mask_a_0 = cv2.erode(wrk_face_mask_a_0, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: wrk_face_mask_a_0 = cv2.dilate(wrk_face_mask_a_0, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) # clip eroded/dilated mask in actual predict area # pad with half blur size in order to accuratelly fade to zero at the boundary clip_size = input_size + blur // 2 wrk_face_mask_a_0[:clip_size, :] = 0 wrk_face_mask_a_0[-clip_size:, :] = 0 wrk_face_mask_a_0[:, :clip_size] = 0 wrk_face_mask_a_0[:, -clip_size:] = 0 if blur > 0: blur = blur + (1 - blur % 2) wrk_face_mask_a_0 = cv2.GaussianBlur(wrk_face_mask_a_0, (blur, blur), 0) wrk_face_mask_a_0 = wrk_face_mask_a_0[input_size:-input_size, input_size:-input_size] wrk_face_mask_a_0 = np.clip(wrk_face_mask_a_0, 0, 1) img_face_mask_a = cv2.warpAffine(wrk_face_mask_a_0, face_mask_output_mat, img_size, np.zeros(img_bgr.shape[0:2], dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)[..., None] img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0) img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise if wrk_face_mask_a_0.shape[0] != output_size: wrk_face_mask_a_0 = cv2.resize(wrk_face_mask_a_0, (output_size, output_size), interpolation=cv2.INTER_CUBIC) wrk_face_mask_a = wrk_face_mask_a_0[..., None] out_img = None out_merging_mask_a = None if cfg.mode == 'original': return img_bgr, img_face_mask_a elif 'raw' in cfg.mode: if cfg.mode == 'raw-rgb': out_img_face = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC) out_img_face_mask = cv2.warpAffine( np.ones_like(prd_face_bgr), face_output_mat, img_size, np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC) out_img = img_bgr * ( 1 - out_img_face_mask) + out_img_face * out_img_face_mask out_merging_mask_a = img_face_mask_a elif cfg.mode == 'raw-predict': out_img = prd_face_bgr out_merging_mask_a = wrk_face_mask_a else: raise ValueError(f"undefined raw type {cfg.mode}") out_img = np.clip(out_img, 0.0, 1.0) else: # Process if the mask meets minimum size maxregion = np.argwhere(img_face_mask_a >= 0.1) if maxregion.size != 0: miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx leny = maxy - miny if min(lenx, leny) >= 4: wrk_face_mask_area_a = wrk_face_mask_a.copy() wrk_face_mask_area_a[wrk_face_mask_area_a > 0] = 1.0 if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0: if cfg.color_transfer_mode == 1: #rct prd_face_bgr = imagelib.reinhard_color_transfer( np.clip(prd_face_bgr * wrk_face_mask_area_a * 255, 0, 255).astype(np.uint8), np.clip(dst_face_bgr * wrk_face_mask_area_a * 255, 0, 255).astype(np.uint8), ) prd_face_bgr = np.clip( prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) elif cfg.color_transfer_mode == 2: #lct prd_face_bgr = imagelib.linear_color_transfer( prd_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 3: #mkl prd_face_bgr = imagelib.color_transfer_mkl( prd_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 4: #mkl-m prd_face_bgr = imagelib.color_transfer_mkl( prd_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a) elif cfg.color_transfer_mode == 5: #idt prd_face_bgr = imagelib.color_transfer_idt( prd_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 6: #idt-m prd_face_bgr = imagelib.color_transfer_idt( prd_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a) elif cfg.color_transfer_mode == 7: #sot-m prd_face_bgr = imagelib.color_transfer_sot( prd_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a, steps=10, batch_size=30) prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0) elif cfg.color_transfer_mode == 8: #mix-m prd_face_bgr = imagelib.color_transfer_mix( prd_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a) if cfg.mode == 'hist-match': hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) if cfg.masked_hist_match: hist_mask_a *= wrk_face_mask_area_a white = (1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_1 = prd_face_bgr * hist_mask_a + white hist_match_1[hist_match_1 > 1.0] = 1.0 hist_match_2 = dst_face_bgr * hist_mask_a + white hist_match_2[hist_match_1 > 1.0] = 1.0 prd_face_bgr = imagelib.color_hist_match( hist_match_1, hist_match_2, cfg.hist_match_threshold).astype(dtype=np.float32) if 'seamless' in cfg.mode: #mask used for cv2.seamlessClone img_face_seamless_mask_a = None for i in range(1, 10): a = img_face_mask_a > i / 10.0 if len(np.argwhere(a)) == 0: continue img_face_seamless_mask_a = img_face_mask_a.copy() img_face_seamless_mask_a[a] = 1.0 img_face_seamless_mask_a[ img_face_seamless_mask_a <= i / 10.0] = 0.0 break out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC) out_img = np.clip(out_img, 0.0, 1.0) if 'seamless' in cfg.mode: try: #calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering) l, t, w, h = cv2.boundingRect( (img_face_seamless_mask_a * 255).astype(np.uint8)) s_maskx, s_masky = int(l + w / 2), int(t + h / 2) out_img = cv2.seamlessClone( (out_img * 255).astype(np.uint8), img_bgr_uint8, (img_face_seamless_mask_a * 255).astype(np.uint8), (s_maskx, s_masky), cv2.NORMAL_CLONE) out_img = out_img.astype(dtype=np.float32) / 255.0 except Exception as e: #seamlessClone may fail in some cases e_str = traceback.format_exc() if 'MemoryError' in e_str: raise Exception( "Seamless fail: " + e_str ) #reraise MemoryError in order to reprocess this data by other processes else: print("Seamless fail: " + e_str) cfg_mp = cfg.motion_blur_power / 100.0 out_img = img_bgr * (1 - img_face_mask_a) + (out_img * img_face_mask_a) if ('seamless' in cfg.mode and cfg.color_transfer_mode != 0) or \ cfg.mode == 'seamless-hist-match' or \ cfg_mp != 0 or \ cfg.blursharpen_amount != 0 or \ cfg.image_denoise_power != 0 or \ cfg.bicubic_degrade_power != 0: out_face_bgr = cv2.warpAffine(out_img, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0: if cfg.color_transfer_mode == 1: out_face_bgr = imagelib.reinhard_color_transfer( np.clip( out_face_bgr * wrk_face_mask_area_a * 255, 0, 255).astype(np.uint8), np.clip( dst_face_bgr * wrk_face_mask_area_a * 255, 0, 255).astype(np.uint8)) out_face_bgr = np.clip( out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) elif cfg.color_transfer_mode == 2: #lct out_face_bgr = imagelib.linear_color_transfer( out_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 3: #mkl out_face_bgr = imagelib.color_transfer_mkl( out_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 4: #mkl-m out_face_bgr = imagelib.color_transfer_mkl( out_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a) elif cfg.color_transfer_mode == 5: #idt out_face_bgr = imagelib.color_transfer_idt( out_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 6: #idt-m out_face_bgr = imagelib.color_transfer_idt( out_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a) elif cfg.color_transfer_mode == 7: #sot-m out_face_bgr = imagelib.color_transfer_sot( out_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a, steps=10, batch_size=30) out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0) elif cfg.color_transfer_mode == 8: #mix-m out_face_bgr = imagelib.color_transfer_mix( out_face_bgr * wrk_face_mask_area_a, dst_face_bgr * wrk_face_mask_area_a) if cfg.mode == 'seamless-hist-match': out_face_bgr = imagelib.color_hist_match( out_face_bgr, dst_face_bgr, cfg.hist_match_threshold) if cfg_mp != 0: k_size = int(frame_info.motion_power * cfg_mp) if k_size >= 1: k_size = np.clip(k_size + 1, 2, 50) if cfg.super_resolution_power != 0: k_size *= 2 out_face_bgr = imagelib.LinearMotionBlur( out_face_bgr, k_size, frame_info.motion_deg) if cfg.blursharpen_amount != 0: out_face_bgr = imagelib.blursharpen( out_face_bgr, cfg.sharpen_mode, 3, cfg.blursharpen_amount) if cfg.image_denoise_power != 0: n = cfg.image_denoise_power while n > 0: img_bgr_denoised = cv2.medianBlur(img_bgr, 5) if int(n / 100) != 0: img_bgr = img_bgr_denoised else: pass_power = (n % 100) / 100.0 img_bgr = img_bgr * ( 1.0 - pass_power) + img_bgr_denoised * pass_power n = max(n - 10, 0) if cfg.bicubic_degrade_power != 0: p = 1.0 - cfg.bicubic_degrade_power / 101.0 img_bgr_downscaled = cv2.resize( img_bgr, (int(img_size[0] * p), int(img_size[1] * p)), interpolation=cv2.INTER_CUBIC) img_bgr = cv2.resize(img_bgr_downscaled, img_size, interpolation=cv2.INTER_CUBIC) new_out = cv2.warpAffine( out_face_bgr, face_mat, img_size, np.empty_like(img_bgr), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC) out_img = np.clip( img_bgr * (1 - img_face_mask_a) + (new_out * img_face_mask_a), 0, 1.0) if cfg.color_degrade_power != 0: out_img_reduced = imagelib.reduce_colors(out_img, 256) if cfg.color_degrade_power == 100: out_img = out_img_reduced else: alpha = cfg.color_degrade_power / 100.0 out_img = (out_img * (1.0 - alpha) + out_img_reduced * alpha) out_merging_mask_a = img_face_mask_a if out_img is None: out_img = img_bgr.copy() return out_img, out_merging_mask_a
def process(samples, sample_process_options, output_sample_types, debug, ct_sample=None): SPTF = SampleProcessor.Types sample_rnd_seed = np.random.randint(0x80000000) outputs = [] for sample in samples: sample_bgr = sample.load_bgr() ct_sample_bgr = None h, w, c = sample_bgr.shape is_face_sample = sample.landmarks is not None if debug and is_face_sample: LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks, (0, 1, 0)) params = imagelib.gen_warp_params( sample_bgr, sample_process_options.random_flip, rotation_range=sample_process_options.rotation_range, scale_range=sample_process_options.scale_range, tx_range=sample_process_options.tx_range, ty_range=sample_process_options.ty_range, rnd_seed=sample_rnd_seed) outputs_sample = [] for opts in output_sample_types: resolution = opts.get('resolution', 0) types = opts.get('types', []) motion_blur = opts.get('motion_blur', None) gaussian_blur = opts.get('gaussian_blur', None) ct_mode = opts.get('ct_mode', 'None') normalize_tanh = opts.get('normalize_tanh', False) data_format = opts.get('data_format', 'NHWC') img_type = SPTF.NONE target_face_type = SPTF.NONE mode_type = SPTF.NONE for t in types: if t >= SPTF.IMG_TYPE_BEGIN and t < SPTF.IMG_TYPE_END: img_type = t elif t >= SPTF.FACE_TYPE_BEGIN and t < SPTF.FACE_TYPE_END: target_face_type = t elif t >= SPTF.MODE_BEGIN and t < SPTF.MODE_END: mode_type = t if mode_type == SPTF.MODE_FACE_MASK_HULL and not is_face_sample: raise ValueError( "MODE_FACE_MASK_HULL applicable only for face samples") if mode_type == SPTF.MODE_FACE_MASK_STRUCT and not is_face_sample: raise ValueError( "MODE_FACE_MASK_STRUCT applicable only for face samples" ) if is_face_sample: if target_face_type == SPTF.NONE: raise ValueError( "target face type must be defined for face samples" ) can_warp = (img_type == SPTF.IMG_WARPED or img_type == SPTF.IMG_WARPED_TRANSFORMED) can_transform = (img_type == SPTF.IMG_WARPED_TRANSFORMED or img_type == SPTF.IMG_TRANSFORMED) if img_type == SPTF.NONE: raise ValueError('expected IMG_ type') if img_type == SPTF.IMG_LANDMARKS_ARRAY: l = sample.landmarks l = np.concatenate([ np.expand_dims(l[:, 0] / w, -1), np.expand_dims(l[:, 1] / h, -1) ], -1) l = np.clip(l, 0.0, 1.0) out_sample = l elif img_type == SPTF.IMG_PITCH_YAW_ROLL or img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID: pitch_yaw_roll = sample.get_pitch_yaw_roll() if params['flip']: yaw = -yaw if img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID: pitch = np.clip((pitch / math.pi) / 2.0 + 0.5, 0, 1) yaw = np.clip((yaw / math.pi) / 2.0 + 0.5, 0, 1) roll = np.clip((roll / math.pi) / 2.0 + 0.5, 0, 1) out_sample = (pitch, yaw, roll) else: if mode_type == SPTF.NONE: raise ValueError('expected MODE_ type') if mode_type == SPTF.MODE_FACE_MASK_HULL: if sample.eyebrows_expand_mod is not None: img = LandmarksProcessor.get_image_hull_mask( sample_bgr.shape, sample.landmarks, eyebrows_expand_mod=sample.eyebrows_expand_mod) else: img = LandmarksProcessor.get_image_hull_mask( sample_bgr.shape, sample.landmarks) if sample.ie_polys is not None: sample.ie_polys.overlay_mask(img) elif mode_type == SPTF.MODE_FACE_MASK_STRUCT: if sample.eyebrows_expand_mod is not None: img = LandmarksProcessor.get_face_struct_mask( sample_bgr.shape, sample.landmarks, eyebrows_expand_mod=sample.eyebrows_expand_mod) else: img = LandmarksProcessor.get_face_struct_mask( sample_bgr.shape, sample.landmarks) else: img = sample_bgr if motion_blur is not None: chance, mb_max_size = motion_blur chance = np.clip(chance, 0, 100) if np.random.randint(100) < chance: img = imagelib.LinearMotionBlur( img, np.random.randint(mb_max_size) + 1, np.random.randint(360)) if gaussian_blur is not None: chance, kernel_max_size = gaussian_blur chance = np.clip(chance, 0, 100) if np.random.randint(100) < chance: img = cv2.GaussianBlur( img, (np.random.randint(kernel_max_size) * 2 + 1, ) * 2, 0) if is_face_sample: target_ft = SampleProcessor.SPTF_FACETYPE_TO_FACETYPE[ target_face_type] if target_ft > sample.face_type: raise Exception( 'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.' % (sample.filename, sample.face_type, target_ft)) if sample.face_type == FaceType.MARK_ONLY: mat = LandmarksProcessor.get_transform_mat( sample.landmarks, sample.shape[0], target_ft) if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT: img = cv2.warpAffine( img, mat, (sample.shape[0], sample.shape[0]), flags=cv2.INTER_CUBIC) img = imagelib.warp_by_params( params, img, can_warp, can_transform, can_flip=True, border_replicate=False) img = cv2.resize(img, (resolution, resolution), cv2.INTER_CUBIC)[..., None] else: img = cv2.warpAffine( img, mat, (sample.shape[0], sample.shape[0]), flags=cv2.INTER_CUBIC) img = imagelib.warp_by_params( params, img, can_warp, can_transform, can_flip=True, border_replicate=True) img = cv2.resize(img, (resolution, resolution), cv2.INTER_CUBIC) else: mat = LandmarksProcessor.get_transform_mat( sample.landmarks, resolution, target_ft) if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT: img = imagelib.warp_by_params( params, img, can_warp, can_transform, can_flip=True, border_replicate=False) img = cv2.warpAffine( img, mat, (resolution, resolution), borderMode=cv2.BORDER_CONSTANT, flags=cv2.INTER_CUBIC)[..., None] else: img = imagelib.warp_by_params( params, img, can_warp, can_transform, can_flip=True, border_replicate=True) img = cv2.warpAffine( img, mat, (resolution, resolution), borderMode=cv2.BORDER_REPLICATE, flags=cv2.INTER_CUBIC) else: img = imagelib.warp_by_params(params, img, can_warp, can_transform, can_flip=True, border_replicate=True) img = cv2.resize(img, (resolution, resolution), cv2.INTER_CUBIC) if mode_type == SPTF.MODE_FACE_MASK_HULL or mode_type == SPTF.MODE_FACE_MASK_STRUCT: out_sample = np.clip(img.astype(np.float32), 0, 1) else: img = np.clip(img.astype(np.float32), 0, 1) if ct_mode is not None and ct_sample is not None: if ct_sample_bgr is None: ct_sample_bgr = ct_sample.load_bgr() img = imagelib.color_transfer( ct_mode, img, cv2.resize(ct_sample_bgr, (resolution, resolution), cv2.INTER_LINEAR)) if mode_type == SPTF.MODE_BGR: out_sample = img elif mode_type == SPTF.MODE_BGR_SHUFFLE: rnd_state = np.random.RandomState(sample_rnd_seed) out_sample = np.take(img, rnd_state.permutation( img.shape[-1]), axis=-1) elif mode_type == SPTF.MODE_BGR_RANDOM_HSV_SHIFT: rnd_state = np.random.RandomState(sample_rnd_seed) hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) h, s, v = cv2.split(hsv) h = (h + rnd_state.randint(360)) % 360 s = np.clip(s + rnd_state.random() - 0.5, 0, 1) v = np.clip(v + rnd_state.random() - 0.5, 0, 1) hsv = cv2.merge([h, s, v]) out_sample = np.clip( cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0, 1) elif mode_type == SPTF.MODE_G: out_sample = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)[..., None] elif mode_type == SPTF.MODE_GGG: out_sample = np.repeat( np.expand_dims( cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), -1), (3, ), -1) if not debug: if normalize_tanh: out_sample = np.clip(out_sample * 2.0 - 1.0, -1.0, 1.0) if data_format == "NCHW": out_sample = np.transpose(out_sample, (2, 0, 1)) outputs_sample.append(out_sample) outputs += [outputs_sample] return outputs
def mask_editor_main(input_dir, confirmed_dir=None, skipped_dir=None, no_default_mask=False): input_path = Path(input_dir) confirmed_path = Path(confirmed_dir) skipped_path = Path(skipped_dir) if not input_path.exists(): raise ValueError('Input directory not found. Please ensure it exists.') if not confirmed_path.exists(): confirmed_path.mkdir(parents=True) if not skipped_path.exists(): skipped_path.mkdir(parents=True) if not no_default_mask: eyebrows_expand_mod = np.clip( io.input_int( "Default eyebrows expand modifier? (0..400, skip:100) : ", 100), 0, 400) / 100.0 else: eyebrows_expand_mod = None wnd_name = "MaskEditor tool" io.named_window(wnd_name) io.capture_mouse(wnd_name) io.capture_keys(wnd_name) cached_images = {} image_paths = [Path(x) for x in Path_utils.get_image_paths(input_path)] done_paths = [] done_images_types = {} image_paths_total = len(image_paths) saved_ie_polys = IEPolys() zoom_factor = 1.0 preview_images_count = 9 target_wh = 256 do_prev_count = 0 do_save_move_count = 0 do_save_count = 0 do_skip_move_count = 0 do_skip_count = 0 def jobs_count(): return do_prev_count + do_save_move_count + do_save_count + do_skip_move_count + do_skip_count is_exit = False while not is_exit: if len(image_paths) > 0: filepath = image_paths.pop(0) else: filepath = None next_image_paths = image_paths[0:preview_images_count] next_image_paths_names = [path.name for path in next_image_paths] prev_image_paths = done_paths[-preview_images_count:] prev_image_paths_names = [path.name for path in prev_image_paths] for key in list(cached_images.keys()): if key not in prev_image_paths_names and \ key not in next_image_paths_names: cached_images.pop(key) for paths in [prev_image_paths, next_image_paths]: for path in paths: if path.name not in cached_images: cached_images[path.name] = cv2_imread(str(path)) / 255.0 if filepath is not None: if filepath.suffix == '.png': dflimg = DFLPNG.load(str(filepath)) elif filepath.suffix == '.jpg': dflimg = DFLJPG.load(str(filepath)) else: dflimg = None if dflimg is None: io.log_err("%s is not a dfl image file" % (filepath.name)) continue else: lmrks = dflimg.get_landmarks() ie_polys = dflimg.get_ie_polys() fanseg_mask = dflimg.get_fanseg_mask() if filepath.name in cached_images: img = cached_images[filepath.name] else: img = cached_images[filepath.name] = cv2_imread( str(filepath)) / 255.0 if fanseg_mask is not None: mask = fanseg_mask else: if no_default_mask: mask = np.zeros((target_wh, target_wh, 3)) else: mask = LandmarksProcessor.get_image_hull_mask( img.shape, lmrks, eyebrows_expand_mod=eyebrows_expand_mod) else: img = np.zeros((target_wh, target_wh, 3)) mask = np.ones((target_wh, target_wh, 3)) ie_polys = None def get_status_lines_func(): return [ 'Progress: %d / %d . Current file: %s' % (len(done_paths), image_paths_total, str(filepath.name) if filepath is not None else "end"), '[Left mouse button] - mark include mask.', '[Right mouse button] - mark exclude mask.', '[Middle mouse button] - finish current poly.', '[Mouse wheel] - undo/redo poly or point. [+ctrl] - undo to begin/redo to end', '[r] - applies edits made to last saved image.', '[q] - prev image. [w] - skip and move to %s. [e] - save and move to %s. ' % (skipped_path.name, confirmed_path.name), '[z] - prev image. [x] - skip. [c] - save. ', 'hold [shift] - speed up the frame counter by 10.', '[-/+] - window zoom [esc] - quit', ] try: ed = MaskEditor(img, [(done_images_types[name], cached_images[name]) for name in prev_image_paths_names], [(0, cached_images[name]) for name in next_image_paths_names], mask, ie_polys, get_status_lines_func) except Exception as e: print(e) continue next = False while not next: io.process_messages(0.005) if jobs_count() == 0: for (x, y, ev, flags) in io.get_mouse_events(wnd_name): x, y = int(x / zoom_factor), int(y / zoom_factor) ed.set_mouse_pos(x, y) if filepath is not None: if ev == io.EVENT_LBUTTONDOWN: ed.mask_point(1) elif ev == io.EVENT_RBUTTONDOWN: ed.mask_point(0) elif ev == io.EVENT_MBUTTONDOWN: ed.mask_finish() elif ev == io.EVENT_MOUSEWHEEL: if flags & 0x80000000 != 0: if flags & 0x8 != 0: ed.undo_to_begin_point() else: ed.undo_point() else: if flags & 0x8 != 0: ed.redo_to_end_point() else: ed.redo_point() for key, chr_key, ctrl_pressed, alt_pressed, shift_pressed in io.get_key_events( wnd_name): if chr_key == 'q' or chr_key == 'z': do_prev_count = 1 if not shift_pressed else 10 elif chr_key == '-': zoom_factor = np.clip(zoom_factor - 0.1, 0.1, 4.0) ed.set_screen_changed() elif chr_key == '+': zoom_factor = np.clip(zoom_factor + 0.1, 0.1, 4.0) ed.set_screen_changed() elif key == 27: #esc is_exit = True next = True break elif filepath is not None: if chr_key == 'e': saved_ie_polys = ed.ie_polys do_save_move_count = 1 if not shift_pressed else 10 elif chr_key == 'c': saved_ie_polys = ed.ie_polys do_save_count = 1 if not shift_pressed else 10 elif chr_key == 'w': do_skip_move_count = 1 if not shift_pressed else 10 elif chr_key == 'x': do_skip_count = 1 if not shift_pressed else 10 elif chr_key == 'r' and saved_ie_polys != None: ed.set_ie_polys(saved_ie_polys) if do_prev_count > 0: do_prev_count -= 1 if len(done_paths) > 0: if filepath is not None: image_paths.insert(0, filepath) filepath = done_paths.pop(-1) done_images_types[filepath.name] = 0 if filepath.parent != input_path: new_filename_path = input_path / filepath.name filepath.rename(new_filename_path) image_paths.insert(0, new_filename_path) else: image_paths.insert(0, filepath) next = True elif filepath is not None: if do_save_move_count > 0: do_save_move_count -= 1 ed.mask_finish() dflimg.embed_and_set( str(filepath), ie_polys=ed.get_ie_polys(), eyebrows_expand_mod=eyebrows_expand_mod) done_paths += [confirmed_path / filepath.name] done_images_types[filepath.name] = 2 filepath.rename(done_paths[-1]) next = True elif do_save_count > 0: do_save_count -= 1 ed.mask_finish() dflimg.embed_and_set( str(filepath), ie_polys=ed.get_ie_polys(), eyebrows_expand_mod=eyebrows_expand_mod) done_paths += [filepath] done_images_types[filepath.name] = 2 next = True elif do_skip_move_count > 0: do_skip_move_count -= 1 done_paths += [skipped_path / filepath.name] done_images_types[filepath.name] = 1 filepath.rename(done_paths[-1]) next = True elif do_skip_count > 0: do_skip_count -= 1 done_paths += [filepath] done_images_types[filepath.name] = 1 next = True else: do_save_move_count = do_save_count = do_skip_move_count = do_skip_count = 0 if jobs_count() == 0: if ed.switch_screen_changed(): screen = ed.make_screen() if zoom_factor != 1.0: h, w, c = screen.shape screen = cv2.resize( screen, (int(w * zoom_factor), int(h * zoom_factor))) io.show_image(wnd_name, screen) io.process_messages(0.005) io.destroy_all_windows()
def convert_face(self, img_bgr, img_face_landmarks, debug): if (self.mask_mode == 3 or self.mask_mode == 4) and self.fan_seg == None: self.fan_seg = FANSegmentator(256, FaceType.toString(FaceType.FULL)) if self.over_res != 1: img_bgr = cv2.resize(img_bgr, (img_bgr.shape[1] * self.over_res, img_bgr.shape[0] * self.over_res)) img_face_landmarks = img_face_landmarks * self.over_res if debug: debugs = [img_bgr.copy()] img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask( img_bgr.shape, img_face_landmarks) face_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, self.output_size, face_type=self.face_type) face_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, self.output_size, face_type=self.face_type, scale=self.output_face_scale) dst_face_bgr = cv2.warpAffine(img_bgr, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4) dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4) predictor_input_bgr = cv2.resize( dst_face_bgr, (self.predictor_input_size, self.predictor_input_size)) predictor_input_mask_a_0 = cv2.resize( dst_face_mask_a_0, (self.predictor_input_size, self.predictor_input_size)) predictor_input_mask_a = np.expand_dims(predictor_input_mask_a_0, -1) predicted_bgra = self.predictor_func( np.concatenate((predictor_input_bgr, predictor_input_mask_a), -1)) prd_face_bgr = np.clip(predicted_bgra[:, :, 0:3], 0, 1.0) prd_face_mask_a_0 = np.clip(predicted_bgra[:, :, 3], 0.0, 1.0) if self.mask_mode == 2: #dst prd_face_mask_a_0 = predictor_input_mask_a_0 elif self.mask_mode == 3: #FAN-prd prd_face_bgr_256 = cv2.resize(prd_face_bgr, (256, 256)) prd_face_bgr_256_mask = self.fan_seg.extract_from_bgr( np.expand_dims(prd_face_bgr_256, 0))[0] prd_face_mask_a_0 = cv2.resize( prd_face_bgr_256_mask, (self.predictor_input_size, self.predictor_input_size)) elif self.mask_mode == 4: #FAN-dst face_256_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, 256, face_type=FaceType.FULL) dst_face_256_bgr = cv2.warpAffine(img_bgr, face_256_mat, (256, 256), flags=cv2.INTER_LANCZOS4) dst_face_256_mask = self.fan_seg.extract_from_bgr( np.expand_dims(dst_face_256_bgr, 0))[0] prd_face_mask_a_0 = cv2.resize( dst_face_256_mask, (self.predictor_input_size, self.predictor_input_size)) prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0 prd_face_mask_a = np.expand_dims(prd_face_mask_a_0, axis=-1) prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1) img_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4) img_face_mask_aaa = np.clip(img_face_mask_aaa, 0.0, 1.0) img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 #get rid of noise if debug: debugs += [img_face_mask_aaa.copy()] if 'seamless' in self.mode: #mask used for cv2.seamlessClone img_face_seamless_mask_aaa = None for i in range(9, 0, -1): a = img_face_mask_aaa > i / 10.0 if len(np.argwhere(a)) == 0: continue img_face_seamless_mask_aaa = img_face_mask_aaa.copy() img_face_seamless_mask_aaa[a] = 1.0 img_face_seamless_mask_aaa[img_face_seamless_mask_aaa <= i / 10.0] = 0.0 out_img = img_bgr.copy() if self.mode == 'raw': if self.raw_mode == 'rgb' or self.raw_mode == 'rgb-mask': out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) if self.raw_mode == 'rgb-mask': out_img = np.concatenate( [out_img, np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1) if self.raw_mode == 'mask-only': out_img = img_face_mask_aaa if self.raw_mode == 'predicted-only': out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(out_img.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) elif ('seamless' not in self.mode) or (img_face_seamless_mask_aaa is not None): #averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask ar = [] for i in range(1, 10): maxregion = np.argwhere(img_face_mask_aaa > i / 10.0) if maxregion.size != 0: miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx leny = maxy - miny maskx = (minx + (lenx / 2)) masky = (miny + (leny / 2)) if lenx >= 4 and leny >= 4: ar += [[lenx, leny, maskx, masky]] if len(ar) > 0: lenx, leny, maskx, masky = np.mean(ar, axis=0) if debug: io.log_info("lenx/leny:(%d/%d) maskx/masky:(%f/%f)" % (lenx, leny, maskx, masky)) maskx = int(maskx) masky = int(masky) lowest_len = min(lenx, leny) if debug: io.log_info("lowest_len = %f" % (lowest_len)) img_mask_blurry_aaa = img_face_mask_aaa if self.erode_mask_modifier != 0: ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) * 0.01 * self.erode_mask_modifier) if debug: io.log_info("erode_size = %d" % (ero)) if ero > 0: img_mask_blurry_aaa = cv2.erode( img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: img_mask_blurry_aaa = cv2.dilate( img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) if self.seamless_erode_mask_modifier != 0: ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) * 0.01 * self.seamless_erode_mask_modifier) if debug: io.log_info("seamless_erode_size = %d" % (ero)) if ero > 0: img_face_seamless_mask_aaa = cv2.erode( img_face_seamless_mask_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: img_face_seamless_mask_aaa = cv2.dilate( img_face_seamless_mask_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) img_face_seamless_mask_aaa = np.clip( img_face_seamless_mask_aaa, 0, 1) if self.clip_hborder_mask_per > 0: #clip hborder before blur prd_hborder_rect_mask_a = np.ones(prd_face_mask_a.shape, dtype=np.float32) prd_border_size = int(prd_hborder_rect_mask_a.shape[1] * self.clip_hborder_mask_per) prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0 prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0 prd_hborder_rect_mask_a = np.expand_dims( cv2.blur(prd_hborder_rect_mask_a, (prd_border_size, prd_border_size)), -1) img_prd_hborder_rect_mask_a = cv2.warpAffine( prd_hborder_rect_mask_a, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4) img_prd_hborder_rect_mask_a = np.expand_dims( img_prd_hborder_rect_mask_a, -1) img_mask_blurry_aaa *= img_prd_hborder_rect_mask_a img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0) if debug: debugs += [img_mask_blurry_aaa.copy()] if self.blur_mask_modifier > 0: blur = int(lowest_len * 0.10 * 0.01 * self.blur_mask_modifier) if debug: io.log_info("blur_size = %d" % (blur)) if blur > 0: img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa, (blur, blur)) img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0) if debug: debugs += [img_mask_blurry_aaa.copy()] if self.color_transfer_mode is not None: if self.color_transfer_mode == 'rct': if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] prd_face_bgr = image_utils.reinhard_color_transfer( np.clip((prd_face_bgr * 255).astype(np.uint8), 0, 255), np.clip((dst_face_bgr * 255).astype(np.uint8), 0, 255), source_mask=prd_face_mask_a, target_mask=prd_face_mask_a) prd_face_bgr = np.clip( prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] elif self.color_transfer_mode == 'lct': if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] prd_face_bgr = image_utils.linear_color_transfer( prd_face_bgr, dst_face_bgr) prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0) if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] if self.mode == 'hist-match-bw': prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY) prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1), (3, ), -1) if self.mode == 'hist-match' or self.mode == 'hist-match-bw': if debug: debugs += [ cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) ] hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) if self.masked_hist_match: hist_mask_a *= prd_face_mask_a hist_match_1 = prd_face_bgr * hist_mask_a + ( 1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_1[hist_match_1 > 1.0] = 1.0 hist_match_2 = dst_face_bgr * hist_mask_a + ( 1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_2[hist_match_1 > 1.0] = 1.0 prd_face_bgr = image_utils.color_hist_match( hist_match_1, hist_match_2, self.hist_match_threshold) if self.mode == 'hist-match-bw': prd_face_bgr = prd_face_bgr.astype(dtype=np.float32) out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) out_img = np.clip(out_img, 0.0, 1.0) if debug: debugs += [out_img.copy()] if self.mode == 'overlay': pass if 'seamless' in self.mode: try: out_img = cv2.seamlessClone( (out_img * 255).astype(np.uint8), (img_bgr * 255).astype(np.uint8), (img_face_seamless_mask_aaa * 255).astype( np.uint8), (maskx, masky), cv2.NORMAL_CLONE) out_img = out_img.astype(dtype=np.float32) / 255.0 except Exception as e: #seamlessClone may fail in some cases e_str = traceback.format_exc() if 'MemoryError' in e_str: raise Exception( "Seamless fail: " + e_str ) #reraise MemoryError in order to reprocess this data by other processes else: print("Seamless fail: " + e_str) if debug: debugs += [out_img.copy()] out_img = np.clip( img_bgr * (1 - img_mask_blurry_aaa) + (out_img * img_mask_blurry_aaa), 0, 1.0) if self.mode == 'seamless-hist-match': out_face_bgr = cv2.warpAffine( out_img, face_mat, (self.output_size, self.output_size)) new_out_face_bgr = image_utils.color_hist_match( out_face_bgr, dst_face_bgr, self.hist_match_threshold) new_out = cv2.warpAffine( new_out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) out_img = np.clip( img_bgr * (1 - img_mask_blurry_aaa) + (new_out * img_mask_blurry_aaa), 0, 1.0) if self.final_image_color_degrade_power != 0: if debug: debugs += [out_img.copy()] out_img_reduced = image_utils.reduce_colors(out_img, 256) if self.final_image_color_degrade_power == 100: out_img = out_img_reduced else: alpha = self.final_image_color_degrade_power / 100.0 out_img = (out_img * (1.0 - alpha) + out_img_reduced * alpha) if self.alpha: out_img = np.concatenate([ out_img, np.expand_dims(img_mask_blurry_aaa[:, :, 0], -1) ], -1) if self.over_res != 1: out_img = cv2.resize(out_img, (img_bgr.shape[1] // self.over_res, img_bgr.shape[0] // self.over_res)) out_img = np.clip(out_img, 0.0, 1.0) if debug: debugs += [out_img.copy()] return debugs if debug else out_img
def process(samples, sample_process_options, output_sample_types, debug, ct_sample=None): SPST = SampleProcessor.SampleType SPCT = SampleProcessor.ChannelType SPFMT = SampleProcessor.FaceMaskType sample_rnd_seed = np.random.randint(0x80000000) outputs = [] for sample in samples: sample_bgr = sample.load_bgr() ct_sample_bgr = None h, w, c = sample_bgr.shape is_face_sample = sample.landmarks is not None if debug and is_face_sample: LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks, (0, 1, 0)) params = imagelib.gen_warp_params( sample_bgr, sample_process_options.random_flip, rotation_range=sample_process_options.rotation_range, scale_range=sample_process_options.scale_range, tx_range=sample_process_options.tx_range, ty_range=sample_process_options.ty_range) outputs_sample = [] for opts in output_sample_types: sample_type = opts.get('sample_type', SPST.NONE) channel_type = opts.get('channel_type', SPCT.NONE) resolution = opts.get('resolution', 0) warp = opts.get('warp', False) transform = opts.get('transform', False) motion_blur = opts.get('motion_blur', None) gaussian_blur = opts.get('gaussian_blur', None) normalize_tanh = opts.get('normalize_tanh', False) ct_mode = opts.get('ct_mode', 'None') data_format = opts.get('data_format', 'NHWC') if sample_type == SPST.FACE_IMAGE or sample_type == SPST.FACE_MASK: if not is_face_sample: raise ValueError( "face_samples should be provided for sample_type FACE_*" ) if is_face_sample: face_type = opts.get('face_type', None) face_mask_type = opts.get('face_mask_type', SPFMT.NONE) if face_type is None: raise ValueError( "face_type must be defined for face samples") if face_type > sample.face_type: raise Exception( 'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.' % (sample.filename, sample.face_type, target_ft)) if sample_type == SPST.FACE_IMAGE or sample_type == SPST.FACE_MASK: if sample_type == SPST.FACE_MASK: if face_mask_type == SPFMT.ALL_HULL or \ face_mask_type == SPFMT.EYES_HULL or \ face_mask_type == SPFMT.ALL_EYES_HULL: if face_mask_type == SPFMT.ALL_HULL or \ face_mask_type == SPFMT.ALL_EYES_HULL: if sample.eyebrows_expand_mod is not None: all_mask = LandmarksProcessor.get_image_hull_mask( sample_bgr.shape, sample.landmarks, eyebrows_expand_mod=sample. eyebrows_expand_mod) else: all_mask = LandmarksProcessor.get_image_hull_mask( sample_bgr.shape, sample.landmarks) all_mask = np.clip(all_mask, 0, 1) if face_mask_type == SPFMT.EYES_HULL or \ face_mask_type == SPFMT.ALL_EYES_HULL: eyes_mask = LandmarksProcessor.get_image_eye_mask( sample_bgr.shape, sample.landmarks) eyes_mask = np.clip(eyes_mask, 0, 1) if face_mask_type == SPFMT.ALL_HULL: img = all_mask elif face_mask_type == SPFMT.EYES_HULL: img = eyes_mask elif face_mask_type == SPFMT.ALL_EYES_HULL: img = all_mask + eyes_mask elif face_mask_type == SPFMT.STRUCT: if sample.eyebrows_expand_mod is not None: img = LandmarksProcessor.get_face_struct_mask( sample_bgr.shape, sample.landmarks, eyebrows_expand_mod=sample. eyebrows_expand_mod) else: img = LandmarksProcessor.get_face_struct_mask( sample_bgr.shape, sample.landmarks) if sample.ie_polys is not None: sample.ie_polys.overlay_mask(img) if sample.face_type == FaceType.MARK_ONLY: mat = LandmarksProcessor.get_transform_mat( sample.landmarks, sample.shape[0], face_type) img = cv2.warpAffine( img, mat, (sample.shape[0], sample.shape[0]), flags=cv2.INTER_LINEAR) img = imagelib.warp_by_params( params, img, warp, transform, can_flip=True, border_replicate=False, cv2_inter=cv2.INTER_LINEAR) img = cv2.resize(img, (resolution, resolution), cv2.INTER_LINEAR)[..., None] else: mat = LandmarksProcessor.get_transform_mat( sample.landmarks, resolution, face_type) img = imagelib.warp_by_params( params, img, warp, transform, can_flip=True, border_replicate=False, cv2_inter=cv2.INTER_LINEAR) img = cv2.warpAffine( img, mat, (resolution, resolution), borderMode=cv2.BORDER_CONSTANT, flags=cv2.INTER_LINEAR)[..., None] if channel_type == SPCT.G: out_sample = img.astype(np.float32) else: raise ValueError( "only channel_type.G supported for the mask") elif sample_type == SPST.FACE_IMAGE: img = sample_bgr if motion_blur is not None: chance, mb_max_size = motion_blur chance = np.clip(chance, 0, 100) l_rnd_state = np.random.RandomState( sample_rnd_seed) mblur_rnd_chance = l_rnd_state.randint(100) mblur_rnd_kernel = l_rnd_state.randint( mb_max_size) + 1 mblur_rnd_deg = l_rnd_state.randint(360) if mblur_rnd_chance < chance: img = imagelib.LinearMotionBlur( img, mblur_rnd_kernel, mblur_rnd_deg) if gaussian_blur is not None: chance, kernel_max_size = gaussian_blur chance = np.clip(chance, 0, 100) l_rnd_state = np.random.RandomState( sample_rnd_seed + 1) gblur_rnd_chance = l_rnd_state.randint(100) gblur_rnd_kernel = l_rnd_state.randint( kernel_max_size) * 2 + 1 if gblur_rnd_chance < chance: img = cv2.GaussianBlur( img, (gblur_rnd_kernel, ) * 2, 0) if sample.face_type == FaceType.MARK_ONLY: mat = LandmarksProcessor.get_transform_mat( sample.landmarks, sample.shape[0], face_type) img = cv2.warpAffine( img, mat, (sample.shape[0], sample.shape[0]), flags=cv2.INTER_CUBIC) img = imagelib.warp_by_params( params, img, warp, transform, can_flip=True, border_replicate=True) img = cv2.resize(img, (resolution, resolution), cv2.INTER_CUBIC) else: mat = LandmarksProcessor.get_transform_mat( sample.landmarks, resolution, face_type) img = imagelib.warp_by_params( params, img, warp, transform, can_flip=True, border_replicate=True) img = cv2.warpAffine( img, mat, (resolution, resolution), borderMode=cv2.BORDER_REPLICATE, flags=cv2.INTER_CUBIC) img = np.clip(img.astype(np.float32), 0, 1) # Apply random color transfer if ct_mode is not None and ct_sample is not None: if ct_sample_bgr is None: ct_sample_bgr = ct_sample.load_bgr() img = imagelib.color_transfer( ct_mode, img, cv2.resize(ct_sample_bgr, (resolution, resolution), cv2.INTER_LINEAR)) # Transform from BGR to desired channel_type if channel_type == SPCT.BGR: out_sample = img elif channel_type == SPCT.BGR_SHUFFLE: l_rnd_state = np.random.RandomState( sample_rnd_seed) out_sample = np.take(img, l_rnd_state.permutation( img.shape[-1]), axis=-1) elif channel_type == SPCT.BGR_RANDOM_HSV_SHIFT: l_rnd_state = np.random.RandomState( sample_rnd_seed) hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) h, s, v = cv2.split(hsv) h = (h + l_rnd_state.randint(360)) % 360 s = np.clip(s + l_rnd_state.random() - 0.5, 0, 1) v = np.clip(v + l_rnd_state.random() - 0.5, 0, 1) hsv = cv2.merge([h, s, v]) out_sample = np.clip( cv2.cvtColor(hsv, cv2.COLOR_HSV2BGR), 0, 1) elif channel_type == SPCT.BGR_RANDOM_RGB_LEVELS: l_rnd_state = np.random.RandomState( sample_rnd_seed) np_rnd = l_rnd_state.rand inBlack = np.array([ np_rnd() * 0.25, np_rnd() * 0.25, np_rnd() * 0.25 ], dtype=np.float32) inWhite = np.array([ 1.0 - np_rnd() * 0.25, 1.0 - np_rnd() * 0.25, 1.0 - np_rnd() * 0.25 ], dtype=np.float32) inGamma = np.array([ 0.5 + np_rnd(), 0.5 + np_rnd(), 0.5 + np_rnd() ], dtype=np.float32) outBlack = np.array([0.0, 0.0, 0.0], dtype=np.float32) outWhite = np.array([1.0, 1.0, 1.0], dtype=np.float32) out_sample = np.clip( (img - inBlack) / (inWhite - inBlack), 0, 1) out_sample = (out_sample**(1 / inGamma)) * ( outWhite - outBlack) + outBlack out_sample = np.clip(out_sample, 0, 1) elif channel_type == SPCT.G: out_sample = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)[..., None] elif channel_type == SPCT.GGG: out_sample = np.repeat( np.expand_dims( cv2.cvtColor(img, cv2.COLOR_BGR2GRAY), -1), (3, ), -1) # Final transformations if not debug: if normalize_tanh: out_sample = np.clip(out_sample * 2.0 - 1.0, -1.0, 1.0) if data_format == "NCHW": out_sample = np.transpose(out_sample, (2, 0, 1)) #else: # img = imagelib.warp_by_params (params, img, warp, transform, can_flip=True, border_replicate=True) # img = cv2.resize( img, (resolution,resolution), cv2.INTER_CUBIC ) elif sample_type == SPST.LANDMARKS_ARRAY: l = sample.landmarks l = np.concatenate([ np.expand_dims(l[:, 0] / w, -1), np.expand_dims(l[:, 1] / h, -1) ], -1) l = np.clip(l, 0.0, 1.0) out_sample = l elif sample_type == SPST.PITCH_YAW_ROLL or sample_type == SPST.PITCH_YAW_ROLL_SIGMOID: pitch_yaw_roll = sample.get_pitch_yaw_roll() if params['flip']: yaw = -yaw if sample_type == SPST.PITCH_YAW_ROLL_SIGMOID: pitch = np.clip((pitch / math.pi) / 2.0 + 0.5, 0, 1) yaw = np.clip((yaw / math.pi) / 2.0 + 0.5, 0, 1) roll = np.clip((roll / math.pi) / 2.0 + 0.5, 0, 1) out_sample = (pitch, yaw, roll) else: raise ValueError('expected sample_type') outputs_sample.append(out_sample) outputs += [outputs_sample] return outputs
def MergeMaskedFace(predictor_func, predictor_input_shape, cfg, frame_info, img_bgr_uint8, img_bgr, img_face_landmarks): img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask( img_bgr.shape, img_face_landmarks) if cfg.mode == 'original': return img_bgr, img_face_mask_a out_img = img_bgr.copy() out_merging_mask_a = None input_size = predictor_input_shape[0] mask_subres_size = input_size * 4 output_size = input_size if cfg.super_resolution_power != 0: output_size *= 4 face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks, output_size, face_type=cfg.face_type) face_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, output_size, face_type=cfg.face_type, scale=1.0 + 0.01 * cfg.output_face_scale) if mask_subres_size == output_size: face_mask_output_mat = face_output_mat else: face_mask_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, mask_subres_size, face_type=cfg.face_type, scale=1.0 + 0.01 * cfg.output_face_scale) dst_face_bgr = cv2.warpAffine(img_bgr, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) dst_face_bgr = np.clip(dst_face_bgr, 0, 1) dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) dst_face_mask_a_0 = np.clip(dst_face_mask_a_0, 0, 1) predictor_input_bgr = cv2.resize(dst_face_bgr, (input_size, input_size)) predicted = predictor_func(predictor_input_bgr) if isinstance(predicted, tuple): #merger return bgr,mask prd_face_bgr = np.clip(predicted[0], 0, 1.0) prd_face_mask_a_0 = np.clip(predicted[1], 0, 1.0) predictor_masked = True else: #merger return bgr only, using dst mask prd_face_bgr = np.clip(predicted, 0, 1.0) prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, (input_size, input_size)) predictor_masked = False if cfg.super_resolution_power != 0: prd_face_bgr_enhanced = cfg.superres_func(prd_face_bgr) mod = cfg.super_resolution_power / 100.0 prd_face_bgr = cv2.resize(prd_face_bgr, (output_size, output_size)) * ( 1.0 - mod) + prd_face_bgr_enhanced * mod prd_face_bgr = np.clip(prd_face_bgr, 0, 1) if cfg.super_resolution_power != 0: if predictor_masked: prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) else: prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) if cfg.mask_mode == 2: #dst prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) elif cfg.mask_mode >= 3 and cfg.mask_mode <= 8: if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6: prd_face_fanseg_bgr = cv2.resize(prd_face_bgr, (cfg.fanseg_input_size, ) * 2) prd_face_fanseg_mask = cfg.fanseg_extract_func( FaceType.FULL, prd_face_fanseg_bgr) FAN_prd_face_mask_a_0 = cv2.resize(prd_face_fanseg_mask, (output_size, output_size), cv2.INTER_CUBIC) if cfg.mask_mode >= 4 and cfg.mask_mode <= 7: full_face_fanseg_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, cfg.fanseg_input_size, face_type=FaceType.FULL) dst_face_fanseg_bgr = cv2.warpAffine(img_bgr, full_face_fanseg_mat, (cfg.fanseg_input_size, ) * 2, flags=cv2.INTER_CUBIC) dst_face_fanseg_mask = cfg.fanseg_extract_func( FaceType.FULL, dst_face_fanseg_bgr) if cfg.face_type == FaceType.FULL: FAN_dst_face_mask_a_0 = cv2.resize(dst_face_fanseg_mask, (output_size, output_size), cv2.INTER_CUBIC) else: face_fanseg_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, cfg.fanseg_input_size, face_type=cfg.face_type) fanseg_rect_corner_pts = np.array( [[0, 0], [cfg.fanseg_input_size - 1, 0], [0, cfg.fanseg_input_size - 1]], dtype=np.float32) a = LandmarksProcessor.transform_points(fanseg_rect_corner_pts, face_fanseg_mat, invert=True) b = LandmarksProcessor.transform_points( a, full_face_fanseg_mat) m = cv2.getAffineTransform(b, fanseg_rect_corner_pts) FAN_dst_face_mask_a_0 = cv2.warpAffine( dst_face_fanseg_mask, m, (cfg.fanseg_input_size, ) * 2, flags=cv2.INTER_CUBIC) FAN_dst_face_mask_a_0 = cv2.resize(FAN_dst_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) if cfg.mask_mode == 3: #FAN-prd prd_face_mask_a_0 = FAN_prd_face_mask_a_0 elif cfg.mask_mode == 4: #FAN-dst prd_face_mask_a_0 = FAN_dst_face_mask_a_0 elif cfg.mask_mode == 5: prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0 elif cfg.mask_mode == 6: prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0 elif cfg.mask_mode == 7: prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0 prd_face_mask_a_0[prd_face_mask_a_0 < (1.0 / 255.0)] = 0.0 # get rid of noise # resize to mask_subres_size if prd_face_mask_a_0.shape[0] != mask_subres_size: prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0, (mask_subres_size, mask_subres_size), cv2.INTER_CUBIC) # process mask in local predicted space if 'raw' not in cfg.mode: # add zero pad prd_face_mask_a_0 = np.pad(prd_face_mask_a_0, input_size) ero = cfg.erode_mask_modifier blur = cfg.blur_mask_modifier if ero > 0: prd_face_mask_a_0 = cv2.erode(prd_face_mask_a_0, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: prd_face_mask_a_0 = cv2.dilate(prd_face_mask_a_0, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) # clip eroded/dilated mask in actual predict area # pad with half blur size in order to accuratelly fade to zero at the boundary clip_size = input_size + blur // 2 prd_face_mask_a_0[:clip_size, :] = 0 prd_face_mask_a_0[-clip_size:, :] = 0 prd_face_mask_a_0[:, :clip_size] = 0 prd_face_mask_a_0[:, -clip_size:] = 0 if blur > 0: blur = blur + (1 - blur % 2) prd_face_mask_a_0 = cv2.GaussianBlur(prd_face_mask_a_0, (blur, blur), 0) prd_face_mask_a_0 = prd_face_mask_a_0[input_size:-input_size, input_size:-input_size] prd_face_mask_a_0 = np.clip(prd_face_mask_a_0, 0, 1) img_face_mask_a = cv2.warpAffine(prd_face_mask_a_0, face_mask_output_mat, img_size, np.zeros(img_bgr.shape[0:2], dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC)[..., None] img_face_mask_a = np.clip(img_face_mask_a, 0.0, 1.0) img_face_mask_a[img_face_mask_a < (1.0 / 255.0)] = 0.0 # get rid of noise if prd_face_mask_a_0.shape[0] != output_size: prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) prd_face_mask_a = prd_face_mask_a_0[..., None] prd_face_mask_area_a = prd_face_mask_a.copy() prd_face_mask_area_a[prd_face_mask_area_a > 0] = 1.0 if 'raw' in cfg.mode: if cfg.mode == 'raw-rgb': out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) out_merging_mask_a = img_face_mask_a out_img = np.clip(out_img, 0.0, 1.0) else: #averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask ar = [] for i in range(1, 10): maxregion = np.argwhere(img_face_mask_a > i / 10.0) if maxregion.size != 0: miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx leny = maxy - miny if min(lenx, leny) >= 4: ar += [[lenx, leny]] if len(ar) > 0: if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0: if cfg.color_transfer_mode == 1: #rct prd_face_bgr = imagelib.reinhard_color_transfer( np.clip(prd_face_bgr * prd_face_mask_area_a * 255, 0, 255).astype(np.uint8), np.clip(dst_face_bgr * prd_face_mask_area_a * 255, 0, 255).astype(np.uint8), ) prd_face_bgr = np.clip( prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) elif cfg.color_transfer_mode == 2: #lct prd_face_bgr = imagelib.linear_color_transfer( prd_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 3: #mkl prd_face_bgr = imagelib.color_transfer_mkl( prd_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 4: #mkl-m prd_face_bgr = imagelib.color_transfer_mkl( prd_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) elif cfg.color_transfer_mode == 5: #idt prd_face_bgr = imagelib.color_transfer_idt( prd_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 6: #idt-m prd_face_bgr = imagelib.color_transfer_idt( prd_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) elif cfg.color_transfer_mode == 7: #sot-m prd_face_bgr = imagelib.color_transfer_sot( prd_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0) elif cfg.color_transfer_mode == 8: #mix-m prd_face_bgr = imagelib.color_transfer_mix( prd_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) if cfg.mode == 'hist-match': hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) if cfg.masked_hist_match: hist_mask_a *= prd_face_mask_area_a white = (1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_1 = prd_face_bgr * hist_mask_a + white hist_match_1[hist_match_1 > 1.0] = 1.0 hist_match_2 = dst_face_bgr * hist_mask_a + white hist_match_2[hist_match_1 > 1.0] = 1.0 prd_face_bgr = imagelib.color_hist_match( hist_match_1, hist_match_2, cfg.hist_match_threshold).astype(dtype=np.float32) if 'seamless' in cfg.mode: #mask used for cv2.seamlessClone img_face_seamless_mask_a = None for i in range(1, 10): a = img_face_mask_a > i / 10.0 if len(np.argwhere(a)) == 0: continue img_face_seamless_mask_a = img_face_mask_a.copy() img_face_seamless_mask_a[a] = 1.0 img_face_seamless_mask_a[img_face_seamless_mask_a <= i / 10.0] = 0.0 break out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) out_img = np.clip(out_img, 0.0, 1.0) if 'seamless' in cfg.mode: try: #calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering) l, t, w, h = cv2.boundingRect( (img_face_seamless_mask_a * 255).astype(np.uint8)) s_maskx, s_masky = int(l + w / 2), int(t + h / 2) out_img = cv2.seamlessClone( (out_img * 255).astype(np.uint8), img_bgr_uint8, (img_face_seamless_mask_a * 255).astype(np.uint8), (s_maskx, s_masky), cv2.NORMAL_CLONE) out_img = out_img.astype(dtype=np.float32) / 255.0 except Exception as e: #seamlessClone may fail in some cases e_str = traceback.format_exc() if 'MemoryError' in e_str: raise Exception( "Seamless fail: " + e_str ) #reraise MemoryError in order to reprocess this data by other processes else: print("Seamless fail: " + e_str) out_img = img_bgr * (1 - img_face_mask_a) + (out_img * img_face_mask_a) out_face_bgr = cv2.warpAffine(out_img, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0: if cfg.color_transfer_mode == 1: out_face_bgr = imagelib.reinhard_color_transfer( np.clip(out_face_bgr * prd_face_mask_area_a * 255, 0, 255).astype(np.uint8), np.clip(dst_face_bgr * prd_face_mask_area_a * 255, 0, 255).astype(np.uint8)) out_face_bgr = np.clip( out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) elif cfg.color_transfer_mode == 2: #lct out_face_bgr = imagelib.linear_color_transfer( out_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 3: #mkl out_face_bgr = imagelib.color_transfer_mkl( out_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 4: #mkl-m out_face_bgr = imagelib.color_transfer_mkl( out_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) elif cfg.color_transfer_mode == 5: #idt out_face_bgr = imagelib.color_transfer_idt( out_face_bgr, dst_face_bgr) elif cfg.color_transfer_mode == 6: #idt-m out_face_bgr = imagelib.color_transfer_idt( out_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) elif cfg.color_transfer_mode == 7: #sot-m out_face_bgr = imagelib.color_transfer_sot( out_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0) elif cfg.color_transfer_mode == 8: #mix-m out_face_bgr = imagelib.color_transfer_mix( out_face_bgr * prd_face_mask_area_a, dst_face_bgr * prd_face_mask_area_a) if cfg.mode == 'seamless-hist-match': out_face_bgr = imagelib.color_hist_match( out_face_bgr, dst_face_bgr, cfg.hist_match_threshold) cfg_mp = cfg.motion_blur_power / 100.0 if cfg_mp != 0: k_size = int(frame_info.motion_power * cfg_mp) if k_size >= 1: k_size = np.clip(k_size + 1, 2, 50) if cfg.super_resolution_power != 0: k_size *= 2 out_face_bgr = imagelib.LinearMotionBlur( out_face_bgr, k_size, frame_info.motion_deg) if cfg.blursharpen_amount != 0: out_face_bgr = cfg.blursharpen_func(out_face_bgr, cfg.sharpen_mode, 3, cfg.blursharpen_amount) if cfg.image_denoise_power != 0: n = cfg.image_denoise_power while n > 0: img_bgr_denoised = cv2.medianBlur(img_bgr, 5) if int(n / 100) != 0: img_bgr = img_bgr_denoised else: pass_power = (n % 100) / 100.0 img_bgr = img_bgr * ( 1.0 - pass_power) + img_bgr_denoised * pass_power n = max(n - 10, 0) if cfg.bicubic_degrade_power != 0: p = 1.0 - cfg.bicubic_degrade_power / 101.0 img_bgr_downscaled = cv2.resize( img_bgr, (int(img_size[0] * p), int(img_size[1] * p)), cv2.INTER_CUBIC) img_bgr = cv2.resize(img_bgr_downscaled, img_size, cv2.INTER_CUBIC) new_out = cv2.warpAffine(out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) out_img = np.clip( img_bgr * (1 - img_face_mask_a) + (new_out * img_face_mask_a), 0, 1.0) if cfg.color_degrade_power != 0: out_img_reduced = imagelib.reduce_colors(out_img, 256) if cfg.color_degrade_power == 100: out_img = out_img_reduced else: alpha = cfg.color_degrade_power / 100.0 out_img = (out_img * (1.0 - alpha) + out_img_reduced * alpha) out_merging_mask_a = img_face_mask_a return out_img, out_merging_mask_a
def convert_face (self, img_bgr, img_face_landmarks, debug): if debug: debugs = [img_bgr.copy()] img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask (img_bgr, img_face_landmarks) face_mat = LandmarksProcessor.get_transform_mat (img_face_landmarks, self.output_size, face_type=self.face_type) dst_face_bgr = cv2.warpAffine( img_bgr , face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4 ) dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4 ) predictor_input_bgr = cv2.resize (dst_face_bgr, (self.predictor_input_size,self.predictor_input_size)) predictor_input_mask_a_0 = cv2.resize (dst_face_mask_a_0, (self.predictor_input_size,self.predictor_input_size)) predictor_input_mask_a = np.expand_dims (predictor_input_mask_a_0, -1) predicted_bgra = self.predictor ( np.concatenate( (predictor_input_bgr, predictor_input_mask_a), -1) ) prd_face_bgr = np.clip (predicted_bgra[:,:,0:3], 0, 1.0 ) prd_face_mask_a_0 = np.clip (predicted_bgra[:,:,3], 0.0, 1.0) prd_face_mask_a_0[ prd_face_mask_a_0 < 0.001 ] = 0.0 prd_face_mask_a = np.expand_dims (prd_face_mask_a_0, axis=-1) prd_face_mask_aaa = np.repeat (prd_face_mask_a, (3,), axis=-1) img_prd_face_mask_aaa = cv2.warpAffine( prd_face_mask_aaa, face_mat, img_size, np.zeros(img_bgr.shape, dtype=float), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4 ) img_prd_face_mask_aaa = np.clip (img_prd_face_mask_aaa, 0.0, 1.0) img_face_mask_aaa = img_prd_face_mask_aaa if debug: debugs += [img_face_mask_aaa.copy()] img_face_mask_aaa [ img_face_mask_aaa <= 0.1 ] = 0.0 img_face_mask_flatten_aaa = img_face_mask_aaa.copy() img_face_mask_flatten_aaa[img_face_mask_flatten_aaa > 0.9] = 1.0 maxregion = np.argwhere(img_face_mask_flatten_aaa==1.0) out_img = img_bgr.copy() if maxregion.size != 0: miny,minx = maxregion.min(axis=0)[:2] maxy,maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx leny = maxy - miny masky = int(minx+(lenx//2)) maskx = int(miny+(leny//2)) lowest_len = min (lenx, leny) if debug: print ("lowest_len = %f" % (lowest_len) ) ero = int( lowest_len * ( 0.126 - lowest_len * 0.00004551365 ) * 0.01*self.erode_mask_modifier ) blur = int( lowest_len * 0.10 * 0.01*self.blur_mask_modifier ) if debug: print ("ero = %d, blur = %d" % (ero, blur) ) img_mask_blurry_aaa = img_face_mask_aaa if self.erode_mask: if ero > 0: img_mask_blurry_aaa = cv2.erode(img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(ero,ero)), iterations = 1 ) elif ero < 0: img_mask_blurry_aaa = cv2.dilate(img_mask_blurry_aaa, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(-ero,-ero)), iterations = 1 ) if self.blur_mask and blur > 0: img_mask_blurry_aaa = cv2.blur(img_mask_blurry_aaa, (blur, blur) ) img_mask_blurry_aaa = np.clip( img_mask_blurry_aaa, 0, 1.0 ) if self.clip_border_mask_per > 0: prd_border_rect_mask_a = np.ones ( prd_face_mask_a.shape, dtype=prd_face_mask_a.dtype) prd_border_size = int ( prd_border_rect_mask_a.shape[1] * self.clip_border_mask_per ) prd_border_rect_mask_a[0:prd_border_size,:,:] = 0 prd_border_rect_mask_a[-prd_border_size:,:,:] = 0 prd_border_rect_mask_a[:,0:prd_border_size,:] = 0 prd_border_rect_mask_a[:,-prd_border_size:,:] = 0 prd_border_rect_mask_a = np.expand_dims(cv2.blur(prd_border_rect_mask_a, (prd_border_size, prd_border_size) ),-1) if self.mode == 'hist-match-bw': prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY) prd_face_bgr = np.repeat( np.expand_dims (prd_face_bgr, -1), (3,), -1 ) if self.mode == 'hist-match' or self.mode == 'hist-match-bw': if debug: debugs += [ cv2.warpAffine( prd_face_bgr, face_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ) ] hist_mask_a = np.ones ( prd_face_bgr.shape[:2] + (1,) , dtype=prd_face_bgr.dtype) if self.masked_hist_match: hist_mask_a *= prd_face_mask_a new_prd_face_bgr = image_utils.color_hist_match(prd_face_bgr*hist_mask_a, dst_face_bgr*hist_mask_a ) prd_face_bgr = new_prd_face_bgr if self.mode == 'hist-match-bw': prd_face_bgr = prd_face_bgr.astype(np.float32) out_img = cv2.warpAffine( prd_face_bgr, face_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ) if debug: debugs += [out_img.copy()] debugs += [img_mask_blurry_aaa.copy()] if self.mode == 'seamless' or self.mode == 'seamless-hist-match': out_img = np.clip( img_bgr*(1-img_face_mask_aaa) + (out_img*img_face_mask_aaa) , 0, 1.0 ) if debug: debugs += [out_img.copy()] out_img = cv2.seamlessClone( (out_img*255).astype(np.uint8), (img_bgr*255).astype(np.uint8), (img_face_mask_flatten_aaa*255).astype(np.uint8), (masky,maskx) , cv2.NORMAL_CLONE ) out_img = out_img.astype(np.float32) / 255.0 if debug: debugs += [out_img.copy()] if self.clip_border_mask_per > 0: img_prd_border_rect_mask_a = cv2.warpAffine( prd_border_rect_mask_a, face_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ) img_prd_border_rect_mask_a = np.expand_dims (img_prd_border_rect_mask_a, -1) out_img = out_img * img_prd_border_rect_mask_a + img_bgr * (1.0 - img_prd_border_rect_mask_a) img_mask_blurry_aaa *= img_prd_border_rect_mask_a out_img = np.clip( img_bgr*(1-img_mask_blurry_aaa) + (out_img*img_mask_blurry_aaa) , 0, 1.0 ) if self.mode == 'seamless-hist-match': out_face_bgr = cv2.warpAffine( out_img, face_mat, (self.output_size, self.output_size) ) new_out_face_bgr = image_utils.color_hist_match(out_face_bgr, dst_face_bgr ) new_out = cv2.warpAffine( new_out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT ) out_img = np.clip( img_bgr*(1-img_mask_blurry_aaa) + (new_out*img_mask_blurry_aaa) , 0, 1.0 ) if self.transfercolor: #making transfer color from original DST image to fake from skimage import io, color lab_clr = color.rgb2lab(img_bgr) #original DST, converting RGB to LAB color space lab_bw = color.rgb2lab(out_img) #fake, converting RGB to LAB color space tmp_channel, a_channel, b_channel = cv2.split(lab_clr) #taking color channel A and B from original dst image l_channel, tmp2_channel, tmp3_channel = cv2.split(lab_bw) #taking lightness channel L from merged fake img_LAB = cv2.merge((l_channel,a_channel, b_channel)) #merging light and color out_img = color.lab2rgb(img_LAB) #converting LAB to RGB if self.alpha: new_image = out_img.copy() new_image = (new_image*255).astype(np.uint8) #convert image to int b_channel, g_channel, r_channel = cv2.split(new_image) #splitting RGB alpha_channel = img_mask_blurry_aaa.copy() #making copy of alpha channel alpha_channel = (alpha_channel*255).astype(np.uint8) alpha_channel, tmp2, tmp3 = cv2.split(alpha_channel) #splitting alpha to three channels, they all same in original alpha channel, we need just one out_img = cv2.merge((b_channel,g_channel, r_channel, alpha_channel)) #mergin RGB with alpha out_img = out_img.astype(np.float32) / 255.0 if debug: debugs += [out_img.copy()] return debugs if debug else out_img
def process(sample, sample_process_options, output_sample_types, debug): SPTF = SampleProcessor.TypeFlags sample_bgr = sample.load_bgr() h, w, c = sample_bgr.shape is_face_sample = sample.landmarks is not None if debug and is_face_sample: LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks, (0, 1, 0)) close_sample = sample.close_target_list[np.random.randint( 0, len(sample.close_target_list) )] if sample.close_target_list is not None else None close_sample_bgr = close_sample.load_bgr( ) if close_sample is not None else None if debug and close_sample_bgr is not None: LandmarksProcessor.draw_landmarks(close_sample_bgr, close_sample.landmarks, (0, 1, 0)) params = imagelib.gen_warp_params( sample_bgr, sample_process_options.random_flip, rotation_range=sample_process_options.rotation_range, scale_range=sample_process_options.scale_range, tx_range=sample_process_options.tx_range, ty_range=sample_process_options.ty_range) images = [[None] * 3 for _ in range(30)] sample_rnd_seed = np.random.randint(0x80000000) outputs = [] for sample_type in output_sample_types: f = sample_type[0] size = sample_type[1] random_sub_size = 0 if len(sample_type) < 3 else min( sample_type[2], size) if f & SPTF.SOURCE != 0: img_type = 0 elif f & SPTF.WARPED != 0: img_type = 1 elif f & SPTF.WARPED_TRANSFORMED != 0: img_type = 2 elif f & SPTF.TRANSFORMED != 0: img_type = 3 elif f & SPTF.LANDMARKS_ARRAY != 0: img_type = 4 else: raise ValueError('expected SampleTypeFlags type') if f & SPTF.RANDOM_CLOSE != 0: img_type += 10 elif f & SPTF.MORPH_TO_RANDOM_CLOSE != 0: img_type += 20 face_mask_type = 0 if f & SPTF.FACE_MASK_FULL != 0: face_mask_type = 1 elif f & SPTF.FACE_MASK_EYES != 0: face_mask_type = 2 target_face_type = -1 if f & SPTF.FACE_TYPE_HALF != 0: target_face_type = FaceType.HALF elif f & SPTF.FACE_TYPE_FULL != 0: target_face_type = FaceType.FULL elif f & SPTF.FACE_TYPE_HEAD != 0: target_face_type = FaceType.HEAD elif f & SPTF.FACE_TYPE_AVATAR != 0: target_face_type = FaceType.AVATAR apply_motion_blur = f & SPTF.OPT_APPLY_MOTION_BLUR != 0 if img_type == 4: l = sample.landmarks l = np.concatenate([ np.expand_dims(l[:, 0] / w, -1), np.expand_dims(l[:, 1] / h, -1) ], -1) l = np.clip(l, 0.0, 1.0) img = l else: if images[img_type][face_mask_type] is None: if img_type >= 10 and img_type <= 19: #RANDOM_CLOSE img_type -= 10 img = close_sample_bgr cur_sample = close_sample elif img_type >= 20 and img_type <= 29: #MORPH_TO_RANDOM_CLOSE img_type -= 20 res = sample.shape[0] s_landmarks = sample.landmarks.copy() d_landmarks = close_sample.landmarks.copy() idxs = list(range(len(s_landmarks))) #remove landmarks near boundaries for i in idxs[:]: s_l = s_landmarks[i] d_l = d_landmarks[i] if s_l[0] < 5 or s_l[1] < 5 or s_l[0] >= res-5 or s_l[1] >= res-5 or \ d_l[0] < 5 or d_l[1] < 5 or d_l[0] >= res-5 or d_l[1] >= res-5: idxs.remove(i) #remove landmarks that close to each other in 5 dist for landmarks in [s_landmarks, d_landmarks]: for i in idxs[:]: s_l = landmarks[i] for j in idxs[:]: if i == j: continue s_l_2 = landmarks[j] diff_l = np.abs(s_l - s_l_2) if np.sqrt(diff_l.dot(diff_l)) < 5: idxs.remove(i) break s_landmarks = s_landmarks[idxs] d_landmarks = d_landmarks[idxs] s_landmarks = np.concatenate([ s_landmarks, [[0, 0], [res // 2, 0], [res - 1, 0], [0, res // 2], [res - 1, res // 2], [0, res - 1], [res // 2, res - 1], [res - 1, res - 1]] ]) d_landmarks = np.concatenate([ d_landmarks, [[0, 0], [res // 2, 0], [res - 1, 0], [0, res // 2], [res - 1, res // 2], [0, res - 1], [res // 2, res - 1], [res - 1, res - 1]] ]) img = imagelib.morph_by_points(sample_bgr, s_landmarks, d_landmarks) cur_sample = close_sample else: img = sample_bgr cur_sample = sample if is_face_sample: if apply_motion_blur and sample_process_options.motion_blur is not None: chance, mb_range = sample_process_options.motion_blur if np.random.randint(100) < chance: dim = mb_range[np.random.randint( len(mb_range))] img = imagelib.LinearMotionBlur( img, dim, np.random.randint(180)) if face_mask_type == 1: mask = cur_sample.load_fanseg_mask( ) #using fanseg_mask if exist if mask is None: mask = LandmarksProcessor.get_image_hull_mask( img.shape, cur_sample.landmarks) if cur_sample.ie_polys is not None: cur_sample.ie_polys.overlay_mask(mask) img = np.concatenate((img, mask), -1) elif face_mask_type == 2: mask = LandmarksProcessor.get_image_eye_mask( img.shape, cur_sample.landmarks) mask = np.expand_dims( cv2.blur(mask, (w // 32, w // 32)), -1) mask[mask > 0.0] = 1.0 img = np.concatenate((img, mask), -1) images[img_type][face_mask_type] = imagelib.warp_by_params( params, img, (img_type == 1 or img_type == 2), (img_type == 2 or img_type == 3), img_type != 0, face_mask_type == 0) img = images[img_type][face_mask_type] if is_face_sample and target_face_type != -1: if target_face_type > sample.face_type: raise Exception( 'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.' % (sample.filename, sample.face_type, target_face_type)) img = cv2.warpAffine(img, LandmarksProcessor.get_transform_mat( sample.landmarks, size, target_face_type), (size, size), flags=cv2.INTER_CUBIC) else: img = cv2.resize(img, (size, size), cv2.INTER_CUBIC) if random_sub_size != 0: sub_size = size - random_sub_size rnd_state = np.random.RandomState(sample_rnd_seed + random_sub_size) start_x = rnd_state.randint(sub_size + 1) start_y = rnd_state.randint(sub_size + 1) img = img[start_y:start_y + sub_size, start_x:start_x + sub_size, :] img_bgr = img[..., 0:3] img_mask = img[..., 3:4] if f & SPTF.MODE_BGR != 0: img = img_bgr elif f & SPTF.MODE_BGR_SHUFFLE != 0: rnd_state = np.random.RandomState(sample_rnd_seed) img_bgr = np.take(img_bgr, rnd_state.permutation(img_bgr.shape[-1]), axis=-1) img = np.concatenate((img_bgr, img_mask), -1) elif f & SPTF.MODE_G != 0: img = np.concatenate((np.expand_dims( cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1), img_mask), -1) elif f & SPTF.MODE_GGG != 0: img = np.concatenate((np.repeat( np.expand_dims( cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1), (3, ), -1), img_mask), -1) elif is_face_sample and f & SPTF.MODE_M != 0: if face_mask_type == 0: raise ValueError('no face_mask_type defined') img = img_mask else: raise ValueError('expected SampleTypeFlags mode') if not debug: if sample_process_options.normalize_tanh: img = np.clip(img * 2.0 - 1.0, -1.0, 1.0) else: img = np.clip(img, 0.0, 1.0) outputs.append(img) if debug: result = [] for output in outputs: if output.shape[2] < 4: result += [ output, ] elif output.shape[2] == 4: result += [ output[..., 0:3] * output[..., 3:4], ] return result else: return outputs
def process(sample, sample_process_options, output_sample_types, debug, ct_sample=None): SPTF = SampleProcessor.Types sample_bgr = sample.load_bgr() ct_sample_bgr = None ct_sample_mask = None h, w, c = sample_bgr.shape is_face_sample = sample.landmarks is not None if debug and is_face_sample: LandmarksProcessor.draw_landmarks(sample_bgr, sample.landmarks, (0, 1, 0)) params = imagelib.gen_warp_params( sample_bgr, sample_process_options.random_flip, rotation_range=sample_process_options.rotation_range, scale_range=sample_process_options.scale_range, tx_range=sample_process_options.tx_range, ty_range=sample_process_options.ty_range) cached_images = collections.defaultdict(dict) sample_rnd_seed = np.random.randint(0x80000000) SPTF_FACETYPE_TO_FACETYPE = { SPTF.FACE_TYPE_HALF: FaceType.HALF, SPTF.FACE_TYPE_FULL: FaceType.FULL, SPTF.FACE_TYPE_HEAD: FaceType.HEAD, SPTF.FACE_TYPE_FULL_NO_ALIGN: FaceType.FULL_NO_ALIGN } outputs = [] for opts in output_sample_types: resolution = opts.get('resolution', 0) types = opts.get('types', []) random_sub_res = opts.get('random_sub_res', 0) normalize_std_dev = opts.get('normalize_std_dev', False) normalize_vgg = opts.get('normalize_vgg', False) motion_blur = opts.get('motion_blur', None) apply_ct = opts.get('apply_ct', False) normalize_tanh = opts.get('normalize_tanh', False) img_type = SPTF.NONE target_face_type = SPTF.NONE face_mask_type = SPTF.NONE mode_type = SPTF.NONE for t in types: if t >= SPTF.IMG_TYPE_BEGIN and t < SPTF.IMG_TYPE_END: img_type = t elif t >= SPTF.FACE_TYPE_BEGIN and t < SPTF.FACE_TYPE_END: target_face_type = t elif t >= SPTF.MODE_BEGIN and t < SPTF.MODE_END: mode_type = t if img_type == SPTF.NONE: raise ValueError('expected IMG_ type') if img_type == SPTF.IMG_LANDMARKS_ARRAY: l = sample.landmarks l = np.concatenate([ np.expand_dims(l[:, 0] / w, -1), np.expand_dims(l[:, 1] / h, -1) ], -1) l = np.clip(l, 0.0, 1.0) img = l elif img_type == SPTF.IMG_PITCH_YAW_ROLL or img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID: pitch_yaw_roll = sample.pitch_yaw_roll if pitch_yaw_roll is not None: pitch, yaw, roll = pitch_yaw_roll else: pitch, yaw, roll = LandmarksProcessor.estimate_pitch_yaw_roll( sample.landmarks) if params['flip']: yaw = -yaw if img_type == SPTF.IMG_PITCH_YAW_ROLL_SIGMOID: pitch = (pitch + 1.0) / 2.0 yaw = (yaw + 1.0) / 2.0 roll = (roll + 1.0) / 2.0 img = (pitch, yaw, roll) else: if mode_type == SPTF.NONE: raise ValueError('expected MODE_ type') def do_transform(img, mask): warp = (img_type == SPTF.IMG_WARPED or img_type == SPTF.IMG_WARPED_TRANSFORMED) transform = (img_type == SPTF.IMG_WARPED_TRANSFORMED or img_type == SPTF.IMG_TRANSFORMED) flip = img_type != SPTF.IMG_WARPED img = imagelib.warp_by_params(params, img, warp, transform, flip, True) if mask is not None: mask = imagelib.warp_by_params(params, mask, warp, transform, flip, False) if len(mask.shape) == 2: mask = mask[..., np.newaxis] img = np.concatenate((img, mask), -1) return img img = cached_images.get(img_type, None) if img is None: img = sample_bgr mask = None cur_sample = sample if is_face_sample: if motion_blur is not None: chance, mb_range = motion_blur chance = np.clip(chance, 0, 100) if np.random.randint(100) < chance: mb_range = [3, 5, 7, 9][:np.clip(mb_range, 0, 3) + 1] dim = mb_range[np.random.randint( len(mb_range))] img = imagelib.LinearMotionBlur( img, dim, np.random.randint(180)) mask = cur_sample.load_fanseg_mask( ) #using fanseg_mask if exist if mask is None: mask = LandmarksProcessor.get_image_hull_mask( img.shape, cur_sample.landmarks) if cur_sample.ie_polys is not None: cur_sample.ie_polys.overlay_mask(mask) if sample.face_type == FaceType.MARK_ONLY: if mask is not None: img = np.concatenate((img, mask), -1) else: img = do_transform(img, mask) cached_images[img_type] = img if is_face_sample and target_face_type != SPTF.NONE: ft = SPTF_FACETYPE_TO_FACETYPE[target_face_type] if ft > sample.face_type: raise Exception( 'sample %s type %s does not match model requirement %s. Consider extract necessary type of faces.' % (sample.filename, sample.face_type, ft)) if sample.face_type == FaceType.MARK_ONLY: img = cv2.warpAffine( img, LandmarksProcessor.get_transform_mat( sample.landmarks, sample.shape[0], ft), (sample.shape[0], sample.shape[0]), flags=cv2.INTER_CUBIC) mask = img[..., 3:4] if img.shape[2] > 3 else None img = img[..., 0:3] img = do_transform(img, mask) img = cv2.resize(img, (resolution, resolution), cv2.INTER_CUBIC) else: img = cv2.warpAffine( img, LandmarksProcessor.get_transform_mat( sample.landmarks, resolution, ft), (resolution, resolution), flags=cv2.INTER_CUBIC) else: img = cv2.resize(img, (resolution, resolution), cv2.INTER_CUBIC) if random_sub_res != 0: sub_size = resolution - random_sub_res rnd_state = np.random.RandomState(sample_rnd_seed + random_sub_res) start_x = rnd_state.randint(sub_size + 1) start_y = rnd_state.randint(sub_size + 1) img = img[start_y:start_y + sub_size, start_x:start_x + sub_size, :] img = np.clip(img, 0, 1) img_bgr = img[..., 0:3] img_mask = img[..., 3:4] if apply_ct and ct_sample is not None: if ct_sample_bgr is None: ct_sample_bgr = ct_sample.load_bgr() ct_sample_bgr_resized = cv2.resize( ct_sample_bgr, (resolution, resolution), cv2.INTER_LINEAR) img_bgr = imagelib.linear_color_transfer( img_bgr, ct_sample_bgr_resized) img_bgr = np.clip(img_bgr, 0.0, 1.0) if normalize_std_dev: img_bgr = (img_bgr - img_bgr.mean((0, 1))) / img_bgr.std( (0, 1)) elif normalize_vgg: img_bgr = np.clip(img_bgr * 255, 0, 255) img_bgr[:, :, 0] -= 103.939 img_bgr[:, :, 1] -= 116.779 img_bgr[:, :, 2] -= 123.68 if mode_type == SPTF.MODE_BGR: img = img_bgr elif mode_type == SPTF.MODE_BGR_SHUFFLE: rnd_state = np.random.RandomState(sample_rnd_seed) img = np.take(img_bgr, rnd_state.permutation(img_bgr.shape[-1]), axis=-1) elif mode_type == SPTF.MODE_G: img = np.concatenate((np.expand_dims( cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1), img_mask), -1) elif mode_type == SPTF.MODE_GGG: img = np.concatenate((np.repeat( np.expand_dims( cv2.cvtColor(img_bgr, cv2.COLOR_BGR2GRAY), -1), (3, ), -1), img_mask), -1) elif mode_type == SPTF.MODE_M and is_face_sample: img = img_mask if not debug: if normalize_tanh: img = np.clip(img * 2.0 - 1.0, -1.0, 1.0) else: img = np.clip(img, 0.0, 1.0) outputs.append(img) if debug: result = [] for output in outputs: if output.shape[2] < 4: result += [ output, ] elif output.shape[2] == 4: result += [ output[..., 0:3] * output[..., 3:4], ] return result else: return outputs
def ConvertMaskedFace(cfg, frame_info, img_bgr_uint8, img_bgr, img_face_landmarks): #if debug: # debugs = [img_bgr.copy()] img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask( img_bgr.shape, img_face_landmarks) if cfg.mode == 'original': if cfg.export_mask_alpha: img_bgr = np.concatenate([img_bgr, img_face_mask_a], -1) return img_bgr, img_face_mask_a out_img = img_bgr.copy() out_merging_mask = None output_size = cfg.predictor_input_shape[0] if cfg.super_resolution_mode != 0: output_size *= 2 face_mat = LandmarksProcessor.get_transform_mat(img_face_landmarks, output_size, face_type=cfg.face_type) face_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, output_size, face_type=cfg.face_type, scale=1.0 + 0.01 * cfg.output_face_scale) dst_face_bgr = cv2.warpAffine(img_bgr, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) dst_face_mask_a_0 = cv2.warpAffine(img_face_mask_a, face_mat, (output_size, output_size), flags=cv2.INTER_CUBIC) predictor_input_bgr = cv2.resize(dst_face_bgr, cfg.predictor_input_shape[0:2]) if cfg.predictor_masked: prd_face_bgr, prd_face_mask_a_0 = cfg.predictor_func( predictor_input_bgr) prd_face_bgr = np.clip(prd_face_bgr, 0, 1.0) prd_face_mask_a_0 = np.clip(prd_face_mask_a_0, 0.0, 1.0) else: predicted = cfg.predictor_func(predictor_input_bgr) prd_face_bgr = np.clip(predicted, 0, 1.0) prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, cfg.predictor_input_shape[0:2]) if cfg.super_resolution_mode: #if debug: # tmp = cv2.resize (prd_face_bgr, (output_size,output_size), cv2.INTER_CUBIC) # debugs += [ np.clip( cv2.warpAffine( tmp, face_output_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] prd_face_bgr = cfg.superres_func(cfg.super_resolution_mode, prd_face_bgr) #if debug: # debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] if cfg.predictor_masked: prd_face_mask_a_0 = cv2.resize(prd_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) else: prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) if cfg.mask_mode == 2: #dst prd_face_mask_a_0 = cv2.resize(dst_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) elif cfg.mask_mode >= 3 and cfg.mask_mode <= 7: if cfg.mask_mode == 3 or cfg.mask_mode == 5 or cfg.mask_mode == 6: prd_face_fanseg_bgr = cv2.resize(prd_face_bgr, (cfg.fanseg_input_size, ) * 2) prd_face_fanseg_mask = cfg.fanseg_extract_func( FaceType.FULL, prd_face_fanseg_bgr) FAN_prd_face_mask_a_0 = cv2.resize(prd_face_fanseg_mask, (output_size, output_size), cv2.INTER_CUBIC) if cfg.mask_mode >= 4 or cfg.mask_mode <= 7: full_face_fanseg_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, cfg.fanseg_input_size, face_type=FaceType.FULL) dst_face_fanseg_bgr = cv2.warpAffine(img_bgr, full_face_fanseg_mat, (cfg.fanseg_input_size, ) * 2, flags=cv2.INTER_CUBIC) dst_face_fanseg_mask = cfg.fanseg_extract_func( FaceType.FULL, dst_face_fanseg_bgr) if cfg.face_type == FaceType.FULL: FAN_dst_face_mask_a_0 = cv2.resize(dst_face_fanseg_mask, (output_size, output_size), cv2.INTER_CUBIC) elif cfg.face_type == FaceType.HALF: half_face_fanseg_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, cfg.fanseg_input_size, face_type=FaceType.HALF) fanseg_rect_corner_pts = np.array( [[0, 0], [cfg.fanseg_input_size - 1, 0], [0, cfg.fanseg_input_size - 1]], dtype=np.float32) a = LandmarksProcessor.transform_points(fanseg_rect_corner_pts, half_face_fanseg_mat, invert=True) b = LandmarksProcessor.transform_points( a, full_face_fanseg_mat) m = cv2.getAffineTransform(b, fanseg_rect_corner_pts) FAN_dst_face_mask_a_0 = cv2.warpAffine( dst_face_fanseg_mask, m, (cfg.fanseg_input_size, ) * 2, flags=cv2.INTER_CUBIC) FAN_dst_face_mask_a_0 = cv2.resize(FAN_dst_face_mask_a_0, (output_size, output_size), cv2.INTER_CUBIC) else: raise ValueError("cfg.face_type unsupported") if cfg.mask_mode == 3: #FAN-prd prd_face_mask_a_0 = FAN_prd_face_mask_a_0 elif cfg.mask_mode == 4: #FAN-dst prd_face_mask_a_0 = FAN_dst_face_mask_a_0 elif cfg.mask_mode == 5: prd_face_mask_a_0 = FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0 elif cfg.mask_mode == 6: prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_prd_face_mask_a_0 * FAN_dst_face_mask_a_0 elif cfg.mask_mode == 7: prd_face_mask_a_0 = prd_face_mask_a_0 * FAN_dst_face_mask_a_0 prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0 prd_face_mask_a = prd_face_mask_a_0[..., np.newaxis] prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1) img_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC) img_face_mask_aaa = np.clip(img_face_mask_aaa, 0.0, 1.0) img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 #get rid of noise #if debug: # debugs += [img_face_mask_aaa.copy()] if 'raw' in cfg.mode: face_corner_pts = np.array( [[0, 0], [output_size - 1, 0], [output_size - 1, output_size - 1], [0, output_size - 1]], dtype=np.float32) square_mask = np.zeros(img_bgr.shape, dtype=np.float32) cv2.fillConvexPoly(square_mask, \ LandmarksProcessor.transform_points (face_corner_pts, face_output_mat, invert=True ).astype(np.int), \ (1,1,1) ) if cfg.mode == 'raw-rgb': out_merging_mask = square_mask if cfg.mode == 'raw-rgb' or cfg.mode == 'raw-rgb-mask': out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) if cfg.mode == 'raw-rgb-mask': out_img = np.concatenate( [out_img, np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1) out_merging_mask = square_mask elif cfg.mode == 'raw-mask-only': out_img = img_face_mask_aaa out_merging_mask = img_face_mask_aaa elif cfg.mode == 'raw-predicted-only': out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) out_merging_mask = square_mask out_img = np.clip(out_img, 0.0, 1.0) else: #averaging [lenx, leny, maskx, masky] by grayscale gradients of upscaled mask ar = [] for i in range(1, 10): maxregion = np.argwhere(img_face_mask_aaa > i / 10.0) if maxregion.size != 0: miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] lenx = maxx - minx leny = maxy - miny if min(lenx, leny) >= 4: ar += [[lenx, leny]] if len(ar) > 0: lenx, leny = np.mean(ar, axis=0) lowest_len = min(lenx, leny) #if debug: # io.log_info ("lenx/leny:(%d/%d) " % (lenx, leny ) ) # io.log_info ("lowest_len = %f" % (lowest_len) ) if cfg.erode_mask_modifier != 0: ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) * 0.01 * cfg.erode_mask_modifier) #if debug: # io.log_info ("erode_size = %d" % (ero) ) if ero > 0: img_face_mask_aaa = cv2.erode(img_face_mask_aaa, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: img_face_mask_aaa = cv2.dilate(img_face_mask_aaa, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) if cfg.clip_hborder_mask_per > 0: #clip hborder before blur prd_hborder_rect_mask_a = np.ones(prd_face_mask_a.shape, dtype=np.float32) prd_border_size = int(prd_hborder_rect_mask_a.shape[1] * cfg.clip_hborder_mask_per) prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0 prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0 prd_hborder_rect_mask_a[-prd_border_size:, :, :] = 0 prd_hborder_rect_mask_a = np.expand_dims( cv2.blur(prd_hborder_rect_mask_a, (prd_border_size, prd_border_size)), -1) img_prd_hborder_rect_mask_a = cv2.warpAffine( prd_hborder_rect_mask_a, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC) img_prd_hborder_rect_mask_a = np.expand_dims( img_prd_hborder_rect_mask_a, -1) img_face_mask_aaa *= img_prd_hborder_rect_mask_a img_face_mask_aaa = np.clip(img_face_mask_aaa, 0, 1.0) #if debug: # debugs += [img_face_mask_aaa.copy()] if cfg.blur_mask_modifier > 0: blur = int(lowest_len * 0.10 * 0.01 * cfg.blur_mask_modifier) #if debug: # io.log_info ("blur_size = %d" % (blur) ) if blur > 0: img_face_mask_aaa = cv2.blur(img_face_mask_aaa, (blur, blur)) img_face_mask_aaa = np.clip(img_face_mask_aaa, 0, 1.0) #if debug: # debugs += [img_face_mask_aaa.copy()] if 'seamless' not in cfg.mode and cfg.color_transfer_mode != 0: if cfg.color_transfer_mode == 1: #if debug: # debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] prd_face_bgr = imagelib.reinhard_color_transfer( np.clip((prd_face_bgr * 255).astype(np.uint8), 0, 255), np.clip((dst_face_bgr * 255).astype(np.uint8), 0, 255), source_mask=prd_face_mask_a, target_mask=prd_face_mask_a) prd_face_bgr = np.clip( prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) #if debug: # debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] elif cfg.color_transfer_mode == 2: #if debug: # debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] prd_face_bgr = imagelib.linear_color_transfer( prd_face_bgr, dst_face_bgr) prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0) #if debug: # debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] if cfg.mode == 'hist-match-bw': prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY) prd_face_bgr = np.repeat(np.expand_dims(prd_face_bgr, -1), (3, ), -1) if cfg.mode == 'hist-match' or cfg.mode == 'hist-match-bw': #if debug: # debugs += [ cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ) ] hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) if cfg.masked_hist_match: hist_mask_a *= prd_face_mask_a white = (1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_1 = prd_face_bgr * hist_mask_a + white hist_match_1[hist_match_1 > 1.0] = 1.0 hist_match_2 = dst_face_bgr * hist_mask_a + white hist_match_2[hist_match_1 > 1.0] = 1.0 prd_face_bgr = imagelib.color_hist_match( hist_match_1, hist_match_2, cfg.hist_match_threshold) #if cfg.masked_hist_match: # prd_face_bgr -= white if cfg.mode == 'hist-match-bw': prd_face_bgr = prd_face_bgr.astype(dtype=np.float32) out_img = cv2.warpAffine(prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) out_img = np.clip(out_img, 0.0, 1.0) #if debug: # debugs += [out_img.copy()] if cfg.mode == 'overlay': pass if 'seamless' in cfg.mode: #mask used for cv2.seamlessClone img_face_seamless_mask_a = None img_face_mask_a = img_face_mask_aaa[..., 0:1] for i in range(1, 10): a = img_face_mask_a > i / 10.0 if len(np.argwhere(a)) == 0: continue img_face_seamless_mask_a = img_face_mask_aaa[..., 0:1].copy() img_face_seamless_mask_a[a] = 1.0 img_face_seamless_mask_a[img_face_seamless_mask_a <= i / 10.0] = 0.0 break try: #calc same bounding rect and center point as in cv2.seamlessClone to prevent jittering (not flickering) l, t, w, h = cv2.boundingRect( (img_face_seamless_mask_a * 255).astype(np.uint8)) s_maskx, s_masky = int(l + w / 2), int(t + h / 2) out_img = cv2.seamlessClone( (out_img * 255).astype(np.uint8), img_bgr_uint8, (img_face_seamless_mask_a * 255).astype(np.uint8), (s_maskx, s_masky), cv2.NORMAL_CLONE) out_img = out_img.astype(dtype=np.float32) / 255.0 except Exception as e: #seamlessClone may fail in some cases e_str = traceback.format_exc() if 'MemoryError' in e_str: raise Exception( "Seamless fail: " + e_str ) #reraise MemoryError in order to reprocess this data by other processes else: print("Seamless fail: " + e_str) #if debug: # debugs += [out_img.copy()] out_img = img_bgr * (1 - img_face_mask_aaa) + (out_img * img_face_mask_aaa) out_face_bgr = cv2.warpAffine(out_img, face_mat, (output_size, output_size)) if 'seamless' in cfg.mode and cfg.color_transfer_mode != 0: if cfg.color_transfer_mode == 1: #if debug: # debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] face_mask_aaa = cv2.warpAffine(img_face_mask_aaa, face_mat, (output_size, output_size)) out_face_bgr = imagelib.reinhard_color_transfer( np.clip((out_face_bgr * 255).astype(np.uint8), 0, 255), np.clip((dst_face_bgr * 255).astype(np.uint8), 0, 255), source_mask=face_mask_aaa, target_mask=face_mask_aaa) out_face_bgr = np.clip( out_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) #if debug: # debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] elif cfg.color_transfer_mode == 2: #if debug: # debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] out_face_bgr = imagelib.linear_color_transfer( out_face_bgr, dst_face_bgr) out_face_bgr = np.clip(out_face_bgr, 0.0, 1.0) #if debug: # debugs += [ np.clip( cv2.warpAffine( out_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT ), 0, 1.0) ] if cfg.mode == 'seamless-hist-match': out_face_bgr = imagelib.color_hist_match( out_face_bgr, dst_face_bgr, cfg.hist_match_threshold) cfg_mp = cfg.motion_blur_power / 100.0 if cfg_mp != 0: k_size = int(frame_info.motion_power * cfg_mp) if k_size >= 1: k_size = np.clip(k_size + 1, 2, 50) if cfg.super_resolution_mode: k_size *= 2 out_face_bgr = imagelib.LinearMotionBlur( out_face_bgr, k_size, frame_info.motion_deg) if cfg.sharpen_mode != 0 and cfg.sharpen_amount != 0: out_face_bgr = cfg.sharpen_func(out_face_bgr, cfg.sharpen_mode, 3, cfg.sharpen_amount) new_out = cv2.warpAffine(out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_CUBIC, cv2.BORDER_TRANSPARENT) out_img = np.clip( img_bgr * (1 - img_face_mask_aaa) + (new_out * img_face_mask_aaa), 0, 1.0) if cfg.color_degrade_power != 0: #if debug: # debugs += [out_img.copy()] out_img_reduced = imagelib.reduce_colors(out_img, 256) if cfg.color_degrade_power == 100: out_img = out_img_reduced else: alpha = cfg.color_degrade_power / 100.0 out_img = (out_img * (1.0 - alpha) + out_img_reduced * alpha) if cfg.export_mask_alpha: out_img = np.concatenate( [out_img, img_face_mask_aaa[:, :, 0:1]], -1) out_merging_mask = img_face_mask_aaa #if debug: # debugs += [out_img.copy()] return out_img, out_merging_mask
def convert_face(self, img_bgr, img_face_landmarks, debug): if debug: debugs = [img_bgr.copy()] img_size = img_bgr.shape[1], img_bgr.shape[0] img_face_mask_a = LandmarksProcessor.get_image_hull_mask( img_bgr.shape, img_face_landmarks) face_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, self.output_size, face_type=self.face_type) face_output_mat = LandmarksProcessor.get_transform_mat( img_face_landmarks, self.output_size, face_type=self.face_type, scale=self.output_face_scale) dst_face_bgr = cv2.warpAffine(img_bgr, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4) dst_face_mask_a_0 = cv2.warpAffine( img_face_mask_a, face_mat, (self.output_size, self.output_size), flags=cv2.INTER_LANCZOS4) predictor_input_bgr = cv2.resize( dst_face_bgr, (self.predictor_input_size, self.predictor_input_size)) predictor_input_mask_a_0 = cv2.resize( dst_face_mask_a_0, (self.predictor_input_size, self.predictor_input_size)) predictor_input_mask_a = np.expand_dims(predictor_input_mask_a_0, -1) predicted_bgra = self.predictor( np.concatenate((predictor_input_bgr, predictor_input_mask_a), -1)) prd_face_bgr = np.clip(predicted_bgra[:, :, 0:3], 0, 1.0) prd_face_mask_a_0 = np.clip(predicted_bgra[:, :, 3], 0.0, 1.0) if not self.use_predicted_mask: prd_face_mask_a_0 = predictor_input_mask_a_0 prd_face_mask_a_0[prd_face_mask_a_0 < 0.001] = 0.0 prd_face_mask_a = np.expand_dims(prd_face_mask_a_0, axis=-1) prd_face_mask_aaa = np.repeat(prd_face_mask_a, (3, ), axis=-1) img_prd_face_mask_aaa = cv2.warpAffine(prd_face_mask_aaa, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), flags=cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4) img_prd_face_mask_aaa = np.clip(img_prd_face_mask_aaa, 0.0, 1.0) img_face_mask_aaa = img_prd_face_mask_aaa if debug: debugs += [img_face_mask_aaa.copy()] img_face_mask_aaa[img_face_mask_aaa <= 0.1] = 0.0 img_face_mask_flatten_aaa = img_face_mask_aaa.copy() img_face_mask_flatten_aaa[img_face_mask_flatten_aaa > 0.9] = 1.0 maxregion = np.argwhere(img_face_mask_flatten_aaa == 1.0) out_img = img_bgr.copy() if self.mode == 'raw': if self.raw_mode == 'rgb' or self.raw_mode == 'rgb-mask': out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) if self.raw_mode == 'rgb-mask': out_img = np.concatenate( [out_img, np.expand_dims(img_face_mask_aaa[:, :, 0], -1)], -1) if self.raw_mode == 'mask-only': out_img = img_face_mask_aaa if self.raw_mode == 'predicted-only': out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(out_img.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) else: if maxregion.size != 0: miny, minx = maxregion.min(axis=0)[:2] maxy, maxx = maxregion.max(axis=0)[:2] if debug: print( "maxregion.size: %d, minx:%d, maxx:%d miny:%d, maxy:%d" % (maxregion.size, minx, maxx, miny, maxy)) lenx = maxx - minx leny = maxy - miny if lenx >= 4 and leny >= 4: masky = int(minx + (lenx // 2)) maskx = int(miny + (leny // 2)) lowest_len = min(lenx, leny) if debug: print("lowest_len = %f" % (lowest_len)) img_mask_blurry_aaa = img_face_mask_aaa if self.erode_mask_modifier != 0: ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) * 0.01 * self.erode_mask_modifier) if debug: print("erode_size = %d" % (ero)) if ero > 0: img_mask_blurry_aaa = cv2.erode( img_mask_blurry_aaa, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: img_mask_blurry_aaa = cv2.dilate( img_mask_blurry_aaa, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) if self.seamless_erode_mask_modifier != 0: ero = int(lowest_len * (0.126 - lowest_len * 0.00004551365) * 0.01 * self.seamless_erode_mask_modifier) if debug: print("seamless_erode_size = %d" % (ero)) if ero > 0: img_face_mask_flatten_aaa = cv2.erode( img_face_mask_flatten_aaa, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (ero, ero)), iterations=1) elif ero < 0: img_face_mask_flatten_aaa = cv2.dilate( img_face_mask_flatten_aaa, cv2.getStructuringElement( cv2.MORPH_ELLIPSE, (-ero, -ero)), iterations=1) if self.clip_hborder_mask_per > 0: #clip hborder before blur prd_hborder_rect_mask_a = np.ones( prd_face_mask_a.shape, dtype=np.float32) prd_border_size = int( prd_hborder_rect_mask_a.shape[1] * self.clip_hborder_mask_per) prd_hborder_rect_mask_a[:, 0:prd_border_size, :] = 0 prd_hborder_rect_mask_a[:, -prd_border_size:, :] = 0 prd_hborder_rect_mask_a = np.expand_dims( cv2.blur(prd_hborder_rect_mask_a, (prd_border_size, prd_border_size)), -1) img_prd_hborder_rect_mask_a = cv2.warpAffine( prd_hborder_rect_mask_a, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4) img_prd_hborder_rect_mask_a = np.expand_dims( img_prd_hborder_rect_mask_a, -1) img_mask_blurry_aaa *= img_prd_hborder_rect_mask_a img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0) if debug: debugs += [img_mask_blurry_aaa.copy()] if self.blur_mask_modifier > 0: blur = int(lowest_len * 0.10 * 0.01 * self.blur_mask_modifier) if debug: print("blur_size = %d" % (blur)) if blur > 0: img_mask_blurry_aaa = cv2.blur( img_mask_blurry_aaa, (blur, blur)) img_mask_blurry_aaa = np.clip(img_mask_blurry_aaa, 0, 1.0) if debug: debugs += [img_mask_blurry_aaa.copy()] if self.color_transfer_mode is not None: if self.color_transfer_mode == 'rct': if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] prd_face_bgr = image_utils.reinhard_color_transfer( np.clip((prd_face_bgr * 255).astype(np.uint8), 0, 255), np.clip((dst_face_bgr * 255).astype(np.uint8), 0, 255), source_mask=prd_face_mask_a, target_mask=prd_face_mask_a) prd_face_bgr = np.clip( prd_face_bgr.astype(np.float32) / 255.0, 0.0, 1.0) if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] elif self.color_transfer_mode == 'lct': if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] prd_face_bgr = image_utils.linear_color_transfer( prd_face_bgr, dst_face_bgr) prd_face_bgr = np.clip(prd_face_bgr, 0.0, 1.0) if debug: debugs += [ np.clip( cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT), 0, 1.0) ] if self.mode == 'hist-match-bw': prd_face_bgr = cv2.cvtColor(prd_face_bgr, cv2.COLOR_BGR2GRAY) prd_face_bgr = np.repeat( np.expand_dims(prd_face_bgr, -1), (3, ), -1) if self.mode == 'hist-match' or self.mode == 'hist-match-bw': if debug: debugs += [ cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, np.zeros(img_bgr.shape, dtype=np.float32), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) ] hist_mask_a = np.ones(prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) if self.masked_hist_match: hist_mask_a *= prd_face_mask_a hist_match_1 = prd_face_bgr * hist_mask_a + ( 1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_1[hist_match_1 > 1.0] = 1.0 hist_match_2 = dst_face_bgr * hist_mask_a + ( 1.0 - hist_mask_a) * np.ones( prd_face_bgr.shape[:2] + (1, ), dtype=np.float32) hist_match_2[hist_match_1 > 1.0] = 1.0 prd_face_bgr = image_utils.color_hist_match( hist_match_1, hist_match_2, self.hist_match_threshold) if self.mode == 'hist-match-bw': prd_face_bgr = prd_face_bgr.astype(dtype=np.float32) out_img = cv2.warpAffine( prd_face_bgr, face_output_mat, img_size, out_img, cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) out_img = np.clip(out_img, 0.0, 1.0) if debug: debugs += [out_img.copy()] if self.mode == 'overlay': pass if self.mode == 'seamless' or self.mode == 'seamless-hist-match': out_img = np.clip( img_bgr * (1 - img_face_mask_aaa) + (out_img * img_face_mask_aaa), 0, 1.0) if debug: debugs += [out_img.copy()] out_img = cv2.seamlessClone( (out_img * 255).astype(np.uint8), (img_bgr * 255).astype(np.uint8), (img_face_mask_flatten_aaa * 255).astype(np.uint8), (masky, maskx), cv2.NORMAL_CLONE) out_img = out_img.astype(dtype=np.float32) / 255.0 if debug: debugs += [out_img.copy()] out_img = np.clip( img_bgr * (1 - img_mask_blurry_aaa) + (out_img * img_mask_blurry_aaa), 0, 1.0) if self.mode == 'seamless-hist-match': out_face_bgr = cv2.warpAffine( out_img, face_mat, (self.output_size, self.output_size)) new_out_face_bgr = image_utils.color_hist_match( out_face_bgr, dst_face_bgr, self.hist_match_threshold) new_out = cv2.warpAffine( new_out_face_bgr, face_mat, img_size, img_bgr.copy(), cv2.WARP_INVERSE_MAP | cv2.INTER_LANCZOS4, cv2.BORDER_TRANSPARENT) out_img = np.clip( img_bgr * (1 - img_mask_blurry_aaa) + (new_out * img_mask_blurry_aaa), 0, 1.0) if self.final_image_color_degrade_power != 0: if debug: debugs += [out_img.copy()] out_img_reduced = image_utils.reduce_colors( out_img, 256) if self.final_image_color_degrade_power == 100: out_img = out_img_reduced else: alpha = self.final_image_color_degrade_power / 100.0 out_img = (out_img * (1.0 - alpha) + out_img_reduced * alpha) if self.alpha: out_img = np.concatenate([ out_img, np.expand_dims(img_mask_blurry_aaa[:, :, 0], -1) ], -1) out_img = np.clip(out_img, 0.0, 1.0) if debug: debugs += [out_img.copy()] return debugs if debug else out_img