def get_data(self, host_dict): if self.type == 'landmarks-manual': need_remark_face = False while len(self.input_data) > 0: data = self.input_data[0] filepath, data_rects, data_landmarks = data.filepath, data.rects, data.landmarks is_frame_done = False if self.image_filepath != filepath: self.image_filepath = filepath if self.cache_original_image[0] == filepath: self.original_image = self.cache_original_image[1] else: self.original_image = imagelib.normalize_channels( cv2_imread(filepath), 3) self.cache_original_image = (filepath, self.original_image) (h, w, c) = self.original_image.shape self.view_scale = 1.0 if self.manual_window_size == 0 else self.manual_window_size / ( h * (16.0 / 9.0)) if self.cache_image[0] == (h, w, c) + (self.view_scale, filepath): self.image = self.cache_image[1] else: self.image = cv2.resize(self.original_image, (int( w * self.view_scale), int(h * self.view_scale)), interpolation=cv2.INTER_LINEAR) self.cache_image = ((h, w, c) + (self.view_scale, filepath), self.image) (h, w, c) = self.image.shape sh = (0, 0, w, min(100, h)) if self.cache_text_lines_img[0] == sh: self.text_lines_img = self.cache_text_lines_img[1] else: self.text_lines_img = (imagelib.get_draw_text_lines( self.image, sh, [ '[L Mouse click] - lock/unlock selection. [Mouse wheel] - change rect', '[R Mouse Click] - manual face rectangle', '[Enter] / [Space] - confirm / skip frame', '[,] [.]- prev frame, next frame. [Q] - skip remaining frames', '[a] - accuracy on/off (more fps)', '[h] - hide this help' ], (1, 1, 1)) * 255).astype(np.uint8) self.cache_text_lines_img = (sh, self.text_lines_img) if need_remark_face: # need remark image from input data that already has a marked face? need_remark_face = False if len( data_rects ) != 0: # If there was already a face then lock the rectangle to it until the mouse is clicked self.rect = data_rects.pop() self.landmarks = data_landmarks.pop() data_rects.clear() data_landmarks.clear() self.rect_locked = True self.rect_size = (self.rect[2] - self.rect[0]) / 2 self.x = (self.rect[0] + self.rect[2]) / 2 self.y = (self.rect[1] + self.rect[3]) / 2 self.redraw() if len(data_rects) == 0: (h, w, c) = self.image.shape while True: io.process_messages(0.0001) self.ea.loop() if not self.force_landmarks: new_x = self.x new_y = self.y new_rect_size = self.rect_size mouse_events = io.get_mouse_events(self.wnd_name) for ev in mouse_events: (x, y, ev, flags) = ev if ev == io.EVENT_MOUSEWHEEL and not self.rect_locked: mod = 1 if flags > 0 else -1 diff = 1 if new_rect_size <= 40 else np.clip( new_rect_size / 10, 1, 10) new_rect_size = max(5, new_rect_size + diff * mod) elif ev == io.EVENT_LBUTTONDOWN: if self.force_landmarks: self.x = new_x self.y = new_y self.force_landmarks = False self.rect_locked = True self.redraw() else: self.rect_locked = not self.rect_locked self.extract_needed = True elif ev == io.EVENT_RBUTTONDOWN: self.ea.right_btn_down = True # self.force_landmarks = not self.force_landmarks # if self.force_landmarks: # self.rect_locked = False elif not self.rect_locked: new_x = np.clip(x, 0, w - 1) / self.view_scale new_y = np.clip(y, 0, h - 1) / self.view_scale key_events = io.get_key_events(self.wnd_name) key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[ -1] if len(key_events) > 0 else (0, 0, False, False, False) if self.ea.right_btn_down and self.rect_locked: is_frame_done = True data_rects.append(self.rect) data_landmarks.append(self.landmarks) self.ea.last_outer = self.ea.cur_outer self.ea.last_landmarks = self.ea.cur_landmarks self.ea.auto = True break elif key == ord('s'): self.ea.auto = False break elif self.ea.auto and len( self.ea.last_outer) > 0 and len( self.ea.last_landmarks) > 0: # 根据上次的外框算出这次的x/y,以及外框大小 border_ratio = 0.6 last_mid = F.mid_point_by_range( self.ea.last_landmarks) last_border = np.linalg.norm( np.array(self.ea.last_outer[0]) - np.array(self.ea.last_outer[1])) last_area = F.poly_area(self.ea.last_outer) x, y = last_mid new_x = np.clip(x, 0, w - 1) / self.view_scale new_y = np.clip(y, 0, h - 1) / self.view_scale new_rect_size = last_border / 2 / self.view_scale * border_ratio # make sure rect and landmarks have been refreshed if len(self.ea.cur_outer) != 0: # 根据本次外框大小算是否valid,通过边长,面积,角度 # temp_mid = F.mid_point(self.temp_outer) cur_mid = F.mid_point_by_range( self.ea.cur_landmarks) dist = np.linalg.norm( np.array(cur_mid) - np.array(last_mid)) dist_r = dist / last_border temp_area = F.poly_area(self.ea.cur_outer) area_r = temp_area / last_area v0 = np.array(last_mid) - np.array( self.ea.last_outer[0]) v1 = np.array(cur_mid) - np.array( self.ea.cur_outer[0]) angle = math.fabs(F.angle_between(v0, v1)) if dist_r < 0.5 and 0.5 < area_r < 1.5 and angle < 0.7: is_frame_done = True self.ea.last_outer = self.ea.cur_outer self.ea.last_landmarks = self.ea.cur_landmarks data_rects.append(self.rect) data_landmarks.append(self.landmarks) self.ea.auto = True break elif self.x != new_x or self.y != new_y: # 可以在等一轮更新后试一下 pass else: self.ea.auto = False F.beep() elif key == ord('n') and len(self.result) > 0: # go prev frame without save and clear result self.rect_locked = False n = 10 if shift_pressed else 1 while n > 0 and len(self.result) > 0: self.input_data.insert(0, self.result.pop()) self.input_data[0].rects.clear() self.input_data[0].landmarks.clear() io.progress_bar_inc(-1) n -= 1 # 直接无视之前的结果,重新标注 self.extract_needed = True break elif key == ord('m') and len(self.input_data) > 0: # go next frame without save self.rect_locked = False n = 10 if shift_pressed else 1 while n > 0 and len(self.input_data) > 0: self.result.append(self.input_data.pop(0)) io.progress_bar_inc(1) n -= 1 # 直接无视之前的结果,重新标注 self.extract_needed = True break elif key == ord('\r') or key == ord('\n'): #confirm frame is_frame_done = True data_rects.append(self.rect) data_landmarks.append(self.landmarks) break elif key == ord(' '): #confirm skip frame is_frame_done = True break elif key == ord(',') and len(self.result) > 0: #go prev frame if self.rect_locked: self.rect_locked = False # Only save the face if the rect is still locked data_rects.append(self.rect) data_landmarks.append(self.landmarks) self.input_data.insert(0, self.result.pop()) io.progress_bar_inc(-1) need_remark_face = True break elif key == ord('.'): #go next frame if self.rect_locked: self.rect_locked = False # Only save the face if the rect is still locked data_rects.append(self.rect) data_landmarks.append(self.landmarks) need_remark_face = True is_frame_done = True break elif key == ord('q'): #skip remaining if self.rect_locked: self.rect_locked = False data_rects.append(self.rect) data_landmarks.append(self.landmarks) while len(self.input_data) > 0: self.result.append(self.input_data.pop(0)) io.progress_bar_inc(1) break elif key == ord('h'): self.hide_help = not self.hide_help break elif key == ord('a'): self.landmarks_accurate = not self.landmarks_accurate break if self.force_landmarks: pt2 = np.float32([new_x, new_y]) pt1 = np.float32([self.x, self.y]) pt_vec_len = npla.norm(pt2 - pt1) pt_vec = pt2 - pt1 if pt_vec_len != 0: pt_vec /= pt_vec_len self.rect_size = pt_vec_len self.rect = (int(self.x - self.rect_size), int(self.y - self.rect_size), int(self.x + self.rect_size), int(self.y + self.rect_size)) if pt_vec_len > 0: lmrks = np.concatenate( (np.zeros((17, 2), np.float32), LandmarksProcessor.landmarks_2D), axis=0) lmrks -= lmrks[30:31, :] mat = cv2.getRotationMatrix2D( (0, 0), -np.arctan2(pt_vec[1], pt_vec[0]) * 180 / math.pi, pt_vec_len) mat[:, 2] += (self.x, self.y) self.landmarks = LandmarksProcessor.transform_points( lmrks, mat) self.redraw() elif self.x != new_x or \ self.y != new_y or \ self.rect_size != new_rect_size or \ self.extract_needed: self.x = new_x self.y = new_y self.rect_size = new_rect_size self.rect = (int(self.x - self.rect_size), int(self.y - self.rect_size), int(self.x + self.rect_size), int(self.y + self.rect_size)) return ExtractSubprocessor.Data( filepath, rects=[self.rect], landmarks_accurate=self.landmarks_accurate) else: is_frame_done = True if is_frame_done: self.result.append(data) self.input_data.pop(0) io.progress_bar_inc(1) self.extract_needed = True self.rect_locked = False self.ea.cur_outer = [] else: if len(self.input_data) > 0: return self.input_data.pop(0) return None
def get_data(self, host_dict): if self.type == 'landmarks-manual': need_remark_face = False redraw_needed = False while len(self.input_data) > 0: data = self.input_data[0] filepath, data_rects, data_landmarks = data.filepath, data.rects, data.landmarks is_frame_done = False if need_remark_face: # need remark image from input data that already has a marked face? need_remark_face = False if len( data_rects ) != 0: # If there was already a face then lock the rectangle to it until the mouse is clicked self.rect = data_rects.pop() self.landmarks = data_landmarks.pop() data_rects.clear() data_landmarks.clear() redraw_needed = True self.rect_locked = True self.rect_size = (self.rect[2] - self.rect[0]) / 2 self.x = (self.rect[0] + self.rect[2]) / 2 self.y = (self.rect[1] + self.rect[3]) / 2 if len(data_rects) == 0: if self.cache_original_image[0] == filepath: self.original_image = self.cache_original_image[1] else: self.original_image = imagelib.normalize_channels( cv2_imread(filepath), 3) self.cache_original_image = (filepath, self.original_image) (h, w, c) = self.original_image.shape self.view_scale = 1.0 if self.manual_window_size == 0 else self.manual_window_size / ( h * (16.0 / 9.0)) if self.cache_image[0] == (h, w, c) + (self.view_scale, filepath): self.image = self.cache_image[1] else: self.image = cv2.resize(self.original_image, (int( w * self.view_scale), int(h * self.view_scale)), interpolation=cv2.INTER_LINEAR) self.cache_image = ((h, w, c) + (self.view_scale, filepath), self.image) (h, w, c) = self.image.shape sh = (0, 0, w, min(100, h)) if self.cache_text_lines_img[0] == sh: self.text_lines_img = self.cache_text_lines_img[1] else: self.text_lines_img = (imagelib.get_draw_text_lines( self.image, sh, [ '[Mouse click] - lock/unlock selection', '[Mouse wheel] - change rect', '[Enter] / [Space] - confirm / skip frame', '[,] [.]- prev frame, next frame. [Q] - skip remaining frames', '[a] - accuracy on/off (more fps)', '[h] - hide this help' ], (1, 1, 1)) * 255).astype(np.uint8) self.cache_text_lines_img = (sh, self.text_lines_img) while True: io.process_messages(0.0001) new_x = self.x new_y = self.y new_rect_size = self.rect_size mouse_events = io.get_mouse_events(self.wnd_name) for ev in mouse_events: (x, y, ev, flags) = ev if ev == io.EVENT_MOUSEWHEEL and not self.rect_locked: mod = 1 if flags > 0 else -1 diff = 1 if new_rect_size <= 40 else np.clip( new_rect_size / 10, 1, 10) new_rect_size = max(5, new_rect_size + diff * mod) elif ev == io.EVENT_LBUTTONDOWN: self.rect_locked = not self.rect_locked self.extract_needed = True elif not self.rect_locked: new_x = np.clip(x, 0, w - 1) / self.view_scale new_y = np.clip(y, 0, h - 1) / self.view_scale key_events = io.get_key_events(self.wnd_name) key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[ -1] if len(key_events) > 0 else (0, 0, False, False, False) if key == ord('\r') or key == ord('\n'): #confirm frame is_frame_done = True data_rects.append(self.rect) data_landmarks.append(self.landmarks) break elif key == ord(' '): #confirm skip frame is_frame_done = True break elif key == ord(',') and len(self.result) > 0: #go prev frame if self.rect_locked: self.rect_locked = False # Only save the face if the rect is still locked data_rects.append(self.rect) data_landmarks.append(self.landmarks) self.input_data.insert(0, self.result.pop()) io.progress_bar_inc(-1) need_remark_face = True break elif key == ord('.'): #go next frame if self.rect_locked: self.rect_locked = False # Only save the face if the rect is still locked data_rects.append(self.rect) data_landmarks.append(self.landmarks) need_remark_face = True is_frame_done = True break elif key == ord('q'): #skip remaining if self.rect_locked: self.rect_locked = False data_rects.append(self.rect) data_landmarks.append(self.landmarks) while len(self.input_data) > 0: self.result.append(self.input_data.pop(0)) io.progress_bar_inc(1) break elif key == ord('h'): self.hide_help = not self.hide_help break elif key == ord('a'): self.landmarks_accurate = not self.landmarks_accurate break if self.x != new_x or \ self.y != new_y or \ self.rect_size != new_rect_size or \ self.extract_needed or \ redraw_needed: self.x = new_x self.y = new_y self.rect_size = new_rect_size self.rect = (int(self.x - self.rect_size), int(self.y - self.rect_size), int(self.x + self.rect_size), int(self.y + self.rect_size)) if redraw_needed: redraw_needed = False return ExtractSubprocessor.Data( filepath, landmarks_accurate=self.landmarks_accurate) else: return ExtractSubprocessor.Data( filepath, rects=[self.rect], landmarks_accurate=self.landmarks_accurate) else: is_frame_done = True if is_frame_done: self.result.append(data) self.input_data.pop(0) io.progress_bar_inc(1) self.extract_needed = True self.rect_locked = False else: if len(self.input_data) > 0: return self.input_data.pop(0) return None
def mask_editor_main(input_dir, confirmed_dir=None, skipped_dir=None, no_default_mask=False): input_path = Path(input_dir) confirmed_path = Path(confirmed_dir) skipped_path = Path(skipped_dir) if not input_path.exists(): raise ValueError('Input directory not found. Please ensure it exists.') if not confirmed_path.exists(): confirmed_path.mkdir(parents=True) if not skipped_path.exists(): skipped_path.mkdir(parents=True) if not no_default_mask: eyebrows_expand_mod = np.clip( io.input_int("Default eyebrows expand modifier?", 100, add_info="0..400"), 0, 400) / 100.0 else: eyebrows_expand_mod = None wnd_name = "MaskEditor tool" io.named_window(wnd_name) io.capture_mouse(wnd_name) io.capture_keys(wnd_name) cached_images = {} image_paths = [Path(x) for x in pathex.get_image_paths(input_path)] done_paths = [] done_images_types = {} image_paths_total = len(image_paths) saved_ie_polys = IEPolys() zoom_factor = 1.0 preview_images_count = 9 target_wh = 256 do_prev_count = 0 do_save_move_count = 0 do_save_count = 0 do_skip_move_count = 0 do_skip_count = 0 def jobs_count(): return do_prev_count + do_save_move_count + do_save_count + do_skip_move_count + do_skip_count is_exit = False while not is_exit: if len(image_paths) > 0: filepath = image_paths.pop(0) else: filepath = None next_image_paths = image_paths[0:preview_images_count] next_image_paths_names = [path.name for path in next_image_paths] prev_image_paths = done_paths[-preview_images_count:] prev_image_paths_names = [path.name for path in prev_image_paths] for key in list(cached_images.keys()): if key not in prev_image_paths_names and \ key not in next_image_paths_names: cached_images.pop(key) for paths in [prev_image_paths, next_image_paths]: for path in paths: if path.name not in cached_images: cached_images[path.name] = cv2_imread(str(path)) / 255.0 if filepath is not None: dflimg = DFLIMG.load(filepath) if dflimg is None: io.log_err("%s is not a dfl image file" % (filepath.name)) continue else: lmrks = dflimg.get_landmarks() ie_polys = IEPolys.load(dflimg.get_ie_polys()) fanseg_mask = dflimg.get_fanseg_mask() if filepath.name in cached_images: img = cached_images[filepath.name] else: img = cached_images[filepath.name] = cv2_imread( str(filepath)) / 255.0 if fanseg_mask is not None: mask = fanseg_mask else: if no_default_mask: mask = np.zeros((target_wh, target_wh, 3)) else: mask = LandmarksProcessor.get_image_hull_mask( img.shape, lmrks, eyebrows_expand_mod=eyebrows_expand_mod) else: img = np.zeros((target_wh, target_wh, 3)) mask = np.ones((target_wh, target_wh, 3)) ie_polys = None def get_status_lines_func(): return [ 'Progress: %d / %d . Current file: %s' % (len(done_paths), image_paths_total, str(filepath.name) if filepath is not None else "end"), '[Left mouse button] - mark include mask.', '[Right mouse button] - mark exclude mask.', '[Middle mouse button] - finish current poly.', '[Mouse wheel] - undo/redo poly or point. [+ctrl] - undo to begin/redo to end', '[r] - applies edits made to last saved image.', '[q] - prev image. [w] - skip and move to %s. [e] - save and move to %s. ' % (skipped_path.name, confirmed_path.name), '[z] - prev image. [x] - skip. [c] - save. ', 'hold [shift] - speed up the frame counter by 10.', '[-/+] - window zoom [esc] - quit', ] try: ed = MaskEditor(img, [(done_images_types[name], cached_images[name]) for name in prev_image_paths_names], [(0, cached_images[name]) for name in next_image_paths_names], mask, ie_polys, get_status_lines_func) except Exception as e: print(e) continue next = False while not next: io.process_messages(0.005) if jobs_count() == 0: for (x, y, ev, flags) in io.get_mouse_events(wnd_name): x, y = int(x / zoom_factor), int(y / zoom_factor) ed.set_mouse_pos(x, y) if filepath is not None: if ev == io.EVENT_LBUTTONDOWN: ed.mask_point(1) elif ev == io.EVENT_RBUTTONDOWN: ed.mask_point(0) elif ev == io.EVENT_MBUTTONDOWN: ed.mask_finish() elif ev == io.EVENT_MOUSEWHEEL: if flags & 0x80000000 != 0: if flags & 0x8 != 0: ed.undo_to_begin_point() else: ed.undo_point() else: if flags & 0x8 != 0: ed.redo_to_end_point() else: ed.redo_point() for key, chr_key, ctrl_pressed, alt_pressed, shift_pressed in io.get_key_events( wnd_name): if chr_key == 'q' or chr_key == 'z': do_prev_count = 1 if not shift_pressed else 10 elif chr_key == '-': zoom_factor = np.clip(zoom_factor - 0.1, 0.1, 4.0) ed.set_screen_changed() elif chr_key == '+': zoom_factor = np.clip(zoom_factor + 0.1, 0.1, 4.0) ed.set_screen_changed() elif key == 27: #esc is_exit = True next = True break elif filepath is not None: if chr_key == 'e': saved_ie_polys = ed.ie_polys do_save_move_count = 1 if not shift_pressed else 10 elif chr_key == 'c': saved_ie_polys = ed.ie_polys do_save_count = 1 if not shift_pressed else 10 elif chr_key == 'w': do_skip_move_count = 1 if not shift_pressed else 10 elif chr_key == 'x': do_skip_count = 1 if not shift_pressed else 10 elif chr_key == 'r' and saved_ie_polys != None: ed.set_ie_polys(saved_ie_polys) if do_prev_count > 0: do_prev_count -= 1 if len(done_paths) > 0: if filepath is not None: image_paths.insert(0, filepath) filepath = done_paths.pop(-1) done_images_types[filepath.name] = 0 if filepath.parent != input_path: new_filename_path = input_path / filepath.name filepath.rename(new_filename_path) image_paths.insert(0, new_filename_path) else: image_paths.insert(0, filepath) next = True elif filepath is not None: if do_save_move_count > 0: do_save_move_count -= 1 ed.mask_finish() dflimg.embed_and_set( str(filepath), ie_polys=ed.get_ie_polys(), eyebrows_expand_mod=eyebrows_expand_mod) done_paths += [confirmed_path / filepath.name] done_images_types[filepath.name] = 2 filepath.rename(done_paths[-1]) next = True elif do_save_count > 0: do_save_count -= 1 ed.mask_finish() dflimg.embed_and_set( str(filepath), ie_polys=ed.get_ie_polys(), eyebrows_expand_mod=eyebrows_expand_mod) done_paths += [filepath] done_images_types[filepath.name] = 2 next = True elif do_skip_move_count > 0: do_skip_move_count -= 1 done_paths += [skipped_path / filepath.name] done_images_types[filepath.name] = 1 filepath.rename(done_paths[-1]) next = True elif do_skip_count > 0: do_skip_count -= 1 done_paths += [filepath] done_images_types[filepath.name] = 1 next = True else: do_save_move_count = do_save_count = do_skip_move_count = do_skip_count = 0 if jobs_count() == 0: if ed.switch_screen_changed(): screen = ed.make_screen() if zoom_factor != 1.0: h, w, c = screen.shape screen = cv2.resize( screen, (int(w * zoom_factor), int(h * zoom_factor))) io.show_image(wnd_name, screen) io.process_messages(0.005) io.destroy_all_windows()