def get_data(self, host_dict):
if self.type == 'landmarks-manual':
need_remark_face = False
while len(self.input_data) > 0:
data = self.input_data[0]
filepath, data_rects, data_landmarks = data.filepath, data.rects, data.landmarks
is_frame_done = False
if self.image_filepath != filepath:
self.image_filepath = filepath
if self.cache_original_image[0] == filepath:
self.original_image = self.cache_original_image[1]
else:
self.original_image = imagelib.normalize_channels(
cv2_imread(filepath), 3)
self.cache_original_image = (filepath,
self.original_image)
(h, w, c) = self.original_image.shape
self.view_scale = 1.0 if self.manual_window_size == 0 else self.manual_window_size / (
h * (16.0 / 9.0))
if self.cache_image[0] == (h, w, c) + (self.view_scale,
filepath):
self.image = self.cache_image[1]
else:
self.image = cv2.resize(self.original_image, (int(
w * self.view_scale), int(h * self.view_scale)),
interpolation=cv2.INTER_LINEAR)
self.cache_image = ((h, w, c) +
(self.view_scale, filepath),
self.image)
(h, w, c) = self.image.shape
sh = (0, 0, w, min(100, h))
if self.cache_text_lines_img[0] == sh:
self.text_lines_img = self.cache_text_lines_img[1]
else:
self.text_lines_img = (imagelib.get_draw_text_lines(
self.image, sh, [
'[L Mouse click] - lock/unlock selection. [Mouse wheel] - change rect',
'[R Mouse Click] - manual face rectangle',
'[Enter] / [Space] - confirm / skip frame',
'[,] [.]- prev frame, next frame. [Q] - skip remaining frames',
'[a] - accuracy on/off (more fps)',
'[h] - hide this help'
], (1, 1, 1)) * 255).astype(np.uint8)
self.cache_text_lines_img = (sh, self.text_lines_img)
if need_remark_face: # need remark image from input data that already has a marked face?
need_remark_face = False
if len(
data_rects
) != 0: # If there was already a face then lock the rectangle to it until the mouse is clicked
self.rect = data_rects.pop()
self.landmarks = data_landmarks.pop()
data_rects.clear()
data_landmarks.clear()
self.rect_locked = True
self.rect_size = (self.rect[2] - self.rect[0]) / 2
self.x = (self.rect[0] + self.rect[2]) / 2
self.y = (self.rect[1] + self.rect[3]) / 2
self.redraw()
if len(data_rects) == 0:
(h, w, c) = self.image.shape
while True:
io.process_messages(0.0001)
self.ea.loop()
if not self.force_landmarks:
new_x = self.x
new_y = self.y
new_rect_size = self.rect_size
mouse_events = io.get_mouse_events(self.wnd_name)
for ev in mouse_events:
(x, y, ev, flags) = ev
if ev == io.EVENT_MOUSEWHEEL and not self.rect_locked:
mod = 1 if flags > 0 else -1
diff = 1 if new_rect_size <= 40 else np.clip(
new_rect_size / 10, 1, 10)
new_rect_size = max(5,
new_rect_size + diff * mod)
elif ev == io.EVENT_LBUTTONDOWN:
if self.force_landmarks:
self.x = new_x
self.y = new_y
self.force_landmarks = False
self.rect_locked = True
self.redraw()
else:
self.rect_locked = not self.rect_locked
self.extract_needed = True
elif ev == io.EVENT_RBUTTONDOWN:
self.ea.right_btn_down = True
# self.force_landmarks = not self.force_landmarks
# if self.force_landmarks:
# self.rect_locked = False
elif not self.rect_locked:
new_x = np.clip(x, 0, w - 1) / self.view_scale
new_y = np.clip(y, 0, h - 1) / self.view_scale
key_events = io.get_key_events(self.wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False,
False, False)
if self.ea.right_btn_down and self.rect_locked:
is_frame_done = True
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.ea.last_outer = self.ea.cur_outer
self.ea.last_landmarks = self.ea.cur_landmarks
self.ea.auto = True
break
elif key == ord('s'):
self.ea.auto = False
break
elif self.ea.auto and len(
self.ea.last_outer) > 0 and len(
self.ea.last_landmarks) > 0:
# 根据上次的外框算出这次的x/y,以及外框大小
border_ratio = 0.6
last_mid = F.mid_point_by_range(
self.ea.last_landmarks)
last_border = np.linalg.norm(
np.array(self.ea.last_outer[0]) -
np.array(self.ea.last_outer[1]))
last_area = F.poly_area(self.ea.last_outer)
x, y = last_mid
new_x = np.clip(x, 0, w - 1) / self.view_scale
new_y = np.clip(y, 0, h - 1) / self.view_scale
new_rect_size = last_border / 2 / self.view_scale * border_ratio
# make sure rect and landmarks have been refreshed
if len(self.ea.cur_outer) != 0:
# 根据本次外框大小算是否valid，通过边长，面积，角度
# temp_mid = F.mid_point(self.temp_outer)
cur_mid = F.mid_point_by_range(
self.ea.cur_landmarks)
dist = np.linalg.norm(
np.array(cur_mid) - np.array(last_mid))
dist_r = dist / last_border
temp_area = F.poly_area(self.ea.cur_outer)
area_r = temp_area / last_area
v0 = np.array(last_mid) - np.array(
self.ea.last_outer[0])
v1 = np.array(cur_mid) - np.array(
self.ea.cur_outer[0])
angle = math.fabs(F.angle_between(v0, v1))
if dist_r < 0.5 and 0.5 < area_r < 1.5 and angle < 0.7:
is_frame_done = True
self.ea.last_outer = self.ea.cur_outer
self.ea.last_landmarks = self.ea.cur_landmarks
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.ea.auto = True
break
elif self.x != new_x or self.y != new_y:
# 可以在等一轮更新后试一下
pass
else:
self.ea.auto = False
F.beep()
elif key == ord('n') and len(self.result) > 0:
# go prev frame without save and clear result
self.rect_locked = False
n = 10 if shift_pressed else 1
while n > 0 and len(self.result) > 0:
self.input_data.insert(0, self.result.pop())
self.input_data[0].rects.clear()
self.input_data[0].landmarks.clear()
io.progress_bar_inc(-1)
n -= 1
# 直接无视之前的结果，重新标注
self.extract_needed = True
break
elif key == ord('m') and len(self.input_data) > 0:
# go next frame without save
self.rect_locked = False
n = 10 if shift_pressed else 1
while n > 0 and len(self.input_data) > 0:
self.result.append(self.input_data.pop(0))
io.progress_bar_inc(1)
n -= 1
# 直接无视之前的结果，重新标注
self.extract_needed = True
break
elif key == ord('\r') or key == ord('\n'):
#confirm frame
is_frame_done = True
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
break
elif key == ord(' '):
#confirm skip frame
is_frame_done = True
break
elif key == ord(',') and len(self.result) > 0:
#go prev frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.input_data.insert(0, self.result.pop())
io.progress_bar_inc(-1)
need_remark_face = True
break
elif key == ord('.'):
#go next frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
need_remark_face = True
is_frame_done = True
break
elif key == ord('q'):
#skip remaining
if self.rect_locked:
self.rect_locked = False
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
while len(self.input_data) > 0:
self.result.append(self.input_data.pop(0))
io.progress_bar_inc(1)
break
elif key == ord('h'):
self.hide_help = not self.hide_help
break
elif key == ord('a'):
self.landmarks_accurate = not self.landmarks_accurate
break
if self.force_landmarks:
pt2 = np.float32([new_x, new_y])
pt1 = np.float32([self.x, self.y])
pt_vec_len = npla.norm(pt2 - pt1)
pt_vec = pt2 - pt1
if pt_vec_len != 0:
pt_vec /= pt_vec_len
self.rect_size = pt_vec_len
self.rect = (int(self.x - self.rect_size),
int(self.y - self.rect_size),
int(self.x + self.rect_size),
int(self.y + self.rect_size))
if pt_vec_len > 0:
lmrks = np.concatenate(
(np.zeros((17, 2), np.float32),
LandmarksProcessor.landmarks_2D),
axis=0)
lmrks -= lmrks[30:31, :]
mat = cv2.getRotationMatrix2D(
(0, 0), -np.arctan2(pt_vec[1], pt_vec[0]) *
180 / math.pi, pt_vec_len)
mat[:, 2] += (self.x, self.y)
self.landmarks = LandmarksProcessor.transform_points(
lmrks, mat)
self.redraw()
elif self.x != new_x or \
self.y != new_y or \
self.rect_size != new_rect_size or \
self.extract_needed:
self.x = new_x
self.y = new_y
self.rect_size = new_rect_size
self.rect = (int(self.x - self.rect_size),
int(self.y - self.rect_size),
int(self.x + self.rect_size),
int(self.y + self.rect_size))
return ExtractSubprocessor.Data(
filepath,
rects=[self.rect],
landmarks_accurate=self.landmarks_accurate)
else:
is_frame_done = True
if is_frame_done:
self.result.append(data)
self.input_data.pop(0)
io.progress_bar_inc(1)
self.extract_needed = True
self.rect_locked = False
self.ea.cur_outer = []
else:
if len(self.input_data) > 0:
return self.input_data.pop(0)
return None
def get_data(self, host_dict):
if self.type == 'landmarks-manual':
need_remark_face = False
redraw_needed = False
while len(self.input_data) > 0:
data = self.input_data[0]
filepath, data_rects, data_landmarks = data.filepath, data.rects, data.landmarks
is_frame_done = False
if need_remark_face: # need remark image from input data that already has a marked face?
need_remark_face = False
if len(
data_rects
) != 0: # If there was already a face then lock the rectangle to it until the mouse is clicked
self.rect = data_rects.pop()
self.landmarks = data_landmarks.pop()
data_rects.clear()
data_landmarks.clear()
redraw_needed = True
self.rect_locked = True
self.rect_size = (self.rect[2] - self.rect[0]) / 2
self.x = (self.rect[0] + self.rect[2]) / 2
self.y = (self.rect[1] + self.rect[3]) / 2
if len(data_rects) == 0:
if self.cache_original_image[0] == filepath:
self.original_image = self.cache_original_image[1]
else:
self.original_image = imagelib.normalize_channels(
cv2_imread(filepath), 3)
self.cache_original_image = (filepath,
self.original_image)
(h, w, c) = self.original_image.shape
self.view_scale = 1.0 if self.manual_window_size == 0 else self.manual_window_size / (
h * (16.0 / 9.0))
if self.cache_image[0] == (h, w, c) + (self.view_scale,
filepath):
self.image = self.cache_image[1]
else:
self.image = cv2.resize(self.original_image, (int(
w * self.view_scale), int(h * self.view_scale)),
interpolation=cv2.INTER_LINEAR)
self.cache_image = ((h, w, c) +
(self.view_scale, filepath),
self.image)
(h, w, c) = self.image.shape
sh = (0, 0, w, min(100, h))
if self.cache_text_lines_img[0] == sh:
self.text_lines_img = self.cache_text_lines_img[1]
else:
self.text_lines_img = (imagelib.get_draw_text_lines(
self.image, sh, [
'[Mouse click] - lock/unlock selection',
'[Mouse wheel] - change rect',
'[Enter] / [Space] - confirm / skip frame',
'[,] [.]- prev frame, next frame. [Q] - skip remaining frames',
'[a] - accuracy on/off (more fps)',
'[h] - hide this help'
], (1, 1, 1)) * 255).astype(np.uint8)
self.cache_text_lines_img = (sh, self.text_lines_img)
while True:
io.process_messages(0.0001)
new_x = self.x
new_y = self.y
new_rect_size = self.rect_size
mouse_events = io.get_mouse_events(self.wnd_name)
for ev in mouse_events:
(x, y, ev, flags) = ev
if ev == io.EVENT_MOUSEWHEEL and not self.rect_locked:
mod = 1 if flags > 0 else -1
diff = 1 if new_rect_size <= 40 else np.clip(
new_rect_size / 10, 1, 10)
new_rect_size = max(5,
new_rect_size + diff * mod)
elif ev == io.EVENT_LBUTTONDOWN:
self.rect_locked = not self.rect_locked
self.extract_needed = True
elif not self.rect_locked:
new_x = np.clip(x, 0, w - 1) / self.view_scale
new_y = np.clip(y, 0, h - 1) / self.view_scale
key_events = io.get_key_events(self.wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False,
False, False)
if key == ord('\r') or key == ord('\n'):
#confirm frame
is_frame_done = True
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
break
elif key == ord(' '):
#confirm skip frame
is_frame_done = True
break
elif key == ord(',') and len(self.result) > 0:
#go prev frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.input_data.insert(0, self.result.pop())
io.progress_bar_inc(-1)
need_remark_face = True
break
elif key == ord('.'):
#go next frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
need_remark_face = True
is_frame_done = True
break
elif key == ord('q'):
#skip remaining
if self.rect_locked:
self.rect_locked = False
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
while len(self.input_data) > 0:
self.result.append(self.input_data.pop(0))
io.progress_bar_inc(1)
break
elif key == ord('h'):
self.hide_help = not self.hide_help
break
elif key == ord('a'):
self.landmarks_accurate = not self.landmarks_accurate
break
if self.x != new_x or \
self.y != new_y or \
self.rect_size != new_rect_size or \
self.extract_needed or \
redraw_needed:
self.x = new_x
self.y = new_y
self.rect_size = new_rect_size
self.rect = (int(self.x - self.rect_size),
int(self.y - self.rect_size),
int(self.x + self.rect_size),
int(self.y + self.rect_size))
if redraw_needed:
redraw_needed = False
return ExtractSubprocessor.Data(
filepath,
landmarks_accurate=self.landmarks_accurate)
else:
return ExtractSubprocessor.Data(
filepath,
rects=[self.rect],
landmarks_accurate=self.landmarks_accurate)
else:
is_frame_done = True
if is_frame_done:
self.result.append(data)
self.input_data.pop(0)
io.progress_bar_inc(1)
self.extract_needed = True
self.rect_locked = False
else:
if len(self.input_data) > 0:
return self.input_data.pop(0)
return None
def mask_editor_main(input_dir,
confirmed_dir=None,
skipped_dir=None,
no_default_mask=False):
input_path = Path(input_dir)
confirmed_path = Path(confirmed_dir)
skipped_path = Path(skipped_dir)
if not input_path.exists():
raise ValueError('Input directory not found. Please ensure it exists.')
if not confirmed_path.exists():
confirmed_path.mkdir(parents=True)
if not skipped_path.exists():
skipped_path.mkdir(parents=True)
if not no_default_mask:
eyebrows_expand_mod = np.clip(
io.input_int("Default eyebrows expand modifier?",
100,
add_info="0..400"), 0, 400) / 100.0
else:
eyebrows_expand_mod = None
wnd_name = "MaskEditor tool"
io.named_window(wnd_name)
io.capture_mouse(wnd_name)
io.capture_keys(wnd_name)
cached_images = {}
image_paths = [Path(x) for x in pathex.get_image_paths(input_path)]
done_paths = []
done_images_types = {}
image_paths_total = len(image_paths)
saved_ie_polys = IEPolys()
zoom_factor = 1.0
preview_images_count = 9
target_wh = 256
do_prev_count = 0
do_save_move_count = 0
do_save_count = 0
do_skip_move_count = 0
do_skip_count = 0
def jobs_count():
return do_prev_count + do_save_move_count + do_save_count + do_skip_move_count + do_skip_count
is_exit = False
while not is_exit:
if len(image_paths) > 0:
filepath = image_paths.pop(0)
else:
filepath = None
next_image_paths = image_paths[0:preview_images_count]
next_image_paths_names = [path.name for path in next_image_paths]
prev_image_paths = done_paths[-preview_images_count:]
prev_image_paths_names = [path.name for path in prev_image_paths]
for key in list(cached_images.keys()):
if key not in prev_image_paths_names and \
key not in next_image_paths_names:
cached_images.pop(key)
for paths in [prev_image_paths, next_image_paths]:
for path in paths:
if path.name not in cached_images:
cached_images[path.name] = cv2_imread(str(path)) / 255.0
if filepath is not None:
dflimg = DFLIMG.load(filepath)
if dflimg is None:
io.log_err("%s is not a dfl image file" % (filepath.name))
continue
else:
lmrks = dflimg.get_landmarks()
ie_polys = IEPolys.load(dflimg.get_ie_polys())
fanseg_mask = dflimg.get_fanseg_mask()
if filepath.name in cached_images:
img = cached_images[filepath.name]
else:
img = cached_images[filepath.name] = cv2_imread(
str(filepath)) / 255.0
if fanseg_mask is not None:
mask = fanseg_mask
else:
if no_default_mask:
mask = np.zeros((target_wh, target_wh, 3))
else:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape,
lmrks,
eyebrows_expand_mod=eyebrows_expand_mod)
else:
img = np.zeros((target_wh, target_wh, 3))
mask = np.ones((target_wh, target_wh, 3))
ie_polys = None
def get_status_lines_func():
return [
'Progress: %d / %d . Current file: %s' %
(len(done_paths), image_paths_total,
str(filepath.name) if filepath is not None else "end"),
'[Left mouse button] - mark include mask.',
'[Right mouse button] - mark exclude mask.',
'[Middle mouse button] - finish current poly.',
'[Mouse wheel] - undo/redo poly or point. [+ctrl] - undo to begin/redo to end',
'[r] - applies edits made to last saved image.',
'[q] - prev image. [w] - skip and move to %s. [e] - save and move to %s. '
% (skipped_path.name, confirmed_path.name),
'[z] - prev image. [x] - skip. [c] - save. ',
'hold [shift] - speed up the frame counter by 10.',
'[-/+] - window zoom [esc] - quit',
]
try:
ed = MaskEditor(img,
[(done_images_types[name], cached_images[name])
for name in prev_image_paths_names],
[(0, cached_images[name])
for name in next_image_paths_names], mask,
ie_polys, get_status_lines_func)
except Exception as e:
print(e)
continue
next = False
while not next:
io.process_messages(0.005)
if jobs_count() == 0:
for (x, y, ev, flags) in io.get_mouse_events(wnd_name):
x, y = int(x / zoom_factor), int(y / zoom_factor)
ed.set_mouse_pos(x, y)
if filepath is not None:
if ev == io.EVENT_LBUTTONDOWN:
ed.mask_point(1)
elif ev == io.EVENT_RBUTTONDOWN:
ed.mask_point(0)
elif ev == io.EVENT_MBUTTONDOWN:
ed.mask_finish()
elif ev == io.EVENT_MOUSEWHEEL:
if flags & 0x80000000 != 0:
if flags & 0x8 != 0:
ed.undo_to_begin_point()
else:
ed.undo_point()
else:
if flags & 0x8 != 0:
ed.redo_to_end_point()
else:
ed.redo_point()
for key, chr_key, ctrl_pressed, alt_pressed, shift_pressed in io.get_key_events(
wnd_name):
if chr_key == 'q' or chr_key == 'z':
do_prev_count = 1 if not shift_pressed else 10
elif chr_key == '-':
zoom_factor = np.clip(zoom_factor - 0.1, 0.1, 4.0)
ed.set_screen_changed()
elif chr_key == '+':
zoom_factor = np.clip(zoom_factor + 0.1, 0.1, 4.0)
ed.set_screen_changed()
elif key == 27: #esc
is_exit = True
next = True
break
elif filepath is not None:
if chr_key == 'e':
saved_ie_polys = ed.ie_polys
do_save_move_count = 1 if not shift_pressed else 10
elif chr_key == 'c':
saved_ie_polys = ed.ie_polys
do_save_count = 1 if not shift_pressed else 10
elif chr_key == 'w':
do_skip_move_count = 1 if not shift_pressed else 10
elif chr_key == 'x':
do_skip_count = 1 if not shift_pressed else 10
elif chr_key == 'r' and saved_ie_polys != None:
ed.set_ie_polys(saved_ie_polys)
if do_prev_count > 0:
do_prev_count -= 1
if len(done_paths) > 0:
if filepath is not None:
image_paths.insert(0, filepath)
filepath = done_paths.pop(-1)
done_images_types[filepath.name] = 0
if filepath.parent != input_path:
new_filename_path = input_path / filepath.name
filepath.rename(new_filename_path)
image_paths.insert(0, new_filename_path)
else:
image_paths.insert(0, filepath)
next = True
elif filepath is not None:
if do_save_move_count > 0:
do_save_move_count -= 1
ed.mask_finish()
dflimg.embed_and_set(
str(filepath),
ie_polys=ed.get_ie_polys(),
eyebrows_expand_mod=eyebrows_expand_mod)
done_paths += [confirmed_path / filepath.name]
done_images_types[filepath.name] = 2
filepath.rename(done_paths[-1])
next = True
elif do_save_count > 0:
do_save_count -= 1
ed.mask_finish()
dflimg.embed_and_set(
str(filepath),
ie_polys=ed.get_ie_polys(),
eyebrows_expand_mod=eyebrows_expand_mod)
done_paths += [filepath]
done_images_types[filepath.name] = 2
next = True
elif do_skip_move_count > 0:
do_skip_move_count -= 1
done_paths += [skipped_path / filepath.name]
done_images_types[filepath.name] = 1
filepath.rename(done_paths[-1])
next = True
elif do_skip_count > 0:
do_skip_count -= 1
done_paths += [filepath]
done_images_types[filepath.name] = 1
next = True
else:
do_save_move_count = do_save_count = do_skip_move_count = do_skip_count = 0
if jobs_count() == 0:
if ed.switch_screen_changed():
screen = ed.make_screen()
if zoom_factor != 1.0:
h, w, c = screen.shape
screen = cv2.resize(
screen,
(int(w * zoom_factor), int(h * zoom_factor)))
io.show_image(wnd_name, screen)
io.process_messages(0.005)
io.destroy_all_windows()
