def update_sample_for_preview(self,
choose_preview_history=False,
force_new=False):
if self.sample_for_preview is None or choose_preview_history or force_new:
if choose_preview_history and io.is_support_windows():
wnd_name = "[p] - next. [space] - switch preview type. [enter] - confirm."
io.log_info(
f"Choose image for the preview history. {wnd_name}")
io.named_window(wnd_name)
io.capture_keys(wnd_name)
choosed = False
preview_id_counter = 0
while not choosed:
self.sample_for_preview = self.generate_next_samples()
previews = self.get_static_previews()
io.show_image(
wnd_name,
(previews[preview_id_counter % len(previews)][1] *
255).astype(np.uint8))
while True:
key_events = io.get_key_events(wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False,
False, False)
if key == ord('\n') or key == ord('\r'):
choosed = True
break
elif key == ord(' '):
preview_id_counter += 1
break
elif key == ord('p'):
break
try:
io.process_messages(0.1)
except KeyboardInterrupt:
choosed = True
io.destroy_window(wnd_name)
else:
self.sample_for_preview = self.generate_next_samples()
try:
self.get_static_previews()
except:
self.sample_for_preview = self.generate_next_samples()
self.last_sample = self.sample_for_preview
def get_data(self, host_dict):
if self.type == 'landmarks-manual':
need_remark_face = False
redraw_needed = False
while len(self.input_data) > 0:
data = self.input_data[0]
filepath, data_rects, data_landmarks = data.filepath, data.rects, data.landmarks
is_frame_done = False
if need_remark_face: # need remark image from input data that already has a marked face?
need_remark_face = False
if len(
data_rects
) != 0: # If there was already a face then lock the rectangle to it until the mouse is clicked
self.rect = data_rects.pop()
self.landmarks = data_landmarks.pop()
data_rects.clear()
data_landmarks.clear()
redraw_needed = True
self.rect_locked = True
self.rect_size = (self.rect[2] - self.rect[0]) / 2
self.x = (self.rect[0] + self.rect[2]) / 2
self.y = (self.rect[1] + self.rect[3]) / 2
if len(data_rects) == 0:
if self.cache_original_image[0] == filepath:
self.original_image = self.cache_original_image[1]
else:
self.original_image = imagelib.normalize_channels(
cv2_imread(filepath), 3)
self.cache_original_image = (filepath,
self.original_image)
(h, w, c) = self.original_image.shape
self.view_scale = 1.0 if self.manual_window_size == 0 else self.manual_window_size / (
h * (16.0 / 9.0))
if self.cache_image[0] == (h, w, c) + (self.view_scale,
filepath):
self.image = self.cache_image[1]
else:
self.image = cv2.resize(self.original_image, (int(
w * self.view_scale), int(h * self.view_scale)),
interpolation=cv2.INTER_LINEAR)
self.cache_image = ((h, w, c) +
(self.view_scale, filepath),
self.image)
(h, w, c) = self.image.shape
sh = (0, 0, w, min(100, h))
if self.cache_text_lines_img[0] == sh:
self.text_lines_img = self.cache_text_lines_img[1]
else:
self.text_lines_img = (imagelib.get_draw_text_lines(
self.image, sh, [
'[Mouse click] - lock/unlock selection',
'[Mouse wheel] - change rect',
'[Enter] / [Space] - confirm / skip frame',
'[,] [.]- prev frame, next frame. [Q] - skip remaining frames',
'[a] - accuracy on/off (more fps)',
'[h] - hide this help'
], (1, 1, 1)) * 255).astype(np.uint8)
self.cache_text_lines_img = (sh, self.text_lines_img)
while True:
io.process_messages(0.0001)
new_x = self.x
new_y = self.y
new_rect_size = self.rect_size
mouse_events = io.get_mouse_events(self.wnd_name)
for ev in mouse_events:
(x, y, ev, flags) = ev
if ev == io.EVENT_MOUSEWHEEL and not self.rect_locked:
mod = 1 if flags > 0 else -1
diff = 1 if new_rect_size <= 40 else np.clip(
new_rect_size / 10, 1, 10)
new_rect_size = max(5,
new_rect_size + diff * mod)
elif ev == io.EVENT_LBUTTONDOWN:
self.rect_locked = not self.rect_locked
self.extract_needed = True
elif not self.rect_locked:
new_x = np.clip(x, 0, w - 1) / self.view_scale
new_y = np.clip(y, 0, h - 1) / self.view_scale
key_events = io.get_key_events(self.wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False,
False, False)
if key == ord('\r') or key == ord('\n'):
#confirm frame
is_frame_done = True
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
break
elif key == ord(' '):
#confirm skip frame
is_frame_done = True
break
elif key == ord(',') and len(self.result) > 0:
#go prev frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.input_data.insert(0, self.result.pop())
io.progress_bar_inc(-1)
need_remark_face = True
break
elif key == ord('.'):
#go next frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
need_remark_face = True
is_frame_done = True
break
elif key == ord('q'):
#skip remaining
if self.rect_locked:
self.rect_locked = False
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
while len(self.input_data) > 0:
self.result.append(self.input_data.pop(0))
io.progress_bar_inc(1)
break
elif key == ord('h'):
self.hide_help = not self.hide_help
break
elif key == ord('a'):
self.landmarks_accurate = not self.landmarks_accurate
break
if self.x != new_x or \
self.y != new_y or \
self.rect_size != new_rect_size or \
self.extract_needed or \
redraw_needed:
self.x = new_x
self.y = new_y
self.rect_size = new_rect_size
self.rect = (int(self.x - self.rect_size),
int(self.y - self.rect_size),
int(self.x + self.rect_size),
int(self.y + self.rect_size))
if redraw_needed:
redraw_needed = False
return ExtractSubprocessor.Data(
filepath,
landmarks_accurate=self.landmarks_accurate)
else:
return ExtractSubprocessor.Data(
filepath,
rects=[self.rect],
landmarks_accurate=self.landmarks_accurate)
else:
is_frame_done = True
if is_frame_done:
self.result.append(data)
self.input_data.pop(0)
io.progress_bar_inc(1)
self.extract_needed = True
self.rect_locked = False
else:
if len(self.input_data) > 0:
return self.input_data.pop(0)
return None
def get_key_events(self):
return io.get_key_events(self.wnd_name)
def main(**kwargs):
io.log_info("Running trainer.\r\n")
no_preview = kwargs.get('no_preview', False)
s2c = queue.Queue()
c2s = queue.Queue()
e = threading.Event()
thread = threading.Thread(target=trainerThread,
args=(s2c, c2s, e),
kwargs=kwargs)
thread.start()
e.wait() #Wait for inital load to occur.
if no_preview:
while True:
if not c2s.empty():
input = c2s.get()
op = input.get('op', '')
if op == 'close':
break
try:
io.process_messages(0.1)
except KeyboardInterrupt:
s2c.put({'op': 'close'})
else:
wnd_name = "Training preview"
io.named_window(wnd_name)
io.capture_keys(wnd_name)
previews = None
loss_history = None
selected_preview = 0
update_preview = False
is_showing = False
is_waiting_preview = False
show_last_history_iters_count = 0
iter = 0
while True:
if not c2s.empty():
input = c2s.get()
op = input['op']
if op == 'show':
is_waiting_preview = False
loss_history = input[
'loss_history'] if 'loss_history' in input.keys(
) else None
previews = input['previews'] if 'previews' in input.keys(
) else None
iter = input['iter'] if 'iter' in input.keys() else 0
if previews is not None:
max_w = 0
max_h = 0
for (preview_name, preview_rgb) in previews:
(h, w, c) = preview_rgb.shape
max_h = max(max_h, h)
max_w = max(max_w, w)
max_size = 800
if max_h > max_size:
max_w = int(max_w / (max_h / max_size))
max_h = max_size
#make all previews size equal
for preview in previews[:]:
(preview_name, preview_rgb) = preview
(h, w, c) = preview_rgb.shape
if h != max_h or w != max_w:
previews.remove(preview)
previews.append(
(preview_name,
cv2.resize(preview_rgb, (max_w, max_h))))
selected_preview = selected_preview % len(previews)
update_preview = True
elif op == 'close':
break
if update_preview:
update_preview = False
selected_preview_name = previews[selected_preview][0]
selected_preview_rgb = previews[selected_preview][1]
(h, w, c) = selected_preview_rgb.shape
# HEAD
head_lines = [
'[s]:save [enter]:exit',
'[p]:update [space]:next preview [l]:change history range',
'Preview: "%s" [%d/%d]' %
(selected_preview_name, selected_preview + 1,
len(previews))
]
head_line_height = 15
head_height = len(head_lines) * head_line_height
head = np.ones((head_height, w, c)) * 0.1
for i in range(0, len(head_lines)):
t = i * head_line_height
b = (i + 1) * head_line_height
head[t:b, 0:w] += imagelib.get_text_image(
(head_line_height, w, c),
head_lines[i],
color=[0.8] * c)
final = head
if loss_history is not None:
if show_last_history_iters_count == 0:
loss_history_to_show = loss_history
else:
loss_history_to_show = loss_history[
-show_last_history_iters_count:]
lh_img = models.ModelBase.get_loss_history_preview(
loss_history_to_show, iter, w, c)
final = np.concatenate([final, lh_img], axis=0)
final = np.concatenate([final, selected_preview_rgb], axis=0)
final = np.clip(final, 0, 1)
io.show_image(wnd_name, (final * 255).astype(np.uint8))
is_showing = True
key_events = io.get_key_events(wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False, False, False)
if key == ord('\n') or key == ord('\r'):
s2c.put({'op': 'close'})
elif key == ord('s'):
s2c.put({'op': 'save'})
elif key == ord('p'):
if not is_waiting_preview:
is_waiting_preview = True
s2c.put({'op': 'preview'})
elif key == ord('l'):
if show_last_history_iters_count == 0:
show_last_history_iters_count = 5000
elif show_last_history_iters_count == 5000:
show_last_history_iters_count = 10000
elif show_last_history_iters_count == 10000:
show_last_history_iters_count = 50000
elif show_last_history_iters_count == 50000:
show_last_history_iters_count = 100000
elif show_last_history_iters_count == 100000:
show_last_history_iters_count = 0
update_preview = True
elif key == ord(' '):
selected_preview = (selected_preview + 1) % len(previews)
update_preview = True
try:
io.process_messages(0.1)
except KeyboardInterrupt:
s2c.put({'op': 'close'})
io.destroy_all_windows()
def get_data(self, host_dict):
if self.type == 'landmarks-manual':
need_remark_face = False
while len(self.input_data) > 0:
data = self.input_data[0]
filepath, data_rects, data_landmarks = data.filepath, data.rects, data.landmarks
is_frame_done = False
if self.image_filepath != filepath:
self.image_filepath = filepath
if self.cache_original_image[0] == filepath:
self.original_image = self.cache_original_image[1]
else:
self.original_image = imagelib.normalize_channels(
cv2_imread(filepath), 3)
self.cache_original_image = (filepath,
self.original_image)
(h, w, c) = self.original_image.shape
self.view_scale = 1.0 if self.manual_window_size == 0 else self.manual_window_size / (
h * (16.0 / 9.0))
if self.cache_image[0] == (h, w, c) + (self.view_scale,
filepath):
self.image = self.cache_image[1]
else:
self.image = cv2.resize(self.original_image, (int(
w * self.view_scale), int(h * self.view_scale)),
interpolation=cv2.INTER_LINEAR)
self.cache_image = ((h, w, c) +
(self.view_scale, filepath),
self.image)
(h, w, c) = self.image.shape
sh = (0, 0, w, min(100, h))
if self.cache_text_lines_img[0] == sh:
self.text_lines_img = self.cache_text_lines_img[1]
else:
self.text_lines_img = (imagelib.get_draw_text_lines(
self.image, sh, [
'[L Mouse click] - lock/unlock selection. [Mouse wheel] - change rect',
'[R Mouse Click] - manual face rectangle',
'[Enter] / [Space] - confirm / skip frame',
'[,] [.]- prev frame, next frame. [Q] - skip remaining frames',
'[a] - accuracy on/off (more fps)',
'[h] - hide this help'
], (1, 1, 1)) * 255).astype(np.uint8)
self.cache_text_lines_img = (sh, self.text_lines_img)
if need_remark_face: # need remark image from input data that already has a marked face?
need_remark_face = False
if len(
data_rects
) != 0: # If there was already a face then lock the rectangle to it until the mouse is clicked
self.rect = data_rects.pop()
self.landmarks = data_landmarks.pop()
data_rects.clear()
data_landmarks.clear()
self.rect_locked = True
self.rect_size = (self.rect[2] - self.rect[0]) / 2
self.x = (self.rect[0] + self.rect[2]) / 2
self.y = (self.rect[1] + self.rect[3]) / 2
self.redraw()
if len(data_rects) == 0:
(h, w, c) = self.image.shape
while True:
io.process_messages(0.0001)
self.ea.loop()
if not self.force_landmarks:
new_x = self.x
new_y = self.y
new_rect_size = self.rect_size
mouse_events = io.get_mouse_events(self.wnd_name)
for ev in mouse_events:
(x, y, ev, flags) = ev
if ev == io.EVENT_MOUSEWHEEL and not self.rect_locked:
mod = 1 if flags > 0 else -1
diff = 1 if new_rect_size <= 40 else np.clip(
new_rect_size / 10, 1, 10)
new_rect_size = max(5,
new_rect_size + diff * mod)
elif ev == io.EVENT_LBUTTONDOWN:
if self.force_landmarks:
self.x = new_x
self.y = new_y
self.force_landmarks = False
self.rect_locked = True
self.redraw()
else:
self.rect_locked = not self.rect_locked
self.extract_needed = True
elif ev == io.EVENT_RBUTTONDOWN:
self.ea.right_btn_down = True
# self.force_landmarks = not self.force_landmarks
# if self.force_landmarks:
# self.rect_locked = False
elif not self.rect_locked:
new_x = np.clip(x, 0, w - 1) / self.view_scale
new_y = np.clip(y, 0, h - 1) / self.view_scale
key_events = io.get_key_events(self.wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False,
False, False)
if self.ea.right_btn_down and self.rect_locked:
is_frame_done = True
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.ea.last_outer = self.ea.cur_outer
self.ea.last_landmarks = self.ea.cur_landmarks
self.ea.auto = True
break
elif key == ord('s'):
self.ea.auto = False
break
elif self.ea.auto and len(
self.ea.last_outer) > 0 and len(
self.ea.last_landmarks) > 0:
# 根据上次的外框算出这次的x/y,以及外框大小
border_ratio = 0.6
last_mid = F.mid_point_by_range(
self.ea.last_landmarks)
last_border = np.linalg.norm(
np.array(self.ea.last_outer[0]) -
np.array(self.ea.last_outer[1]))
last_area = F.poly_area(self.ea.last_outer)
x, y = last_mid
new_x = np.clip(x, 0, w - 1) / self.view_scale
new_y = np.clip(y, 0, h - 1) / self.view_scale
new_rect_size = last_border / 2 / self.view_scale * border_ratio
# make sure rect and landmarks have been refreshed
if len(self.ea.cur_outer) != 0:
# 根据本次外框大小算是否valid，通过边长，面积，角度
# temp_mid = F.mid_point(self.temp_outer)
cur_mid = F.mid_point_by_range(
self.ea.cur_landmarks)
dist = np.linalg.norm(
np.array(cur_mid) - np.array(last_mid))
dist_r = dist / last_border
temp_area = F.poly_area(self.ea.cur_outer)
area_r = temp_area / last_area
v0 = np.array(last_mid) - np.array(
self.ea.last_outer[0])
v1 = np.array(cur_mid) - np.array(
self.ea.cur_outer[0])
angle = math.fabs(F.angle_between(v0, v1))
if dist_r < 0.5 and 0.5 < area_r < 1.5 and angle < 0.7:
is_frame_done = True
self.ea.last_outer = self.ea.cur_outer
self.ea.last_landmarks = self.ea.cur_landmarks
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.ea.auto = True
break
elif self.x != new_x or self.y != new_y:
# 可以在等一轮更新后试一下
pass
else:
self.ea.auto = False
F.beep()
elif key == ord('n') and len(self.result) > 0:
# go prev frame without save and clear result
self.rect_locked = False
n = 10 if shift_pressed else 1
while n > 0 and len(self.result) > 0:
self.input_data.insert(0, self.result.pop())
self.input_data[0].rects.clear()
self.input_data[0].landmarks.clear()
io.progress_bar_inc(-1)
n -= 1
# 直接无视之前的结果，重新标注
self.extract_needed = True
break
elif key == ord('m') and len(self.input_data) > 0:
# go next frame without save
self.rect_locked = False
n = 10 if shift_pressed else 1
while n > 0 and len(self.input_data) > 0:
self.result.append(self.input_data.pop(0))
io.progress_bar_inc(1)
n -= 1
# 直接无视之前的结果，重新标注
self.extract_needed = True
break
elif key == ord('\r') or key == ord('\n'):
#confirm frame
is_frame_done = True
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
break
elif key == ord(' '):
#confirm skip frame
is_frame_done = True
break
elif key == ord(',') and len(self.result) > 0:
#go prev frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
self.input_data.insert(0, self.result.pop())
io.progress_bar_inc(-1)
need_remark_face = True
break
elif key == ord('.'):
#go next frame
if self.rect_locked:
self.rect_locked = False
# Only save the face if the rect is still locked
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
need_remark_face = True
is_frame_done = True
break
elif key == ord('q'):
#skip remaining
if self.rect_locked:
self.rect_locked = False
data_rects.append(self.rect)
data_landmarks.append(self.landmarks)
while len(self.input_data) > 0:
self.result.append(self.input_data.pop(0))
io.progress_bar_inc(1)
break
elif key == ord('h'):
self.hide_help = not self.hide_help
break
elif key == ord('a'):
self.landmarks_accurate = not self.landmarks_accurate
break
if self.force_landmarks:
pt2 = np.float32([new_x, new_y])
pt1 = np.float32([self.x, self.y])
pt_vec_len = npla.norm(pt2 - pt1)
pt_vec = pt2 - pt1
if pt_vec_len != 0:
pt_vec /= pt_vec_len
self.rect_size = pt_vec_len
self.rect = (int(self.x - self.rect_size),
int(self.y - self.rect_size),
int(self.x + self.rect_size),
int(self.y + self.rect_size))
if pt_vec_len > 0:
lmrks = np.concatenate(
(np.zeros((17, 2), np.float32),
LandmarksProcessor.landmarks_2D),
axis=0)
lmrks -= lmrks[30:31, :]
mat = cv2.getRotationMatrix2D(
(0, 0), -np.arctan2(pt_vec[1], pt_vec[0]) *
180 / math.pi, pt_vec_len)
mat[:, 2] += (self.x, self.y)
self.landmarks = LandmarksProcessor.transform_points(
lmrks, mat)
self.redraw()
elif self.x != new_x or \
self.y != new_y or \
self.rect_size != new_rect_size or \
self.extract_needed:
self.x = new_x
self.y = new_y
self.rect_size = new_rect_size
self.rect = (int(self.x - self.rect_size),
int(self.y - self.rect_size),
int(self.x + self.rect_size),
int(self.y + self.rect_size))
return ExtractSubprocessor.Data(
filepath,
rects=[self.rect],
landmarks_accurate=self.landmarks_accurate)
else:
is_frame_done = True
if is_frame_done:
self.result.append(data)
self.input_data.pop(0)
io.progress_bar_inc(1)
self.extract_needed = True
self.rect_locked = False
self.ea.cur_outer = []
else:
if len(self.input_data) > 0:
return self.input_data.pop(0)
return None
def __init__(self,
is_training=False,
saved_models_path=None,
training_data_src_path=None,
training_data_dst_path=None,
pretraining_data_path=None,
pretrained_model_path=None,
no_preview=False,
force_model_name=None,
force_gpu_idxs=None,
cpu_only=False,
debug=False,
**kwargs):
self.is_training = is_training
self.saved_models_path = saved_models_path
self.training_data_src_path = training_data_src_path
self.training_data_dst_path = training_data_dst_path
self.pretraining_data_path = pretraining_data_path
self.pretrained_model_path = pretrained_model_path
self.no_preview = no_preview
self.debug = debug
self.model_class_name = model_class_name = Path(
inspect.getmodule(self).__file__).parent.name.rsplit("_", 1)[1]
if force_model_name is not None:
self.model_name = force_model_name
else:
while True:
# gather all model dat files
saved_models_names = []
for filepath in pathex.get_file_paths(saved_models_path):
filepath_name = filepath.name
if filepath_name.endswith(f'{model_class_name}_data.dat'):
saved_models_names += [(filepath_name.split('_')[0],
os.path.getmtime(filepath))]
# sort by modified datetime
saved_models_names = sorted(saved_models_names,
key=operator.itemgetter(1),
reverse=True)
saved_models_names = [x[0] for x in saved_models_names]
if len(saved_models_names) != 0:
io.log_info(
"Choose one of saved models, or enter a name to create a new model."
)
io.log_info("[r] : rename")
io.log_info("[d] : delete")
io.log_info("")
for i, model_name in enumerate(saved_models_names):
s = f"[{i}] : {model_name} "
if i == 0:
s += "- latest"
io.log_info(s)
inp = io.input_str(f"", "0", show_default_value=False)
model_idx = -1
try:
model_idx = np.clip(int(inp), 0,
len(saved_models_names) - 1)
except:
pass
if model_idx == -1:
if len(inp) == 1:
is_rename = inp[0] == 'r'
is_delete = inp[0] == 'd'
if is_rename or is_delete:
if len(saved_models_names) != 0:
if is_rename:
name = io.input_str(
f"Enter the name of the model you want to rename"
)
elif is_delete:
name = io.input_str(
f"Enter the name of the model you want to delete"
)
if name in saved_models_names:
if is_rename:
new_model_name = io.input_str(
f"Enter new name of the model")
for filepath in pathex.get_paths(
saved_models_path):
filepath_name = filepath.name
model_filename, remain_filename = filepath_name.split(
'_', 1)
if model_filename == name:
if is_rename:
new_filepath = filepath.parent / (
new_model_name + '_' +
remain_filename)
filepath.rename(
new_filepath)
elif is_delete:
filepath.unlink()
continue
self.model_name = inp
else:
self.model_name = saved_models_names[model_idx]
else:
self.model_name = io.input_str(
f"No saved models found. Enter a name of a new model",
"noname")
break
self.model_name = self.model_name + '_' + self.model_class_name
self.iter = 0
self.options = {}
self.loss_history = []
self.sample_for_preview = None
self.choosed_gpu_indexes = None
model_data = {}
self.model_data_path = Path(
self.get_strpath_storage_for_file('data.dat'))
if self.model_data_path.exists():
io.log_info(f"Loading {self.model_name} model...")
model_data = pickle.loads(self.model_data_path.read_bytes())
self.iter = model_data.get('iter', 0)
if self.iter != 0:
self.options = model_data['options']
self.loss_history = model_data.get('loss_history', [])
self.sample_for_preview = model_data.get(
'sample_for_preview', None)
self.choosed_gpu_indexes = model_data.get(
'choosed_gpu_indexes', None)
if self.is_first_run():
io.log_info("\nModel first run.")
self.device_config = nn.DeviceConfig.GPUIndexes( force_gpu_idxs or nn.ask_choose_device_idxs(suggest_best_multi_gpu=True)) \
if not cpu_only else nn.DeviceConfig.CPU()
nn.initialize(self.device_config)
####
self.default_options_path = saved_models_path / f'{self.model_class_name}_default_options.dat'
self.default_options = {}
if self.default_options_path.exists():
try:
self.default_options = pickle.loads(
self.default_options_path.read_bytes())
except:
pass
self.choose_preview_history = False
self.batch_size = self.load_or_def_option('batch_size', 1)
#####
io.input_skip_pending()
self.on_initialize_options()
if self.is_first_run():
# save as default options only for first run model initialize
self.default_options_path.write_bytes(pickle.dumps(self.options))
self.autobackup = self.options.get('autobackup', False)
self.write_preview_history = self.options.get('write_preview_history',
False)
self.target_iter = self.options.get('target_iter', 0)
self.random_flip = self.options.get('random_flip', True)
self.on_initialize()
self.options['batch_size'] = self.batch_size
if self.is_training:
self.preview_history_path = self.saved_models_path / (
f'{self.get_model_name()}_history')
self.autobackups_path = self.saved_models_path / (
f'{self.get_model_name()}_autobackups')
if self.autobackup:
self.autobackup_current_hour = time.localtime().tm_hour
if not self.autobackups_path.exists():
self.autobackups_path.mkdir(exist_ok=True)
if self.write_preview_history or io.is_colab():
if not self.preview_history_path.exists():
self.preview_history_path.mkdir(exist_ok=True)
else:
if self.iter == 0:
for filename in pathex.get_image_paths(
self.preview_history_path):
Path(filename).unlink()
if self.generator_list is None:
raise ValueError('You didnt set_training_data_generators()')
else:
for i, generator in enumerate(self.generator_list):
if not isinstance(generator, SampleGeneratorBase):
raise ValueError(
'training data generator is not subclass of SampleGeneratorBase'
)
if self.sample_for_preview is None or self.choose_preview_history:
if self.choose_preview_history and io.is_support_windows():
io.log_info(
"Choose image for the preview history. [p] - next. [enter] - confirm."
)
wnd_name = "[p] - next. [enter] - confirm."
io.named_window(wnd_name)
io.capture_keys(wnd_name)
choosed = False
while not choosed:
self.sample_for_preview = self.generate_next_samples()
preview = self.get_static_preview()
io.show_image(wnd_name,
(preview * 255).astype(np.uint8))
while True:
key_events = io.get_key_events(wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[
-1] if len(key_events) > 0 else (0, 0, False,
False, False)
if key == ord('\n') or key == ord('\r'):
choosed = True
break
elif key == ord('p'):
break
try:
io.process_messages(0.1)
except KeyboardInterrupt:
choosed = True
io.destroy_window(wnd_name)
else:
self.sample_for_preview = self.generate_next_samples()
try:
self.get_static_preview()
except:
self.sample_for_preview = self.generate_next_samples()
self.last_sample = self.sample_for_preview
io.log_info(self.get_summary_text())
def mask_editor_main(input_dir,
confirmed_dir=None,
skipped_dir=None,
no_default_mask=False):
input_path = Path(input_dir)
confirmed_path = Path(confirmed_dir)
skipped_path = Path(skipped_dir)
if not input_path.exists():
raise ValueError('Input directory not found. Please ensure it exists.')
if not confirmed_path.exists():
confirmed_path.mkdir(parents=True)
if not skipped_path.exists():
skipped_path.mkdir(parents=True)
if not no_default_mask:
eyebrows_expand_mod = np.clip(
io.input_int("Default eyebrows expand modifier?",
100,
add_info="0..400"), 0, 400) / 100.0
else:
eyebrows_expand_mod = None
wnd_name = "MaskEditor tool"
io.named_window(wnd_name)
io.capture_mouse(wnd_name)
io.capture_keys(wnd_name)
cached_images = {}
image_paths = [Path(x) for x in pathex.get_image_paths(input_path)]
done_paths = []
done_images_types = {}
image_paths_total = len(image_paths)
saved_ie_polys = IEPolys()
zoom_factor = 1.0
preview_images_count = 9
target_wh = 256
do_prev_count = 0
do_save_move_count = 0
do_save_count = 0
do_skip_move_count = 0
do_skip_count = 0
def jobs_count():
return do_prev_count + do_save_move_count + do_save_count + do_skip_move_count + do_skip_count
is_exit = False
while not is_exit:
if len(image_paths) > 0:
filepath = image_paths.pop(0)
else:
filepath = None
next_image_paths = image_paths[0:preview_images_count]
next_image_paths_names = [path.name for path in next_image_paths]
prev_image_paths = done_paths[-preview_images_count:]
prev_image_paths_names = [path.name for path in prev_image_paths]
for key in list(cached_images.keys()):
if key not in prev_image_paths_names and \
key not in next_image_paths_names:
cached_images.pop(key)
for paths in [prev_image_paths, next_image_paths]:
for path in paths:
if path.name not in cached_images:
cached_images[path.name] = cv2_imread(str(path)) / 255.0
if filepath is not None:
dflimg = DFLIMG.load(filepath)
if dflimg is None:
io.log_err("%s is not a dfl image file" % (filepath.name))
continue
else:
lmrks = dflimg.get_landmarks()
ie_polys = IEPolys.load(dflimg.get_ie_polys())
fanseg_mask = dflimg.get_fanseg_mask()
if filepath.name in cached_images:
img = cached_images[filepath.name]
else:
img = cached_images[filepath.name] = cv2_imread(
str(filepath)) / 255.0
if fanseg_mask is not None:
mask = fanseg_mask
else:
if no_default_mask:
mask = np.zeros((target_wh, target_wh, 3))
else:
mask = LandmarksProcessor.get_image_hull_mask(
img.shape,
lmrks,
eyebrows_expand_mod=eyebrows_expand_mod)
else:
img = np.zeros((target_wh, target_wh, 3))
mask = np.ones((target_wh, target_wh, 3))
ie_polys = None
def get_status_lines_func():
return [
'Progress: %d / %d . Current file: %s' %
(len(done_paths), image_paths_total,
str(filepath.name) if filepath is not None else "end"),
'[Left mouse button] - mark include mask.',
'[Right mouse button] - mark exclude mask.',
'[Middle mouse button] - finish current poly.',
'[Mouse wheel] - undo/redo poly or point. [+ctrl] - undo to begin/redo to end',
'[r] - applies edits made to last saved image.',
'[q] - prev image. [w] - skip and move to %s. [e] - save and move to %s. '
% (skipped_path.name, confirmed_path.name),
'[z] - prev image. [x] - skip. [c] - save. ',
'hold [shift] - speed up the frame counter by 10.',
'[-/+] - window zoom [esc] - quit',
]
try:
ed = MaskEditor(img,
[(done_images_types[name], cached_images[name])
for name in prev_image_paths_names],
[(0, cached_images[name])
for name in next_image_paths_names], mask,
ie_polys, get_status_lines_func)
except Exception as e:
print(e)
continue
next = False
while not next:
io.process_messages(0.005)
if jobs_count() == 0:
for (x, y, ev, flags) in io.get_mouse_events(wnd_name):
x, y = int(x / zoom_factor), int(y / zoom_factor)
ed.set_mouse_pos(x, y)
if filepath is not None:
if ev == io.EVENT_LBUTTONDOWN:
ed.mask_point(1)
elif ev == io.EVENT_RBUTTONDOWN:
ed.mask_point(0)
elif ev == io.EVENT_MBUTTONDOWN:
ed.mask_finish()
elif ev == io.EVENT_MOUSEWHEEL:
if flags & 0x80000000 != 0:
if flags & 0x8 != 0:
ed.undo_to_begin_point()
else:
ed.undo_point()
else:
if flags & 0x8 != 0:
ed.redo_to_end_point()
else:
ed.redo_point()
for key, chr_key, ctrl_pressed, alt_pressed, shift_pressed in io.get_key_events(
wnd_name):
if chr_key == 'q' or chr_key == 'z':
do_prev_count = 1 if not shift_pressed else 10
elif chr_key == '-':
zoom_factor = np.clip(zoom_factor - 0.1, 0.1, 4.0)
ed.set_screen_changed()
elif chr_key == '+':
zoom_factor = np.clip(zoom_factor + 0.1, 0.1, 4.0)
ed.set_screen_changed()
elif key == 27: #esc
is_exit = True
next = True
break
elif filepath is not None:
if chr_key == 'e':
saved_ie_polys = ed.ie_polys
do_save_move_count = 1 if not shift_pressed else 10
elif chr_key == 'c':
saved_ie_polys = ed.ie_polys
do_save_count = 1 if not shift_pressed else 10
elif chr_key == 'w':
do_skip_move_count = 1 if not shift_pressed else 10
elif chr_key == 'x':
do_skip_count = 1 if not shift_pressed else 10
elif chr_key == 'r' and saved_ie_polys != None:
ed.set_ie_polys(saved_ie_polys)
if do_prev_count > 0:
do_prev_count -= 1
if len(done_paths) > 0:
if filepath is not None:
image_paths.insert(0, filepath)
filepath = done_paths.pop(-1)
done_images_types[filepath.name] = 0
if filepath.parent != input_path:
new_filename_path = input_path / filepath.name
filepath.rename(new_filename_path)
image_paths.insert(0, new_filename_path)
else:
image_paths.insert(0, filepath)
next = True
elif filepath is not None:
if do_save_move_count > 0:
do_save_move_count -= 1
ed.mask_finish()
dflimg.embed_and_set(
str(filepath),
ie_polys=ed.get_ie_polys(),
eyebrows_expand_mod=eyebrows_expand_mod)
done_paths += [confirmed_path / filepath.name]
done_images_types[filepath.name] = 2
filepath.rename(done_paths[-1])
next = True
elif do_save_count > 0:
do_save_count -= 1
ed.mask_finish()
dflimg.embed_and_set(
str(filepath),
ie_polys=ed.get_ie_polys(),
eyebrows_expand_mod=eyebrows_expand_mod)
done_paths += [filepath]
done_images_types[filepath.name] = 2
next = True
elif do_skip_move_count > 0:
do_skip_move_count -= 1
done_paths += [skipped_path / filepath.name]
done_images_types[filepath.name] = 1
filepath.rename(done_paths[-1])
next = True
elif do_skip_count > 0:
do_skip_count -= 1
done_paths += [filepath]
done_images_types[filepath.name] = 1
next = True
else:
do_save_move_count = do_save_count = do_skip_move_count = do_skip_count = 0
if jobs_count() == 0:
if ed.switch_screen_changed():
screen = ed.make_screen()
if zoom_factor != 1.0:
h, w, c = screen.shape
screen = cv2.resize(
screen,
(int(w * zoom_factor), int(h * zoom_factor)))
io.show_image(wnd_name, screen)
io.process_messages(0.005)
io.destroy_all_windows()
def main(**kwargs):
io.log_info("Running trainer.\r\n")
no_preview = kwargs.get('no_preview', False)
flask_preview = kwargs.get('flask_preview', False)
s2c = queue.Queue()
c2s = queue.Queue()
e = threading.Event()
previews = None
loss_history = None
selected_preview = 0
update_preview = False
is_waiting_preview = False
show_last_history_iters_count = 0
iteration = 0
batch_size = 1
zoom = Zoom.ZOOM_100
if flask_preview:
from flaskr.app import create_flask_app
s2flask = queue.Queue()
socketio, flask_app = create_flask_app(s2c, c2s, s2flask, kwargs)
thread = threading.Thread(target=trainerThread, args=(s2c, c2s, e, socketio), kwargs=kwargs)
thread.start()
e.wait() # Wait for inital load to occur.
flask_t = threading.Thread(target=socketio.run, args=(flask_app,),
kwargs={'debug': True, 'use_reloader': False})
flask_t.start()
while True:
if not c2s.empty():
item = c2s.get()
op = item['op']
if op == 'show':
is_waiting_preview = False
loss_history = item['loss_history'] if 'loss_history' in item.keys() else None
previews = item['previews'] if 'previews' in item.keys() else None
iteration = item['iter'] if 'iter' in item.keys() else 0
# batch_size = input['batch_size'] if 'iter' in input.keys() else 1
if previews is not None:
update_preview = True
elif op == 'update':
if not is_waiting_preview:
is_waiting_preview = True
s2c.put({'op': 'preview'})
elif op == 'next_preview':
selected_preview = (selected_preview + 1) % len(previews)
update_preview = True
elif op == 'change_history_range':
if show_last_history_iters_count == 0:
show_last_history_iters_count = 5000
elif show_last_history_iters_count == 5000:
show_last_history_iters_count = 10000
elif show_last_history_iters_count == 10000:
show_last_history_iters_count = 50000
elif show_last_history_iters_count == 50000:
show_last_history_iters_count = 100000
elif show_last_history_iters_count == 100000:
show_last_history_iters_count = 0
update_preview = True
elif op == 'close':
s2c.put({'op': 'close'})
break
elif op == 'zoom_prev':
zoom = zoom.prev()
update_preview = True
elif op == 'zoom_next':
zoom = zoom.next()
update_preview = True
if update_preview:
update_preview = False
selected_preview = selected_preview % len(previews)
preview_pane_image = create_preview_pane_image(previews,
selected_preview,
loss_history,
show_last_history_iters_count,
iteration,
batch_size,
zoom)
# io.show_image(wnd_name, preview_pane_image)
model_path = Path(kwargs.get('saved_models_path', ''))
filename = 'preview.png'
preview_file = str(model_path / filename)
cv2.imwrite(preview_file, preview_pane_image)
s2flask.put({'op': 'show'})
socketio.emit('preview', {'iter': iteration, 'loss': loss_history[-1]})
try:
io.process_messages(0.01)
except KeyboardInterrupt:
s2c.put({'op': 'close'})
else:
thread = threading.Thread(target=trainerThread, args=(s2c, c2s, e), kwargs=kwargs)
thread.start()
e.wait() # Wait for inital load to occur.
if no_preview:
while True:
if not c2s.empty():
item = c2s.get()
op = item.get('op', '')
if op == 'close':
break
try:
io.process_messages(0.1)
except KeyboardInterrupt:
s2c.put({'op': 'close'})
else:
wnd_name = "Training preview"
io.named_window(wnd_name)
io.capture_keys(wnd_name)
previews = None
loss_history = None
selected_preview = 0
update_preview = False
is_showing = False
is_waiting_preview = False
show_last_history_iters_count = 0
iter = 0
while True:
if not c2s.empty():
item = c2s.get()
op = item['op']
if op == 'show':
is_waiting_preview = False
loss_history = item['loss_history'] if 'loss_history' in item.keys() else None
previews = item['previews'] if 'previews' in item.keys() else None
iter = item['iter'] if 'iter' in item.keys() else 0
if previews is not None:
max_w = 0
max_h = 0
for (preview_name, preview_rgb) in previews:
(h, w, c) = preview_rgb.shape
max_h = max(max_h, h)
max_w = max(max_w, w)
max_size = 800
if max_h > max_size:
max_w = int(max_w / (max_h / max_size))
max_h = max_size
# make all previews size equal
for preview in previews[:]:
(preview_name, preview_rgb) = preview
(h, w, c) = preview_rgb.shape
if h != max_h or w != max_w:
previews.remove(preview)
previews.append((preview_name, cv2.resize(preview_rgb, (max_w, max_h))))
selected_preview = selected_preview % len(previews)
update_preview = True
elif op == 'close':
break
if update_preview:
update_preview = False
selected_preview_name = previews[selected_preview][0]
selected_preview_rgb = previews[selected_preview][1]
(h, w, c) = selected_preview_rgb.shape
# HEAD
head_lines = [
'[s]:save [b]:backup [enter]:exit',
'[p]:update [space]:next preview [l]:change history range',
'Preview: "%s" [%d/%d]' % (selected_preview_name, selected_preview + 1, len(previews))
]
head_line_height = 15
head_height = len(head_lines) * head_line_height
head = np.ones((head_height, w, c)) * 0.1
for i in range(0, len(head_lines)):
t = i * head_line_height
b = (i + 1) * head_line_height
head[t:b, 0:w] += imagelib.get_text_image((head_line_height, w, c), head_lines[i], color=[0.8] * c)
final = head
if loss_history is not None:
if show_last_history_iters_count == 0:
loss_history_to_show = loss_history
else:
loss_history_to_show = loss_history[-show_last_history_iters_count:]
lh_img = models.ModelBase.get_loss_history_preview(loss_history_to_show, iter, w, c)
final = np.concatenate([final, lh_img], axis=0)
final = np.concatenate([final, selected_preview_rgb], axis=0)
final = np.clip(final, 0, 1)
io.show_image(wnd_name, (final * 255).astype(np.uint8))
is_showing = True
key_events = io.get_key_events(wnd_name)
key, chr_key, ctrl_pressed, alt_pressed, shift_pressed = key_events[-1] if len(key_events) > 0 else (
0, 0, False, False, False)
if key == ord('\n') or key == ord('\r'):
s2c.put({'op': 'close'})
elif key == ord('s'):
s2c.put({'op': 'save'})
elif key == ord('b'):
s2c.put({'op': 'backup'})
elif key == ord('p'):
if not is_waiting_preview:
is_waiting_preview = True
s2c.put({'op': 'preview'})
elif key == ord('l'):
if show_last_history_iters_count == 0:
show_last_history_iters_count = 5000
elif show_last_history_iters_count == 5000:
show_last_history_iters_count = 10000
elif show_last_history_iters_count == 10000:
show_last_history_iters_count = 50000
elif show_last_history_iters_count == 50000:
show_last_history_iters_count = 100000
elif show_last_history_iters_count == 100000:
show_last_history_iters_count = 0
update_preview = True
elif key == ord(' '):
selected_preview = (selected_preview + 1) % len(previews)
update_preview = True
try:
io.process_messages(0.1)
except KeyboardInterrupt:
s2c.put({'op': 'close'})
io.destroy_all_windows()