def update_image(self):
self.update_pose()
if (not self.needs_update) and self.last_pose is not None and (
(self.last_pose - self.current_pose).abs().sum().item() < 1e-5):
self.master.after(1000 // 30, self.update_image)
return
if self.source_image is None:
self.master.after(1000 // 30, self.update_image)
return
self.last_pose = self.current_pose
posed_image = self.poser.pose(self.source_image,
self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
pil_image = PIL.Image.fromarray(numpy.uint8(
numpy.rint(numpy_image * 255.0)),
mode='RGBA')
photo_image = PIL.ImageTk.PhotoImage(image=pil_image)
self.posed_image_label.configure(image=photo_image, text="")
self.posed_image_label.image = photo_image
self.posed_image_label.pack()
self.needs_update = False
self.master.after(1000 // 30, self.update_image)
def update_image(self):
there_is_frame, frame = self.video_capture.read()
if not there_is_frame:
return
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = self.face_detector(rgb_frame)
euler_angles = None
face_landmarks = None
if len(faces) > 0:
face_rect = faces[0]
face_landmarks = self.landmark_locator(rgb_frame, face_rect)
face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(face_landmarks)
self.draw_face_landmarks(rgb_frame, face_landmarks)
self.draw_face_box(rgb_frame, face_box_points)
resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1)
pil_image = PIL.Image.fromarray(resized_frame, mode='RGB')
photo_image = PIL.ImageTk.PhotoImage(image=pil_image)
self.video_capture_label.configure(image=photo_image, text="")
self.video_capture_label.image = photo_image
self.video_capture_label.pack()
if euler_angles is not None and self.source_image is not None:
self.current_pose = torch.zeros(self.pose_size, device=self.torch_device)
self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0)
self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0)
self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0)
if self.last_pose is None:
self.last_pose = self.current_pose
else:
self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5
self.last_pose = self.current_pose
eye_min_ratio = 0.15
eye_max_ratio = 0.25
left_eye_normalized_ratio = compute_left_eye_normalized_ratio(face_landmarks, eye_min_ratio, eye_max_ratio)
self.current_pose[3] = 1 - left_eye_normalized_ratio
right_eye_normalized_ratio = compute_right_eye_normalized_ratio(face_landmarks,
eye_min_ratio,
eye_max_ratio)
self.current_pose[4] = 1 - right_eye_normalized_ratio
min_mouth_ratio = 0.02
max_mouth_ratio = 0.3
mouth_normalized_ratio = compute_mouth_normalized_ratio(face_landmarks, min_mouth_ratio, max_mouth_ratio)
self.current_pose[5] = mouth_normalized_ratio
self.current_pose = self.current_pose.unsqueeze(dim=0)
posed_image = self.poser.pose(self.source_image, self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA')
photo_image = PIL.ImageTk.PhotoImage(image=pil_image)
self.posed_image_label.configure(image=photo_image, text="")
self.posed_image_label.image = photo_image
self.posed_image_label.pack()
self.master.after(1000 // 60, self.update_image)
def update_image(self, frame):
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = self.face_detector(rgb_frame)
euler_angles = None
face_landmarks = None
if len(faces) > 0:
face_rect = faces[0]
face_landmarks = self.landmark_locator(rgb_frame, face_rect)
face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(
face_landmarks)
resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1)
if euler_angles is not None and self.source_image is not None:
self.current_pose = torch.zeros(self.pose_size,
device=self.torch_device)
self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0),
-1.0)
self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0),
-1.0)
self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0),
-1.0)
if self.last_pose is None:
self.last_pose = self.current_pose
else:
self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5
self.last_pose = self.current_pose
eye_min_ratio = 0.15
eye_max_ratio = 0.25
left_eye_normalized_ratio = compute_left_eye_normalized_ratio(
face_landmarks, eye_min_ratio, eye_max_ratio)
self.current_pose[3] = 1 - left_eye_normalized_ratio
right_eye_normalized_ratio = compute_right_eye_normalized_ratio(
face_landmarks, eye_min_ratio, eye_max_ratio)
self.current_pose[4] = 1 - right_eye_normalized_ratio
min_mouth_ratio = 0.02
max_mouth_ratio = 0.3
mouth_normalized_ratio = compute_mouth_normalized_ratio(
face_landmarks, min_mouth_ratio, max_mouth_ratio)
self.current_pose[5] = mouth_normalized_ratio
self.current_pose = self.current_pose.unsqueeze(dim=0)
posed_image = self.poser.pose(self.source_image,
self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
numpy_image = cv2.cvtColor(numpy_image, cv2.COLOR_BGR2RGB)
#pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA')
return numpy_image
def create_frame(self, frame):
there_is_frame = True
if not there_is_frame:
return
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = self.face_detector(rgb_frame)
euler_angles = None
face_landmarks = None
if len(faces) > 0:
face_rect = faces[0]
face_landmarks = self.landmark_locator(rgb_frame, face_rect)
face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(
face_landmarks)
self.draw_face_landmarks(rgb_frame, face_landmarks)
self.draw_face_box(rgb_frame, face_box_points)
resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1)
pil_image = PIL.Image.fromarray(resized_frame, mode='RGB')
if euler_angles is not None and self.source_image is not None:
self.current_pose = torch.zeros(self.pose_size,
device=self.torch_device)
self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0),
-1.0)
self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0),
-1.0)
self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0),
-1.0)
if self.last_pose is None:
self.last_pose = self.current_pose
else:
self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5
self.last_pose = self.current_pose
min_mouth_ratio = 0.02
max_mouth_ratio = 0.3
mouth_normalized_ratio = compute_mouth_normalized_ratio(
face_landmarks, min_mouth_ratio, max_mouth_ratio)
self.current_pose[5] = mouth_normalized_ratio
self.current_pose = self.current_pose.unsqueeze(dim=0)
posed_image = self.poser.pose(self.source_image,
self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
pil_image = PIL.Image.fromarray(np.uint8(
np.rint(numpy_image * 255.0)),
mode='RGBA')
photo_image = np.array(pil_image)
self.anime_video_frames.append(photo_image)
def run(self, source_img, frame, save_path):
# rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
# faces = self.face_detector(rgb_frame)
# euler_angles = None
# face_landmarks = None
# if len(faces) > 0:
# face_rect = faces[0]
# face_landmarks = self.landmark_locator(rgb_frame, face_rect)
# face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(face_landmarks)
# # np.save("euler_angle.npy",euler_angles)
# # euler_angles = np.load("euler_angle.npy")
#
# if euler_angles is not None and source_img is not None:
# self.current_pose = torch.zeros(self.pose_size, device=self.torch_device)
# self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0)
# self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0)
# self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0)
#
# # if self.last_pose is None:
# # self.last_pose = self.current_pose
# # else:
# # self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5 # smoothing
# # self.last_pose = self.current_pose
#
# eye_min_ratio = 0.15
# eye_max_ratio = 0.25
# left_eye_normalized_ratio = compute_left_eye_normalized_ratio(face_landmarks, eye_min_ratio, eye_max_ratio)
# self.current_pose[3] = 1 - left_eye_normalized_ratio
# right_eye_normalized_ratio = compute_right_eye_normalized_ratio(face_landmarks,
# eye_min_ratio,
# eye_max_ratio)
# self.current_pose[4] = 1 - right_eye_normalized_ratio
#
# min_mouth_ratio = 0.02
# max_mouth_ratio = 0.3
# mouth_normalized_ratio = compute_mouth_normalized_ratio(face_landmarks, min_mouth_ratio, max_mouth_ratio)
# self.current_pose[5] = mouth_normalized_ratio
# self.current_pose = self.current_pose.unsqueeze(dim=0)
# np.save("current_pose.npy",self.current_pose.cpu())
self.current_pose = torch.Tensor(np.load("current_pose.npy")).cuda()
st = time.time()
posed_image = self.poser.pose(source_image=source_img,
pose=self.current_pose).detach().cpu()
print("Core Time(poser.pose): ", time.time() - st)
numpy_image = rgba_to_numpy_image(posed_image[0])
pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)),
mode='RGBA')
pil_image.save(save_path)
# TODO Core of demo, compress this
return