def update_image(self):
there_is_frame, frame = self.video_capture.read()
if not there_is_frame:
return
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = self.face_detector(rgb_frame)
euler_angles = None
face_landmarks = None
if len(faces) > 0:
face_rect = faces[0]
face_landmarks = self.landmark_locator(rgb_frame, face_rect)
face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(face_landmarks)
self.draw_face_landmarks(rgb_frame, face_landmarks)
self.draw_face_box(rgb_frame, face_box_points)
resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1)
pil_image = PIL.Image.fromarray(resized_frame, mode='RGB')
photo_image = PIL.ImageTk.PhotoImage(image=pil_image)
self.video_capture_label.configure(image=photo_image, text="")
self.video_capture_label.image = photo_image
self.video_capture_label.pack()
if euler_angles is not None and self.source_image is not None:
self.current_pose = torch.zeros(self.pose_size, device=self.torch_device)
self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0)
self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0)
self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0)
if self.last_pose is None:
self.last_pose = self.current_pose
else:
self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5
self.last_pose = self.current_pose
eye_min_ratio = 0.15
eye_max_ratio = 0.25
left_eye_normalized_ratio = compute_left_eye_normalized_ratio(face_landmarks, eye_min_ratio, eye_max_ratio)
self.current_pose[3] = 1 - left_eye_normalized_ratio
right_eye_normalized_ratio = compute_right_eye_normalized_ratio(face_landmarks,
eye_min_ratio,
eye_max_ratio)
self.current_pose[4] = 1 - right_eye_normalized_ratio
min_mouth_ratio = 0.02
max_mouth_ratio = 0.3
mouth_normalized_ratio = compute_mouth_normalized_ratio(face_landmarks, min_mouth_ratio, max_mouth_ratio)
self.current_pose[5] = mouth_normalized_ratio
self.current_pose = self.current_pose.unsqueeze(dim=0)
posed_image = self.poser.pose(self.source_image, self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA')
photo_image = PIL.ImageTk.PhotoImage(image=pil_image)
self.posed_image_label.configure(image=photo_image, text="")
self.posed_image_label.image = photo_image
self.posed_image_label.pack()
self.master.after(1000 // 60, self.update_image)
def update_image(self, frame):
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = self.face_detector(rgb_frame)
euler_angles = None
face_landmarks = None
if len(faces) > 0:
face_rect = faces[0]
face_landmarks = self.landmark_locator(rgb_frame, face_rect)
face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(
face_landmarks)
resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1)
if euler_angles is not None and self.source_image is not None:
self.current_pose = torch.zeros(self.pose_size,
device=self.torch_device)
self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0),
-1.0)
self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0),
-1.0)
self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0),
-1.0)
if self.last_pose is None:
self.last_pose = self.current_pose
else:
self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5
self.last_pose = self.current_pose
eye_min_ratio = 0.15
eye_max_ratio = 0.25
left_eye_normalized_ratio = compute_left_eye_normalized_ratio(
face_landmarks, eye_min_ratio, eye_max_ratio)
self.current_pose[3] = 1 - left_eye_normalized_ratio
right_eye_normalized_ratio = compute_right_eye_normalized_ratio(
face_landmarks, eye_min_ratio, eye_max_ratio)
self.current_pose[4] = 1 - right_eye_normalized_ratio
min_mouth_ratio = 0.02
max_mouth_ratio = 0.3
mouth_normalized_ratio = compute_mouth_normalized_ratio(
face_landmarks, min_mouth_ratio, max_mouth_ratio)
self.current_pose[5] = mouth_normalized_ratio
self.current_pose = self.current_pose.unsqueeze(dim=0)
posed_image = self.poser.pose(self.source_image,
self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
numpy_image = cv2.cvtColor(numpy_image, cv2.COLOR_BGR2RGB)
#pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA')
return numpy_image
def create_frame(self, frame):
there_is_frame = True
if not there_is_frame:
return
rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = self.face_detector(rgb_frame)
euler_angles = None
face_landmarks = None
if len(faces) > 0:
face_rect = faces[0]
face_landmarks = self.landmark_locator(rgb_frame, face_rect)
face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(
face_landmarks)
self.draw_face_landmarks(rgb_frame, face_landmarks)
self.draw_face_box(rgb_frame, face_box_points)
resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1)
pil_image = PIL.Image.fromarray(resized_frame, mode='RGB')
if euler_angles is not None and self.source_image is not None:
self.current_pose = torch.zeros(self.pose_size,
device=self.torch_device)
self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0),
-1.0)
self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0),
-1.0)
self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0),
-1.0)
if self.last_pose is None:
self.last_pose = self.current_pose
else:
self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5
self.last_pose = self.current_pose
min_mouth_ratio = 0.02
max_mouth_ratio = 0.3
mouth_normalized_ratio = compute_mouth_normalized_ratio(
face_landmarks, min_mouth_ratio, max_mouth_ratio)
self.current_pose[5] = mouth_normalized_ratio
self.current_pose = self.current_pose.unsqueeze(dim=0)
posed_image = self.poser.pose(self.source_image,
self.current_pose).detach().cpu()
numpy_image = rgba_to_numpy_image(posed_image[0])
pil_image = PIL.Image.fromarray(np.uint8(
np.rint(numpy_image * 255.0)),
mode='RGBA')
photo_image = np.array(pil_image)
self.anime_video_frames.append(photo_image)