def update_image(self): self.update_pose() if (not self.needs_update) and self.last_pose is not None and ( (self.last_pose - self.current_pose).abs().sum().item() < 1e-5): self.master.after(1000 // 30, self.update_image) return if self.source_image is None: self.master.after(1000 // 30, self.update_image) return self.last_pose = self.current_pose posed_image = self.poser.pose(self.source_image, self.current_pose).detach().cpu() numpy_image = rgba_to_numpy_image(posed_image[0]) pil_image = PIL.Image.fromarray(numpy.uint8( numpy.rint(numpy_image * 255.0)), mode='RGBA') photo_image = PIL.ImageTk.PhotoImage(image=pil_image) self.posed_image_label.configure(image=photo_image, text="") self.posed_image_label.image = photo_image self.posed_image_label.pack() self.needs_update = False self.master.after(1000 // 30, self.update_image)
def update_image(self): there_is_frame, frame = self.video_capture.read() if not there_is_frame: return rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) faces = self.face_detector(rgb_frame) euler_angles = None face_landmarks = None if len(faces) > 0: face_rect = faces[0] face_landmarks = self.landmark_locator(rgb_frame, face_rect) face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(face_landmarks) self.draw_face_landmarks(rgb_frame, face_landmarks) self.draw_face_box(rgb_frame, face_box_points) resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1) pil_image = PIL.Image.fromarray(resized_frame, mode='RGB') photo_image = PIL.ImageTk.PhotoImage(image=pil_image) self.video_capture_label.configure(image=photo_image, text="") self.video_capture_label.image = photo_image self.video_capture_label.pack() if euler_angles is not None and self.source_image is not None: self.current_pose = torch.zeros(self.pose_size, device=self.torch_device) self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0) self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0) self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0) if self.last_pose is None: self.last_pose = self.current_pose else: self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5 self.last_pose = self.current_pose eye_min_ratio = 0.15 eye_max_ratio = 0.25 left_eye_normalized_ratio = compute_left_eye_normalized_ratio(face_landmarks, eye_min_ratio, eye_max_ratio) self.current_pose[3] = 1 - left_eye_normalized_ratio right_eye_normalized_ratio = compute_right_eye_normalized_ratio(face_landmarks, eye_min_ratio, eye_max_ratio) self.current_pose[4] = 1 - right_eye_normalized_ratio min_mouth_ratio = 0.02 max_mouth_ratio = 0.3 mouth_normalized_ratio = compute_mouth_normalized_ratio(face_landmarks, min_mouth_ratio, max_mouth_ratio) self.current_pose[5] = mouth_normalized_ratio self.current_pose = self.current_pose.unsqueeze(dim=0) posed_image = self.poser.pose(self.source_image, self.current_pose).detach().cpu() numpy_image = rgba_to_numpy_image(posed_image[0]) pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA') photo_image = PIL.ImageTk.PhotoImage(image=pil_image) self.posed_image_label.configure(image=photo_image, text="") self.posed_image_label.image = photo_image self.posed_image_label.pack() self.master.after(1000 // 60, self.update_image)
def update_image(self, frame): rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) faces = self.face_detector(rgb_frame) euler_angles = None face_landmarks = None if len(faces) > 0: face_rect = faces[0] face_landmarks = self.landmark_locator(rgb_frame, face_rect) face_box_points, euler_angles = self.head_pose_solver.solve_head_pose( face_landmarks) resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1) if euler_angles is not None and self.source_image is not None: self.current_pose = torch.zeros(self.pose_size, device=self.torch_device) self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0) self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0) self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0) if self.last_pose is None: self.last_pose = self.current_pose else: self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5 self.last_pose = self.current_pose eye_min_ratio = 0.15 eye_max_ratio = 0.25 left_eye_normalized_ratio = compute_left_eye_normalized_ratio( face_landmarks, eye_min_ratio, eye_max_ratio) self.current_pose[3] = 1 - left_eye_normalized_ratio right_eye_normalized_ratio = compute_right_eye_normalized_ratio( face_landmarks, eye_min_ratio, eye_max_ratio) self.current_pose[4] = 1 - right_eye_normalized_ratio min_mouth_ratio = 0.02 max_mouth_ratio = 0.3 mouth_normalized_ratio = compute_mouth_normalized_ratio( face_landmarks, min_mouth_ratio, max_mouth_ratio) self.current_pose[5] = mouth_normalized_ratio self.current_pose = self.current_pose.unsqueeze(dim=0) posed_image = self.poser.pose(self.source_image, self.current_pose).detach().cpu() numpy_image = rgba_to_numpy_image(posed_image[0]) numpy_image = cv2.cvtColor(numpy_image, cv2.COLOR_BGR2RGB) #pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA') return numpy_image
def create_frame(self, frame): there_is_frame = True if not there_is_frame: return rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) faces = self.face_detector(rgb_frame) euler_angles = None face_landmarks = None if len(faces) > 0: face_rect = faces[0] face_landmarks = self.landmark_locator(rgb_frame, face_rect) face_box_points, euler_angles = self.head_pose_solver.solve_head_pose( face_landmarks) self.draw_face_landmarks(rgb_frame, face_landmarks) self.draw_face_box(rgb_frame, face_box_points) resized_frame = cv2.flip(cv2.resize(rgb_frame, (192, 256)), 1) pil_image = PIL.Image.fromarray(resized_frame, mode='RGB') if euler_angles is not None and self.source_image is not None: self.current_pose = torch.zeros(self.pose_size, device=self.torch_device) self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0) self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0) self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0) if self.last_pose is None: self.last_pose = self.current_pose else: self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5 self.last_pose = self.current_pose min_mouth_ratio = 0.02 max_mouth_ratio = 0.3 mouth_normalized_ratio = compute_mouth_normalized_ratio( face_landmarks, min_mouth_ratio, max_mouth_ratio) self.current_pose[5] = mouth_normalized_ratio self.current_pose = self.current_pose.unsqueeze(dim=0) posed_image = self.poser.pose(self.source_image, self.current_pose).detach().cpu() numpy_image = rgba_to_numpy_image(posed_image[0]) pil_image = PIL.Image.fromarray(np.uint8( np.rint(numpy_image * 255.0)), mode='RGBA') photo_image = np.array(pil_image) self.anime_video_frames.append(photo_image)
def run(self, source_img, frame, save_path): # rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) # faces = self.face_detector(rgb_frame) # euler_angles = None # face_landmarks = None # if len(faces) > 0: # face_rect = faces[0] # face_landmarks = self.landmark_locator(rgb_frame, face_rect) # face_box_points, euler_angles = self.head_pose_solver.solve_head_pose(face_landmarks) # # np.save("euler_angle.npy",euler_angles) # # euler_angles = np.load("euler_angle.npy") # # if euler_angles is not None and source_img is not None: # self.current_pose = torch.zeros(self.pose_size, device=self.torch_device) # self.current_pose[0] = max(min(-euler_angles.item(0) / 15.0, 1.0), -1.0) # self.current_pose[1] = max(min(-euler_angles.item(1) / 15.0, 1.0), -1.0) # self.current_pose[2] = max(min(euler_angles.item(2) / 15.0, 1.0), -1.0) # # # if self.last_pose is None: # # self.last_pose = self.current_pose # # else: # # self.current_pose = self.current_pose * 0.5 + self.last_pose * 0.5 # smoothing # # self.last_pose = self.current_pose # # eye_min_ratio = 0.15 # eye_max_ratio = 0.25 # left_eye_normalized_ratio = compute_left_eye_normalized_ratio(face_landmarks, eye_min_ratio, eye_max_ratio) # self.current_pose[3] = 1 - left_eye_normalized_ratio # right_eye_normalized_ratio = compute_right_eye_normalized_ratio(face_landmarks, # eye_min_ratio, # eye_max_ratio) # self.current_pose[4] = 1 - right_eye_normalized_ratio # # min_mouth_ratio = 0.02 # max_mouth_ratio = 0.3 # mouth_normalized_ratio = compute_mouth_normalized_ratio(face_landmarks, min_mouth_ratio, max_mouth_ratio) # self.current_pose[5] = mouth_normalized_ratio # self.current_pose = self.current_pose.unsqueeze(dim=0) # np.save("current_pose.npy",self.current_pose.cpu()) self.current_pose = torch.Tensor(np.load("current_pose.npy")).cuda() st = time.time() posed_image = self.poser.pose(source_image=source_img, pose=self.current_pose).detach().cpu() print("Core Time(poser.pose): ", time.time() - st) numpy_image = rgba_to_numpy_image(posed_image[0]) pil_image = PIL.Image.fromarray(np.uint8(np.rint(numpy_image * 255.0)), mode='RGBA') pil_image.save(save_path) # TODO Core of demo, compress this return