def test_optimization_multiple_images(): image = Image.open('./images/faces_sparser_sampling.gif') lambda_alpha = 50 lambda_delta = 0.3 lr = .128 num_epochs = 300 num_frames = 100 trained_model, frames = optimization( num_epochs=num_epochs, path='./images/faces_sparser_sampling.gif', shape=(image.width, image.height), num_frames=num_frames, lambda_alpha=lambda_alpha, lambda_delta=lambda_delta, lr=lr) pca = trained_model.p for frame_idx, frame in enumerate(frames): # obtain texture from mask on image points_2d = trained_model.forward( frame_idx, only_lm=False).detach().numpy() # 28588, 2 tex = np.array(texture(np.array(frame), points_2d)) # look at 3D visualization of the estimated face shape points_3d = get_face_point_cloud(pca, trained_model.alpha, trained_model.delta[frame_idx]).view( (-1, 3)) """ mesh = trimesh.base.Trimesh(vertices=points_3d.detach().numpy(), faces=triangles, vertex_colors=tex) mesh.show() """ # get T matrix for only rotation T = np.eye(4) T[:3, :3] = rotation_matrix(np.array([0, 0, 0]), is_numpy=True) # save resulting rotated face G = points_3d.detach().numpy().T G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0) mesh = Mesh(vertices=(T @ G_h)[:3].T, colors=tex, triangles=triangles) mesh_to_png(f"./results/optimization/multiple/shapes/{frame_idx}.png", mesh, z_camera_translation=280) # show estimated landmarks landmarks = trained_model.forward(frame_idx) landmarks = landmarks.detach().numpy().T plt.figure(figsize=(600 / 100, 600 / 100), dpi=100) #plt.scatter(landmarks[0], landmarks[1]) plt.imshow(np.array(frame)) plt.axis('off') plt.savefig( f"./results/optimization/multiple/landmarks/{frame_idx}.png", bbox_inches='tight', pad_inches=0, dpi=100)
def test_optimization_single_frame(path='./images/first_frame.png'): image = Image.open(path) lambda_alpha = 30 lambda_delta = 0.3 lr = .128 num_epochs = 400 num_frames = 1 trained_model, frames = optimization(num_epochs=num_epochs, path=path, shape=(image.width, image.height), num_frames=num_frames, lambda_alpha=lambda_alpha, lambda_delta=lambda_delta, lr=lr) pca = trained_model.p # look at 3D visualization of the estimated face shape points_3d = get_face_point_cloud(pca, trained_model.alpha, trained_model.delta[0]).view( (-1, 3)) # 28588, 3 # obtain texture from mask on image points_2d = trained_model.forward( 0, only_lm=False).detach().numpy() # 28588, 2 tex = np.array(texture(np.array(image), points_2d)) #mesh = trimesh.base.Trimesh(vertices=points_3d.detach().numpy(), faces=triangles, vertex_colors=tex) #mesh.show() # show estimated landmarks landmarks = trained_model.forward(0) landmarks = landmarks.detach().numpy().T plt.scatter(landmarks[0], landmarks[1]) plt.imshow(np.array(frames[0])) plt.axis('off') plt.savefig(f"./results/optimization/landmarks.png", bbox_inches='tight', pad_inches=0, dpi=100) # show from different angles G = points_3d.detach().numpy().T G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0) for w in [[0, 0, 0], [0, -30, 0], [0, -45, 0], [0, -90, 0]]: w = np.array(w) # get T matrix for only rotation T = np.eye(4) T[:3, :3] = rotation_matrix(w, is_numpy=True) # save resulting rotated face mesh = Mesh(vertices=(T @ G_h)[:3].T, colors=tex, triangles=triangles) mesh_to_png(f"./results/optimization/single/tex_{w}.png", mesh, z_camera_translation=280)
def rotate_face(angles): """ Task 3.1 :param angles: list of angles. each angle has three entries [theta_x, theta_y, theta_z] :return: """ # sample face pca = read_pca_model() G = random_face_point_cloud(pca).T # transform to homogeneous coordinates G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0) for w in angles: w = np.array(w) # get T matrix for only rotation T = np.eye(4) T[:3, :3] = rotation_matrix(w, is_numpy=True) # save resulting rotated face mesh = Mesh(vertices=(T @ G_h)[:3].T, colors=mean_tex, triangles=triangles) mesh_to_png("./results/rotation/"+str(w)+".png", mesh) return
def facial_landmarks(alpha, delta, w, t): """ Construct facial landmarks from facial geometry latent parameters alpha, delta and object transformation w, t. :param alpha: array, 30dim :param delta: array, 20dim :param w: rotation angles around x,y, z. Given as list [theta_x, theta_y, theta_z]. :param t: translation in x,y,z space. Given as list [translation_x, translation_y, translation_z] :return: """ landmarks_idx = np.loadtxt("./models/Landmarks68_model2017-1_face12_nomouth.anl", dtype=int) pca = read_pca_model() G = get_face_point_cloud(pca, alpha, delta).reshape((-1, 3))[landmarks_idx].T G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0) # get T matrix T = np.eye(4) T[:3, :3] = rotation_matrix(w, is_numpy=True) T[:3, 3] = t # Get V and P matrices W = H = 255 # angle 10 P = perspective_projection_matrix(W, H, 300, 2000, is_numpy=True) V = viewport_matrix(right=W, left=0, top=H, bottom=0, is_numpy=True) i = V @ P @ T @ G_h # cartesian i /= i[3,:] # two-dimensional return i[:2, :]
# transformation = matrices.translation_matrix(-200, 0, -300) # wireframes[0].transform(transformation) FOCAL_LIMITS = 20., 500. FOCAL_STEP = 2. TRANSLATION_STEP = 10. ROTATION_STEP = np.radians(0.8) left_translation = matrices.translation_matrix(TRANSLATION_STEP, 0, 0) right_translation = matrices.translation_matrix(-TRANSLATION_STEP, 0, 0) forwart_translation = matrices.translation_matrix(0, -TRANSLATION_STEP, 0) backward_translation = matrices.translation_matrix(0, +TRANSLATION_STEP, 0) up_translation = matrices.translation_matrix(0, 0, -TRANSLATION_STEP) down_translation = matrices.translation_matrix(0, 0, +TRANSLATION_STEP) cunter_clockwise_rotation = matrices.rotation_matrix(ROTATION_STEP, 'z') clokcwise_rotation = matrices.rotation_matrix(-ROTATION_STEP, 'z') up_rotation = matrices.rotation_matrix(ROTATION_STEP, 'x') down_rotation = matrices.rotation_matrix(-ROTATION_STEP, 'x') left_rotation = matrices.rotation_matrix(ROTATION_STEP * 2, 'y') right_rotation = matrices.rotation_matrix(-ROTATION_STEP * 2, 'y') running = True while running: for event in pygame.event.get(): if event.type == pygame.QUIT: running = False keys = pygame.key.get_pressed() if keys[pygame.K_MINUS]: if focal-FOCAL_STEP > FOCAL_LIMITS[0]: focal -= FOCAL_STEP