def texture3D(gt_image, alpha, beta, omega, t, i):
# generate face based on given parameters alpha and beta and transform this face with omage and t.
[_, pCexp, triangles, transformed_face] = generate_face(alpha, beta, omega, t)
projection2D = transformed_face[:, :2] # the 2D projected datapoints (x, y) for each of points in 3D pointcloud pCexp
pCexp = pCexp.data.numpy()
projection2D = projection2D.data.numpy()
# create texture file:
[noPoints, _] = np.shape(projection2D)
projected_tex = np.zeros((noPoints, 3)) # same size as mean_tex
# for each point in 2Dprojected generated face, find its texture from the original gt_image by bilinear interpolation
for Point in range(noPoints):
[xc, yc] = projection2D[Point, :]
x1 = floor(xc)
x2 = ceil(xc)
y1 = floor(yc)
y2 = ceil(yc)
[_, ySize, _] = np.shape(gt_image)
v11 = gt_image[ySize - y1, x1, :]
v12 = gt_image[ySize - y2, x1, :]
v21 = gt_image[ySize - y1, x2, :]
v22 = gt_image[ySize - y2, x2, :]
vc = calc_bilinear_interpol(x1, x2, y1, y2, xc, yc, v11, v12, v21, v22) # vc: dim 3 vector containing RGB values
# make this RGB value the value for pointcloud
projected_tex[Point, :] = vc
# print the result - RESULTS LOOK FUNNY SINCE I HAVE CHOSEN NOT OPTIMAL PARAMETERS alpha, beta, omega, t
mesh = Mesh(pCexp, projected_tex, triangles)
mesh_to_png("./output/generated_faces/run_1_face_{}.png".format(i), mesh)
return
def test_optimization_multiple_images():
image = Image.open('./images/faces_sparser_sampling.gif')
lambda_alpha = 50
lambda_delta = 0.3
lr = .128
num_epochs = 300
num_frames = 100
trained_model, frames = optimization(
num_epochs=num_epochs,
path='./images/faces_sparser_sampling.gif',
shape=(image.width, image.height),
num_frames=num_frames,
lambda_alpha=lambda_alpha,
lambda_delta=lambda_delta,
lr=lr)
pca = trained_model.p
for frame_idx, frame in enumerate(frames):
# obtain texture from mask on image
points_2d = trained_model.forward(
frame_idx, only_lm=False).detach().numpy() # 28588, 2
tex = np.array(texture(np.array(frame), points_2d))
# look at 3D visualization of the estimated face shape
points_3d = get_face_point_cloud(pca, trained_model.alpha,
trained_model.delta[frame_idx]).view(
(-1, 3))
"""
mesh = trimesh.base.Trimesh(vertices=points_3d.detach().numpy(), faces=triangles, vertex_colors=tex)
mesh.show()
"""
# get T matrix for only rotation
T = np.eye(4)
T[:3, :3] = rotation_matrix(np.array([0, 0, 0]), is_numpy=True)
# save resulting rotated face
G = points_3d.detach().numpy().T
G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0)
mesh = Mesh(vertices=(T @ G_h)[:3].T, colors=tex, triangles=triangles)
mesh_to_png(f"./results/optimization/multiple/shapes/{frame_idx}.png",
mesh,
z_camera_translation=280)
# show estimated landmarks
landmarks = trained_model.forward(frame_idx)
landmarks = landmarks.detach().numpy().T
plt.figure(figsize=(600 / 100, 600 / 100), dpi=100)
#plt.scatter(landmarks[0], landmarks[1])
plt.imshow(np.array(frame))
plt.axis('off')
plt.savefig(
f"./results/optimization/multiple/landmarks/{frame_idx}.png",
bbox_inches='tight',
pad_inches=0,
dpi=100)
def test_optimization_single_frame(path='./images/first_frame.png'):
image = Image.open(path)
lambda_alpha = 30
lambda_delta = 0.3
lr = .128
num_epochs = 400
num_frames = 1
trained_model, frames = optimization(num_epochs=num_epochs,
path=path,
shape=(image.width, image.height),
num_frames=num_frames,
lambda_alpha=lambda_alpha,
lambda_delta=lambda_delta,
lr=lr)
pca = trained_model.p
# look at 3D visualization of the estimated face shape
points_3d = get_face_point_cloud(pca, trained_model.alpha,
trained_model.delta[0]).view(
(-1, 3)) # 28588, 3
# obtain texture from mask on image
points_2d = trained_model.forward(
0, only_lm=False).detach().numpy() # 28588, 2
tex = np.array(texture(np.array(image), points_2d))
#mesh = trimesh.base.Trimesh(vertices=points_3d.detach().numpy(), faces=triangles, vertex_colors=tex)
#mesh.show()
# show estimated landmarks
landmarks = trained_model.forward(0)
landmarks = landmarks.detach().numpy().T
plt.scatter(landmarks[0], landmarks[1])
plt.imshow(np.array(frames[0]))
plt.axis('off')
plt.savefig(f"./results/optimization/landmarks.png",
bbox_inches='tight',
pad_inches=0,
dpi=100)
# show from different angles
G = points_3d.detach().numpy().T
G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0)
for w in [[0, 0, 0], [0, -30, 0], [0, -45, 0], [0, -90, 0]]:
w = np.array(w)
# get T matrix for only rotation
T = np.eye(4)
T[:3, :3] = rotation_matrix(w, is_numpy=True)
# save resulting rotated face
mesh = Mesh(vertices=(T @ G_h)[:3].T, colors=tex, triangles=triangles)
mesh_to_png(f"./results/optimization/single/tex_{w}.png",
mesh,
z_camera_translation=280)
def sample_face_pointclouds(num_samples=24):
# read pca model from files
pca_model = read_pca_model()
for i in range(num_samples):
# sample new face with given formula
alpha = U(i=1)
delta = U(i=1)
f_pc = sample_face(pca_model, alpha, delta).reshape(
(-1, 3)) # (3N, )= (85764,)
# show mesh:
# mesh = trimesh.base.Trimesh(vertices=f_pc, faces=triangles, vertex_colors=mean_tex)
# mesh.show()
# save mesh
mesh = Mesh(vertices=f_pc, colors=mean_tex, triangles=triangles)
mesh_to_png(f"./results/morphable_model/{str(i)}.png", mesh)
def generate_face_images(num_samples=24):
# read pca model from files
pca_model = read_pca_model()
for i in range(num_samples):
# sample new face with given formula
f_pc = random_face_point_cloud(pca_model) # (3N, )= (85764,)
# show mesh:
# mesh = trimesh.base.Trimesh(vertices=f_pc, faces=triangles, vertex_colors=mean_tex)
# mesh.show()
# save mesh
mesh = Mesh(vertices=f_pc, colors=mean_tex, triangles=triangles)
mesh_to_png(f"./results/morphable_model/{str(i)}.png",
mesh,
width=400,
z_camera_translation=280)
return
def load_obj(obj_path):
vertices = []
colors = []
triangles = []
with open(obj_path, 'r') as f:
for line in f:
tokens = line.split()
if len(tokens) == 0:
continue
if tokens[0] == 'v':
vertices.append(list(map(float, tokens[1:4])))
if len(tokens) > 4:
colors.append(list(map(float, tokens[4:])))
elif tokens[0] == 'f':
triangles.append(
list(map(lambda x: int(x.split('/', 1)[0]) - 1, tokens[1:4])))
vertices = np.asarray(vertices, dtype=np.float32)
colors = np.asarray(colors, dtype=np.float32)
triangles = np.asarray(triangles, dtype=np.int32)
return Mesh(vertices, colors, triangles)
def rotate_face(angles):
"""
Task 3.1
:param angles: list of angles. each angle has three entries [theta_x, theta_y, theta_z]
:return:
"""
# sample face
pca = read_pca_model()
G = random_face_point_cloud(pca).T
# transform to homogeneous coordinates
G_h = np.append(G, np.ones(G.shape[1]).reshape((1, -1)), axis=0)
for w in angles:
w = np.array(w)
# get T matrix for only rotation
T = np.eye(4)
T[:3, :3] = rotation_matrix(w, is_numpy=True)
# save resulting rotated face
mesh = Mesh(vertices=(T @ G_h)[:3].T, colors=mean_tex, triangles=triangles)
mesh_to_png("./results/rotation/"+str(w)+".png", mesh)
return
def geo_to_im(geo, color, tri, resolution=None):
mesh = Mesh(geo, color, tri)
png = mesh_to_png(mesh, resolution=resolution)
im = png_to_im(png)
return im
import numpy as np
import trimesh
import pyrender
import h5py
from data_def import PCAModel, Mesh
bfm = h5py.File("model2017-1_face12_nomouth.h5", 'r')
mean_shape = np.asarray(bfm['shape/model/mean'], dtype=np.float32).reshape(
(-1, 3))
mean_tex = np.asarray(bfm['color/model/mean'], dtype=np.float32).reshape(
(-1, 3))
triangles = np.asarray(bfm['shape/representer/cells'], dtype=np.int32).T
def mesh_to_png(file_name, mesh):
mesh = trimesh.base.Trimesh(vertices=mesh.vertices,
faces=mesh.triangles,
vertex_colors=mesh.colors)
png = mesh.scene().save_image()
with open(file_name, 'wb') as f:
f.write(png)
if __name__ == '__main__':
mesh = Mesh(mean_shape, mean_tex, triangles)
mesh_to_png("debug.png", mesh)
for i in range(len(landmark_indices)):
mean_tex[landmark_indices[i]] = [0, 0, 1]
else:
landmarks = None
return transformed_pCexp, landmarks
if __name__ == "__main__":
# because of rendering problem I needed to run this program for every mesh_to_png seperately
alpha = torch.tensor(np.random.uniform(-1, 1, 30), dtype=torch.float64)
delta = torch.tensor(np.random.uniform(-1, 1, 20), dtype=torch.float64)
pCid, pCexp, mean_tex, triangles = load_faces(alpha, delta)
mesh = Mesh(pCexp, mean_tex, triangles)
mesh_to_png('pinhole__.png', mesh)
translation = torch.tensor([0, 0, 0], dtype=torch.float64)
# -10 degrees around y axis
transformed_face, _ = run_pinhole_camera(torch.tensor([0, -10, 0],
dtype=torch.float64),
translation,
pCexp,
mean_tex,
test=True)
mesh = Mesh(transformed_face[:, :-1], mean_tex, triangles)
mesh_to_png('pinhole_-10__.png', mesh)
# +10 degrees around y axis
from PIL import Image, ImageSequence
from optimization import optimization_one_image
from mesh_to_png import triangles, mean_tex, mesh_to_png
from data_def import Mesh
from morphable_model import get_face_point_cloud
source_image = Image.open('./images/expression.gif')
target = Image.open('./images/first_frame.png')
target_model = optimization_one_image(
300, # Can be changed to multiple images
target,
lambda_alpha=45,
lambda_delta=15,
lr=.128)
alpha = target_model.alpha
for i, frame in enumerate(ImageSequence.Iterator(source_image)):
frame = frame.convert('RGB')
model = optimization_one_image(300, frame, lr=.128)
points = get_face_point_cloud(model.p, alpha, model.delta).view((-1, 3))
mesh = Mesh(vertices=points.detach().numpy(),
colors=mean_tex,
triangles=triangles)
mesh_to_png("./results/expression/frame_{}.png".format(i), mesh)
scene.add(mesh, pose=np.eye(4))
scene.add(light, pose=np.eye(4))
# Added camera translated z_camera_translation in the 0z direction w.r.t. the origin
scene.add(camera, pose=[[ 1, 0, 0, 0],
[ 0, 1, 0, 0],
[ 0, 0, 1, z_camera_translation],
[ 0, 0, 0, 1]])
# render scene
r = pyrender.OffscreenRenderer(width, height)
color, _ = r.render(scene)
imsave(file_name, color)
if __name__ == '__main__':
mesh = Mesh(mean_shape, mean_tex, triangles)
mesh_to_png("debug.png", mesh)
mesh_to_png("debug1234.png", mesh)
=======
png = mesh.scene().save_image()
with open(file_name, 'wb') as f:
f.write(png)
if __name__ == '__main__':
mesh = Mesh(mean_shape, mean_tex, triangles)
mesh_to_png("debug.png", mesh)
>>>>>>> Stashed changes
mesh = trimesh.base.Trimesh(vertices=mesh.vertices,
faces=mesh.triangles,
vertex_colors=mesh.colors)
scene = mesh.scene()
png = scene.save_image(visible=True)
with open(file_name, 'wb') as f:
f.write(png)
#scene = mesh.scene()
#aux = trimesh.viewer.notebook.scene_to_html(scene)
if __name__ == '__main__':
#raise ValueError
print('shape of face_triangles: ', face_triangles.shape)
mesh = Mesh(G, color_mean, face_triangles)
mesh_to_png("sample.png", mesh)
# Rotate pointcloud
pointcloud = G
last_col = torch.ones((pointcloud.shape[0], 1))
pointcloud = torch.cat((pointcloud, last_col), 1)
thetas = torch.Tensor([0, 10, 0])
trans = torch.Tensor([0, 0, 0])
T = transformMatrix(thetas, trans)
rotated_pointcloud = (T @ pointcloud.t()).t()[:, :-1]
mesh = Mesh(rotated_pointcloud, color_mean, face_triangles)
mesh_to_png("sample_rotated.png", mesh)
from data_def import Mesh
from utils import load_data, mesh_to_png
(texture, identity, expression, triangles) = load_data()
# create mesh from mean data
mesh = Mesh(identity.mean, texture.mean, triangles)
png = mesh_to_png(mesh)
with open('results/debug.png', 'wb') as f:
f.write(png)
png = mesh.scene().save_image()
with open(file_name, 'wb') as f:
f.write(png)
return
if __name__ == '__main__':
# output: only mean_shape)
# mesh = Mesh(mean_shape, mean_tex, triangles)
# question 1 - multiple point clouds with several sampled alpha and beta
# because of unsolved rendering problem, I need to run this each time to produce an image
imagenr = 6
alpha = 0.0 * (np.ones(30))
delta = -0.6 * (np.ones(20))
# alpha = np.random.uniform(-1, 1, (30))
# delta = np.random.uniform(-1, 1, (20))
[_, pCexpression, mean_tex, triangles] = load_faces(alpha, delta)
mesh = Mesh(pCexpression, mean_tex, triangles)
filename = 'expr'+ str(imagenr) + '.png'
mesh_to_png(filename, mesh)
# output: all combined
mesh = Mesh(pCexpression, mean_tex, triangles)
mesh_to_png("pCexp.png", mesh)