def pinhole_demo(rotation=None, translation=None, save_fname="pinhole", idx=0):
# assignment: section 3
bfm = h5py.File(MODELS_PATH + BFM_FNAME, "r")
bfm_params, color, triangles = utils.read_bfm(bfm)
lms = utils.read_landmarks(MODELS_PATH + LM_FNAME) # landmark annotations
rotation = torch.tensor([0., 0., 0.]) if rotation is None else torch.tensor(rotation)
translation = torch.tensor([0., 0., -500.]) if translation is None else torch.tensor(translation)
G = morph.compute_G(bfm_params)
G_transformed = pinhole.transform(G, rotation, translation)
G_pinhole = pinhole.camera_model(G, rotation, translation)
save_obj(OBJ_3D_PATH + save_fname + str(idx) + "_3d.obj", G_transformed, color, triangles)
save_obj(OBJ_2D_PATH + save_fname + str(idx) + "_2d.obj", G_pinhole, color, triangles)
print("Rendering...")
img_2d = utils.get_image(G_pinhole, color, triangles) # render img
img_lm = utils.get_landmarks(G_pinhole[:, :2], lms) # landmark coords
utils.show_face(img_2d)
utils.flip_y()
plt.savefig(PINHOLE_PATH + save_fname + str(idx) + ".pdf")
utils.show_landmarks(img_lm, indices=True) # overlays landmarks on image
plt.savefig(PINHOLE_PATH + save_fname + str(idx) + "_lm.pdf")
plt.close()
def estimate_params(bfm_all, lms, lm_real, id_comps=30, exp_comps=20, reg_a=10., reg_d=10., h=480., w=640.,
steps=10000, lr=.1, threshold=1., R_init=None):
bfm_params, color, triangles = bfm_all
# define parameters to be optimized
alpha, delta = morph.sample_alpha_delta(id_comps=id_comps, exp_comps=exp_comps)
alpha, delta = Variable(alpha, requires_grad=True), Variable(delta, requires_grad=True)
rotation = Variable(torch.rand(3)*2-1, requires_grad=True) if R_init is None else Variable(R_init, requires_grad=True)
translation = Variable(torch.cat((torch.rand(2)*2-1, torch.tensor([-500.]))), requires_grad=True)
optimizer = torch.optim.Adam([alpha, delta, rotation, translation], lr=lr)
losses = []
print("Optimizing...")
# optimize for the specified loss function
for i in range(steps):
optimizer.zero_grad()
G = morph.compute_G(bfm_params, alpha=alpha, delta=delta)
G_pinhole = pinhole.camera_model(G, rotation, translation, h=h, w=w)
lm_pred = utils.get_landmarks(G_pinhole[:, :2], lms)
loss = loss_fn(lm_pred, lm_real, alpha, delta, reg_a=reg_a, reg_d=reg_d)
loss.backward()
optimizer.step()
losses.append(loss.item())
if i > 0 and losses[-2] - loss < threshold:
# stop if difference with prev loss is less than threshold
print(f"... stopping early at iteration {i}")
break
return alpha.detach(), delta.detach(), rotation.detach(), translation.detach(), losses
def multiple_demo(load_fnames, save_fname="multi", reg_a=10., reg_d=10.):
# assignment: section 6
bfm = h5py.File(MODELS_PATH + BFM_FNAME, "r")
bfm_params, color, triangles = utils.read_bfm(bfm)
lms = utils.read_landmarks(MODELS_PATH + LM_FNAME) # landmark annotations
N = len(load_fnames) # number of images to be loaded
imgs_real = [utils.read_image(IMAGE_PATH + fname) for fname in load_fnames] # load all images
hs = [np.size(img, 0) for img in imgs_real] # store all heights
ws = [np.size(img, 1) for img in imgs_real] # store all widths
lms_real = [torch.from_numpy(detect_landmark(img)) for img in imgs_real] # detect all ground truth landmarks
lms_real_flip = [utils.flip_ycoords(lms_real[i], H=hs[i]) for i in range(N)] # flip y axis
alpha, deltas, rotations, translations, loss = multiple.estimate_params((bfm_params, color, triangles),
lms, lms_real_flip, hs=hs, ws=ws,
reg_a=reg_a, reg_d=reg_d)
utils.save_loss(loss, save_fname=save_fname + "_loss.pdf")
# save results for each image
for i in range(N):
print(load_fnames[i] + ":") # print stats for each image (alpha is the same for each img)
utils.print_stats(alpha, deltas[i], rotations[i], translations[i])
G = morph.compute_G(bfm_params, alpha=alpha, delta=deltas[i])
G_transformed = pinhole.transform(G, rotations[i], translations[i])
G_pinhole = pinhole.camera_model(G, rotations[i], translations[i], h=hs[i], w=ws[i])
color = texture.get_color(imgs_real[i], G_pinhole[:, :2])
print("Rendering...")
img_pred = utils.get_image(G_pinhole, color, triangles, h=hs[i], w=ws[i])
utils.show_face(img_pred)
utils.flip_y()
plt.savefig(PINHOLE_PATH + save_fname + str(i) + ".pdf")
plt.close()
save_obj(OBJ_3D_PATH + save_fname + str(i) + "_3d.obj", G_transformed, color, triangles)
save_obj(OBJ_2D_PATH + save_fname + str(i) + "_2d.obj", G_pinhole, color, triangles)
lm_pred_flip = utils.get_landmarks(G_pinhole[:, :2], lms)
lm_pred = utils.flip_ycoords(lm_pred_flip, H=hs[i])
utils.show_face(imgs_real[i], white_background=False)
utils.show_landmarks(lms_real[i], indices=False, label="ground-truth")
try:
utils.show_landmarks(lm_pred, indices=False, label="model")
except TypeError:
print("... unable to show predicted landmarks")
plt.savefig(PINHOLE_PATH + save_fname + str(i) + "_lm.pdf")
plt.close()
def texture_demo(load_fname="yke_neutral.jpeg", save_fname="texture", reg_a=10., reg_d=10., idx=0):
# assignment: section 5
bfm = h5py.File(MODELS_PATH + BFM_FNAME, "r")
bfm_params, color, triangles = utils.read_bfm(bfm)
lms = utils.read_landmarks(MODELS_PATH + LM_FNAME) # landmark annotations
img_real = utils.read_image(IMAGE_PATH + load_fname) # load image of face we want to reconstruct
h, w, _ = np.shape(img_real)
lm_real = torch.from_numpy(detect_landmark(img_real)) # detect ground-truth landmarks
lm_real_flip = utils.flip_ycoords(lm_real, H=h) # flip y axis because img is upside down compared to pinhole output
alpha, delta, rotation, translation, loss = latent.estimate_params((bfm_params, color, triangles),
lms, lm_real_flip, h=h, w=w,
reg_a=reg_a, reg_d=reg_d)
utils.print_stats(alpha, delta, rotation, translation) # latent params statistics
utils.save_loss(loss, save_fname=save_fname + str(idx) + "_loss.pdf")
G = morph.compute_G(bfm_params, alpha=alpha, delta=delta)
G_pinhole = pinhole.camera_model(G, rotation, translation, h=h, w=w)
color = texture.get_color(img_real, G_pinhole[:, :2]) # obtain vertex colors from provided image
save_obj(OBJ_3D_PATH + save_fname + str(idx) + "_3d.obj", G, color, triangles)
save_obj(OBJ_2D_PATH + save_fname + str(idx) + "_2d.obj", G_pinhole, color, triangles)
print("Rendering...")
img_pred = utils.get_image(G_pinhole, color, triangles, h=h, w=w)
utils.show_face(img_pred)
utils.flip_y()
plt.savefig(PINHOLE_PATH + save_fname + str(idx) + ".pdf")
plt.close()
lm_pred_flip = utils.get_landmarks(G_pinhole[:, :2], lms)
lm_pred = utils.flip_ycoords(lm_pred_flip, H=h)
utils.show_face(img_real, white_background=False)
utils.show_landmarks(lm_real, indices=False, label="ground-truth")
try:
utils.show_landmarks(lm_pred, indices=False, label="model")
except TypeError:
print("... unable to show predicted landmarks")
plt.savefig(PINHOLE_PATH + save_fname + str(idx) + "_lm.pdf")
plt.close()
def estimate_params(bfm_all,
lms,
lms_real,
id_comps=30,
exp_comps=20,
reg_a=10.,
reg_d=10.,
hs=480.,
ws=640.,
steps=10000,
lr=.1,
threshold=1.):
bfm_params, color, triangles = bfm_all
lms_real = lms_real if isinstance(lms_real, list) else [lms_real]
hs = hs if isinstance(hs, list) else [hs]
ws = ws if isinstance(ws, list) else [ws]
N = len(lms_real)
deltas = []
for i in range(N):
alpha, delta = morph.sample_alpha_delta(id_comps=id_comps,
exp_comps=exp_comps)
deltas.append(delta)
alpha = Variable(alpha, requires_grad=True)
deltas = [Variable(delta, requires_grad=True) for delta in deltas]
rotations = [
Variable(torch.rand(3) * 2 - 1, requires_grad=True) for i in range(N)
]
translations = [
Variable(torch.cat((torch.rand(2) * 2 - 1, torch.tensor([-500.]))),
requires_grad=True) for i in range(N)
]
optimizer = torch.optim.Adam([alpha] + deltas + rotations + translations,
lr=lr)
losses = []
print("Optimizing...")
# optimize for the specified loss function
for i in range(steps):
optimizer.zero_grad()
Gs = [
morph.compute_G(bfm_params, alpha=alpha, delta=delta)
for delta in deltas
]
Gs_pinhole = [
pinhole.camera_model(Gs[i],
rotations[i],
translations[i],
h=hs[i],
w=ws[i]) for i in range(N)
]
lms_pred = [
utils.get_landmarks(G_pinhole[:, :2], lms)
for G_pinhole in Gs_pinhole
]
loss = loss_fn(lms_pred,
lms_real,
alpha,
deltas,
reg_a=reg_a,
reg_d=reg_d)
loss.backward()
optimizer.step()
losses.append(loss.item())
if i > 0 and losses[-2] - loss < threshold:
# stop if difference with prev loss is less than threshold
print(f"... stopping early at iteration {i}")
break
return alpha.detach(), [delta.detach() for delta in deltas], \
[rotation.detach() for rotation in rotations], \
[translation.detach() for translation in translations], \
losses