def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False):
options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]}
options['disp'] = 1.0
options['delta_0'] = 0.1
options['e_3'] = 1e-4
s = ch.ones(1)
r = ch.zeros(3)
R = Rodrigues(r)
t = ch.zeros(3)
trafo_mesh = s*(R.dot(mesh.v.T)).T + t
sampler = sample_from_mesh(scan, sample_type='vertices')
s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False)
if visualize:
#Visualization code
mv = MeshViewer()
mv.set_static_meshes([scan])
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
def on_show(_):
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
else:
def on_show(_):
pass
ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options)
return s,Rodrigues(r),t
def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def sample_texture(model_fname, texture_fname, num_samples, out_path):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
texture_model = np.load(texture_fname)
tex_dim = texture_model['tex_dir'].shape[-1]
tf_tex_params = tf.Variable(np.zeros((1,tex_dim)), name="pose", dtype=tf.float64, trainable=True)
tf_tex_mean = tf.Variable(np.reshape(texture_model['mean'], (1,-1)), name='tex_mean', dtype=tf.float64, trainable=False)
tf_tex_dir = tf.Variable(np.reshape(texture_model['tex_dir'], (-1, tex_dim)).T, name='tex_dir', dtype=tf.float64, trainable=False)
tf_tex = tf.add(tf_tex_mean, tf.matmul(tf_tex_params, tf_tex_dir)),
tf_tex = tf.reshape(tf_tex, (texture_model['tex_dir'].shape[0], texture_model['tex_dir'].shape[1], texture_model['tex_dir'].shape[2]))
tf_tex = tf.cast(tf.clip_by_value(tf_tex, 0.0, 255.0), tf.int64)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_tex = tf.assign(tf_tex_params, np.random.randn(tex_dim)[np.newaxis,:])
session.run([assign_tex])
v, tex = session.run([tf_model, tf_tex])
out_mesh = Mesh(v, smpl.f)
out_mesh.vt = texture_model['vt']
out_mesh.ft = texture_model['ft']
mv.set_dynamic_meshes([out_mesh], blocking=True)
key = six.moves.input('Press (s) to save sample, any other key to continue ')
if key == 's':
out_mesh_fname = os.path.join(out_path, 'tex_sample_%02d.obj' % (i+1))
out_tex_fname = out_mesh_fname.replace('obj', 'png')
cv2.imwrite(out_tex_fname, tex)
out_mesh.set_texture_image(out_tex_fname)
out_mesh.write_obj(out_mesh_fname)
def sample_FLAME(model_fname, num_samples, out_path, visualize, sample_VOCA_template=False):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param model_fname saved FLAME model
:param num_samples number of samples
:param out_path output path to save the generated templates (no templates are saved if path is empty)
:param visualize visualize samples
:param sample_VOCA_template sample template in 'zero pose' that can be used e.g. for speech-driven animation in VOCA
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
if visualize:
mv = MeshViewer()
for i in range(num_samples):
if sample_VOCA_template:
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
session.run([assign_shape])
out_fname = os.path.join(out_path, 'VOCA_template_%02d.ply' % (i+1))
else:
# assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:])
assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.02)
assign_shape = tf.assign(tf_shape, np.hstack((np.random.randn(100), np.zeros(200)))[np.newaxis,:])
assign_exp = tf.assign(tf_exp, np.hstack((0.5*np.random.randn(50), np.zeros(50)))[np.newaxis,:])
session.run([assign_rot, assign_pose, assign_shape, assign_exp])
out_fname = os.path.join(out_path, 'FLAME_sample_%02d.ply' % (i+1))
sample_mesh = Mesh(session.run(tf_model), smpl.f)
if visualize:
mv.set_dynamic_meshes([sample_mesh], blocking=True)
key = six.moves.input('Press (s) to save sample, any other key to continue ')
if key == 's':
sample_mesh.write_ply(out_fname)
else:
sample_mesh.write_ply(out_fname)
def sample_FLAME(template_fname, tf_model_fname, num_samples):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
mv = MeshViewer()
for i in range(num_samples):
assign_trans = tf.assign(tf_trans, np.random.randn(3))
assign_rot = tf.assign(tf_rot, np.random.randn(3) * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12) * 0.03)
assign_shape = tf.assign(tf_shape, np.random.randn(300) * 1.0)
assign_exp = tf.assign(tf_exp, np.random.randn(100) * 0.5)
session.run([
assign_trans, assign_rot, assign_pose, assign_shape, assign_exp
])
mv.set_dynamic_meshes(
[Mesh(session.run(tf_model), template_mesh.f)], blocking=True)
raw_input('Press key to continue')
def sample_FLAME(template_fname, model_fname, num_samples):
'''
Sample the FLAME model to demonstrate how to vary the model parameters.FLAME has parameters to
- model identity-dependent shape variations (paramters: shape),
- articulation of neck (paramters: pose[0:3]), jaw (paramters: pose[3:6]), and eyeballs (paramters: pose[6:12])
- model facial expressions, i.e. all expression motion that does not involve opening the mouth (paramters: exp)
- global translation (paramters: trans)
- global rotation (paramters: rot)
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param model_fname: saved FLAME model
'''
template_mesh = Mesh(filename=template_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mv = MeshViewer()
for i in range(num_samples):
assign_trans = tf.assign(tf_trans, np.random.randn(3)[np.newaxis,:])
assign_rot = tf.assign(tf_rot, np.random.randn(3)[np.newaxis,:] * 0.03)
assign_pose = tf.assign(tf_pose, np.random.randn(12)[np.newaxis,:] * 0.03)
assign_shape = tf.assign(tf_shape, np.random.randn(300)[np.newaxis,:] * 1.0)
assign_exp = tf.assign(tf_exp, np.random.randn(100)[np.newaxis,:] * 0.5)
session.run([assign_trans, assign_rot, assign_pose, assign_shape, assign_exp])
mv.set_dynamic_meshes([Mesh(session.run(tf_model), template_mesh.f)], blocking=True)
six.moves.input('Press key to continue')
def fit_lmk3d(target_3d_lmks,
template_fname,
tf_model_fname,
lmk_face_idx,
lmk_b_coords,
weights,
show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
lmks = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
lmk_dist = tf.reduce_sum(
tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(
create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes(
[Mesh(session.run(tf_model), template_mesh.f)] +
create_lmk_spheres(session.run(lmks), 0.001,
[0.0, 0.0, 255.0]),
blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), template_mesh.f)
args = parser.parse_args()
sequence_path = args.sequence_path
audio_fname = args.audio_fname
out_path = args.out_path
img_path = os.path.join(out_path, 'img')
if not os.path.exists(img_path):
os.makedirs(img_path)
mv = MeshViewer()
sequence_fnames = sorted(glob.glob(os.path.join(sequence_path, '*.obj')))
if len(sequence_fnames) == 0:
print('No meshes found')
# Render images
for frame_idx, mesh_fname in enumerate(sequence_fnames):
frame_mesh = Mesh(filename=mesh_fname)
mv.set_dynamic_meshes([frame_mesh], blocking=True)
img_fname = os.path.join(img_path, '%05d.png' % frame_idx)
mv.save_snapshot(img_fname)
# Encode images to video
cmd_audio = []
if os.path.exists(audio_fname):
cmd_audio += ['-i', audio_fname]
out_video_fname = os.path.join(out_path, 'video.mp4')
cmd = ['ffmpeg', '-framerate', '60', '-pattern_type', 'glob', '-i', os.path.join(img_path, '*.png')] + cmd_audio + [out_video_fname]
call(cmd)
