def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False):
options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]}
options['disp'] = 1.0
options['delta_0'] = 0.1
options['e_3'] = 1e-4
s = ch.ones(1)
r = ch.zeros(3)
R = Rodrigues(r)
t = ch.zeros(3)
trafo_mesh = s*(R.dot(mesh.v.T)).T + t
sampler = sample_from_mesh(scan, sample_type='vertices')
s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False)
if visualize:
#Visualization code
mv = MeshViewer()
mv.set_static_meshes([scan])
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
def on_show(_):
tmp_mesh = Mesh(trafo_mesh.r, mesh.f)
tmp_mesh.set_vertex_colors('light sky blue')
mv.set_dynamic_meshes([tmp_mesh])
else:
def on_show(_):
pass
ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options)
return s,Rodrigues(r),t
def fit_3D_mesh(target_3d_mesh_fname, model_fname, weights, show_fitting=True):
'''
Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology)
:param target_3d_mesh_fname: target 3D mesh filename
:param model_fname: saved FLAME model
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
target_mesh = Mesh(filename=target_3d_mesh_fname)
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = mesh_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['data'] * mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize fitting
mv = MeshViewer()
fitting_mesh = Mesh(session.run(tf_model), smpl.f)
fitting_mesh.set_vertex_colors('light sky blue')
mv.set_static_meshes([target_mesh, fitting_mesh])
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True)
tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True)
tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True)
tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True)
tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True)
smpl = SMPL(model_fname)
tf_model = tf.squeeze(smpl(tf_trans,
tf.concat((tf_shape, tf_exp), axis=-1),
tf.concat((tf_rot, tf_pose), axis=-1)))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords)
lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), smpl.f)
def visualize(model, self_intersection_ids, default_model, vertices, faces):
radius = 0.01
from psbody.mesh.meshviewer import MeshViewer
# from mesh.mesh.meshviewer import MeshViewer
mv = MeshViewer(window_width=800, window_height=800)
mesh = Mesh(v=default_model.r, f=default_model.f)
mesh.f = []
if self_intersection_ids == []:
return
mv.set_static_meshes([mesh, meanSphere], blocking=True)
else:
m = Mesh(v=model.v, f=model.f, vc='SeaGreen')
m.vc[m.f[faces][self_intersection_ids].flatten()] = ones(1)
mv.set_static_meshes([m], blocking=True)
raw_input('Press Enter to exit')
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0])))))
for i in range(len(results))
]
result = ch.dstack(results2)
return result, results_global
def compute_r(self):
return self.v.r
def compute_dr_wrt(self, wrt):
if wrt is not self.trans and wrt is not self.betas and wrt is not self.pose and wrt is not self.v_personal and wrt is not self.v_template:
return None
return self.v.dr_wrt(wrt)
if __name__ == '__main__':
from utils.smpl_paths import SmplPaths
dp = SmplPaths(gender='neutral')
smpl = Smpl(dp.get_smpl_file())
from psbody.mesh.meshviewer import MeshViewer
from psbody.mesh import Mesh
mv = MeshViewer()
mv.set_static_meshes([Mesh(smpl.r, smpl.f)])
input("Press Enter to continue...")
def fit_3D_mesh(target_3d_mesh_fname, template_fname, tf_model_fname, weights, show_fitting=True):
'''
Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology)
:param target_3d_mesh_fname: target 3D mesh filename
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
target_mesh = Mesh(filename=target_3d_mesh_fname)
template_mesh = Mesh(filename=template_fname)
if target_mesh.v.shape[0] != template_mesh.v.shape[0]:
print('Target mesh does not have the same number of vertices')
return
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = mesh_dist
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize fitting
mv = MeshViewer()
fitting_mesh = Mesh(session.run(tf_model), template_mesh.f)
fitting_mesh.set_vertex_colors('light sky blue')
mv.set_static_meshes([target_mesh, fitting_mesh])
raw_input('Press key to continue')
return Mesh(session.run(tf_model), template_mesh.f)
def fit_lmk3d(target_3d_lmks,
template_fname,
tf_model_fname,
lmk_face_idx,
lmk_b_coords,
weights,
show_fitting=True):
'''
Fit FLAME to 3D landmarks
:param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix
:param template_fname: template mesh in FLAME topology (only the face information are used)
:param tf_model_fname: saved Tensorflow FLAME model
:param lmk_face_idx: face indices of the landmark embedding in the FLAME topology
:param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology
(i.e. weighting of the three vertices for the trinagle, the landmark is embedded in
:param weights: weights of the individual objective functions
:return: a mesh with the fitting results
'''
template_mesh = Mesh(filename=template_fname)
saver = tf.train.import_meta_graph(tf_model_fname + '.meta')
graph = tf.get_default_graph()
tf_model = graph.get_tensor_by_name(u'vertices:0')
with tf.Session() as session:
saver.restore(session, tf_model_fname)
# Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable
# tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0]
tf_trans = [x for x in tf.trainable_variables()
if 'trans' in x.name][0]
tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0]
tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0]
tf_shape = [x for x in tf.trainable_variables()
if 'shape' in x.name][0]
tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0]
lmks = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx,
lmk_b_coords)
lmk_dist = tf.reduce_sum(
tf.square(1000 * tf.subtract(lmks, target_3d_lmks)))
neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3]))
jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6]))
eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:]))
shape_reg = tf.reduce_sum(tf.square(tf_shape))
exp_reg = tf.reduce_sum(tf.square(tf_exp))
# Optimize global transformation first
vars = [tf_trans, tf_rot]
loss = weights['lmk'] * lmk_dist
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize rigid transformation')
optimizer.minimize(session)
# Optimize for the model parameters
vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp]
loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \
weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg
optimizer = scipy_pt(loss=loss,
var_list=vars,
method='L-BFGS-B',
options={
'disp': 1,
'ftol': 5e-6
})
print('Optimize model parameters')
optimizer.minimize(session)
print('Fitting done')
if show_fitting:
# Visualize landmark fitting
mv = MeshViewer()
mv.set_static_meshes(
create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0]))
mv.set_dynamic_meshes(
[Mesh(session.run(tf_model), template_mesh.f)] +
create_lmk_spheres(session.run(lmks), 0.001,
[0.0, 0.0, 255.0]),
blocking=True)
six.moves.input('Press key to continue')
return Mesh(session.run(tf_model), template_mesh.f)