def rigid_scan_2_mesh_alignment(scan, mesh, visualize=False): options = {'sparse_solver': lambda A, x: cg(A, x, maxiter=2000)[0]} options['disp'] = 1.0 options['delta_0'] = 0.1 options['e_3'] = 1e-4 s = ch.ones(1) r = ch.zeros(3) R = Rodrigues(r) t = ch.zeros(3) trafo_mesh = s*(R.dot(mesh.v.T)).T + t sampler = sample_from_mesh(scan, sample_type='vertices') s2m = ScanToMesh(scan, trafo_mesh, mesh.f, scan_sampler=sampler, signed=False, normalize=False) if visualize: #Visualization code mv = MeshViewer() mv.set_static_meshes([scan]) tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) def on_show(_): tmp_mesh = Mesh(trafo_mesh.r, mesh.f) tmp_mesh.set_vertex_colors('light sky blue') mv.set_dynamic_meshes([tmp_mesh]) else: def on_show(_): pass ch.minimize(fun={'dist': s2m, 's_reg': 100*(ch.abs(s)-s)}, x0=[s, r, t], callback=on_show, options=options) return s,Rodrigues(r),t
def fit_3D_mesh(target_3d_mesh_fname, model_fname, weights, show_fitting=True): ''' Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology) :param target_3d_mesh_fname: target 3D mesh filename :param model_fname: saved FLAME model :param weights: weights of the individual objective functions :return: a mesh with the fitting results ''' target_mesh = Mesh(filename=target_3d_mesh_fname) tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True) tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True) tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True) tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True) tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True) smpl = SMPL(model_fname) tf_model = tf.squeeze(smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1), tf.concat((tf_rot, tf_pose), axis=-1))) with tf.Session() as session: session.run(tf.global_variables_initializer()) mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v))) neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3])) jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6])) eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:])) shape_reg = tf.reduce_sum(tf.square(tf_shape)) exp_reg = tf.reduce_sum(tf.square(tf_exp)) # Optimize global transformation first vars = [tf_trans, tf_rot] loss = mesh_dist optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1}) print('Optimize rigid transformation') optimizer.minimize(session) # Optimize for the model parameters vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp] loss = weights['data'] * mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \ weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg optimizer = scipy_pt(loss=loss, var_list=vars, method='BFGS', options={'disp': 1}) print('Optimize model parameters') optimizer.minimize(session) print('Fitting done') if show_fitting: # Visualize fitting mv = MeshViewer() fitting_mesh = Mesh(session.run(tf_model), smpl.f) fitting_mesh.set_vertex_colors('light sky blue') mv.set_static_meshes([target_mesh, fitting_mesh]) six.moves.input('Press key to continue') return Mesh(session.run(tf_model), smpl.f)
def fit_lmk3d(target_3d_lmks, model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True): ''' Fit FLAME to 3D landmarks :param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix :param model_fname: saved Tensorflow FLAME model :param lmk_face_idx: face indices of the landmark embedding in the FLAME topology :param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology (i.e. weighting of the three vertices for the trinagle, the landmark is embedded in :param weights: weights of the individual objective functions :return: a mesh with the fitting results ''' tf_trans = tf.Variable(np.zeros((1,3)), name="trans", dtype=tf.float64, trainable=True) tf_rot = tf.Variable(np.zeros((1,3)), name="pose", dtype=tf.float64, trainable=True) tf_pose = tf.Variable(np.zeros((1,12)), name="pose", dtype=tf.float64, trainable=True) tf_shape = tf.Variable(np.zeros((1,300)), name="shape", dtype=tf.float64, trainable=True) tf_exp = tf.Variable(np.zeros((1,100)), name="expression", dtype=tf.float64, trainable=True) smpl = SMPL(model_fname) tf_model = tf.squeeze(smpl(tf_trans, tf.concat((tf_shape, tf_exp), axis=-1), tf.concat((tf_rot, tf_pose), axis=-1))) with tf.Session() as session: session.run(tf.global_variables_initializer()) lmks = tf_get_model_lmks(tf_model, smpl.f, lmk_face_idx, lmk_b_coords) lmk_dist = tf.reduce_sum(tf.square(1000 * tf.subtract(lmks, target_3d_lmks))) neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,:3])) jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,3:6])) eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[:,6:])) shape_reg = tf.reduce_sum(tf.square(tf_shape)) exp_reg = tf.reduce_sum(tf.square(tf_exp)) # Optimize global transformation first vars = [tf_trans, tf_rot] loss = weights['lmk'] * lmk_dist optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6}) print('Optimize rigid transformation') optimizer.minimize(session) # Optimize for the model parameters vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp] loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \ weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1, 'ftol': 5e-6}) print('Optimize model parameters') optimizer.minimize(session) print('Fitting done') if show_fitting: # Visualize landmark fitting mv = MeshViewer() mv.set_static_meshes(create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0])) mv.set_dynamic_meshes([Mesh(session.run(tf_model), smpl.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True) six.moves.input('Press key to continue') return Mesh(session.run(tf_model), smpl.f)
def visualize(model, self_intersection_ids, default_model, vertices, faces): radius = 0.01 from psbody.mesh.meshviewer import MeshViewer # from mesh.mesh.meshviewer import MeshViewer mv = MeshViewer(window_width=800, window_height=800) mesh = Mesh(v=default_model.r, f=default_model.f) mesh.f = [] if self_intersection_ids == []: return mv.set_static_meshes([mesh, meanSphere], blocking=True) else: m = Mesh(v=model.v, f=model.f, vc='SeaGreen') m.vc[m.f[faces][self_intersection_ids].flatten()] = ones(1) mv.set_static_meshes([m], blocking=True) raw_input('Press Enter to exit')
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0]))))) for i in range(len(results)) ] result = ch.dstack(results2) return result, results_global def compute_r(self): return self.v.r def compute_dr_wrt(self, wrt): if wrt is not self.trans and wrt is not self.betas and wrt is not self.pose and wrt is not self.v_personal and wrt is not self.v_template: return None return self.v.dr_wrt(wrt) if __name__ == '__main__': from utils.smpl_paths import SmplPaths dp = SmplPaths(gender='neutral') smpl = Smpl(dp.get_smpl_file()) from psbody.mesh.meshviewer import MeshViewer from psbody.mesh import Mesh mv = MeshViewer() mv.set_static_meshes([Mesh(smpl.r, smpl.f)]) input("Press Enter to continue...")
def fit_3D_mesh(target_3d_mesh_fname, template_fname, tf_model_fname, weights, show_fitting=True): ''' Fit FLAME to 3D mesh in correspondence to the FLAME mesh (i.e. same number of vertices, same mesh topology) :param target_3d_mesh_fname: target 3D mesh filename :param template_fname: template mesh in FLAME topology (only the face information are used) :param tf_model_fname: saved Tensorflow FLAME model :param weights: weights of the individual objective functions :return: a mesh with the fitting results ''' target_mesh = Mesh(filename=target_3d_mesh_fname) template_mesh = Mesh(filename=template_fname) if target_mesh.v.shape[0] != template_mesh.v.shape[0]: print('Target mesh does not have the same number of vertices') return saver = tf.train.import_meta_graph(tf_model_fname + '.meta') graph = tf.get_default_graph() tf_model = graph.get_tensor_by_name(u'vertices:0') with tf.Session() as session: saver.restore(session, tf_model_fname) # Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable # tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0] tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0] tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0] tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0] tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0] tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0] mesh_dist = tf.reduce_sum(tf.square(tf.subtract(tf_model, target_mesh.v))) neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3])) jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6])) eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:])) shape_reg = tf.reduce_sum(tf.square(tf_shape)) exp_reg = tf.reduce_sum(tf.square(tf_exp)) # Optimize global transformation first vars = [tf_trans, tf_rot] loss = mesh_dist optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1}) print('Optimize rigid transformation') optimizer.minimize(session) # Optimize for the model parameters vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp] loss = mesh_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \ weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={'disp': 1}) print('Optimize model parameters') optimizer.minimize(session) print('Fitting done') if show_fitting: # Visualize fitting mv = MeshViewer() fitting_mesh = Mesh(session.run(tf_model), template_mesh.f) fitting_mesh.set_vertex_colors('light sky blue') mv.set_static_meshes([target_mesh, fitting_mesh]) raw_input('Press key to continue') return Mesh(session.run(tf_model), template_mesh.f)
def fit_lmk3d(target_3d_lmks, template_fname, tf_model_fname, lmk_face_idx, lmk_b_coords, weights, show_fitting=True): ''' Fit FLAME to 3D landmarks :param target_3d_lmks: target 3D landmarks provided as (num_lmks x 3) matrix :param template_fname: template mesh in FLAME topology (only the face information are used) :param tf_model_fname: saved Tensorflow FLAME model :param lmk_face_idx: face indices of the landmark embedding in the FLAME topology :param lmk_b_coords: barycentric coordinates of the landmark embedding in the FLAME topology (i.e. weighting of the three vertices for the trinagle, the landmark is embedded in :param weights: weights of the individual objective functions :return: a mesh with the fitting results ''' template_mesh = Mesh(filename=template_fname) saver = tf.train.import_meta_graph(tf_model_fname + '.meta') graph = tf.get_default_graph() tf_model = graph.get_tensor_by_name(u'vertices:0') with tf.Session() as session: saver.restore(session, tf_model_fname) # Workaround as existing tf.Variable cannot be retrieved back with tf.get_variable # tf_v_template = [x for x in tf.trainable_variables() if 'v_template' in x.name][0] tf_trans = [x for x in tf.trainable_variables() if 'trans' in x.name][0] tf_rot = [x for x in tf.trainable_variables() if 'rot' in x.name][0] tf_pose = [x for x in tf.trainable_variables() if 'pose' in x.name][0] tf_shape = [x for x in tf.trainable_variables() if 'shape' in x.name][0] tf_exp = [x for x in tf.trainable_variables() if 'exp' in x.name][0] lmks = tf_get_model_lmks(tf_model, template_mesh, lmk_face_idx, lmk_b_coords) lmk_dist = tf.reduce_sum( tf.square(1000 * tf.subtract(lmks, target_3d_lmks))) neck_pose_reg = tf.reduce_sum(tf.square(tf_pose[:3])) jaw_pose_reg = tf.reduce_sum(tf.square(tf_pose[3:6])) eyeballs_pose_reg = tf.reduce_sum(tf.square(tf_pose[6:])) shape_reg = tf.reduce_sum(tf.square(tf_shape)) exp_reg = tf.reduce_sum(tf.square(tf_exp)) # Optimize global transformation first vars = [tf_trans, tf_rot] loss = weights['lmk'] * lmk_dist optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={ 'disp': 1, 'ftol': 5e-6 }) print('Optimize rigid transformation') optimizer.minimize(session) # Optimize for the model parameters vars = [tf_trans, tf_rot, tf_pose, tf_shape, tf_exp] loss = weights['lmk'] * lmk_dist + weights['shape'] * shape_reg + weights['expr'] * exp_reg + \ weights['neck_pose'] * neck_pose_reg + weights['jaw_pose'] * jaw_pose_reg + weights['eyeballs_pose'] * eyeballs_pose_reg optimizer = scipy_pt(loss=loss, var_list=vars, method='L-BFGS-B', options={ 'disp': 1, 'ftol': 5e-6 }) print('Optimize model parameters') optimizer.minimize(session) print('Fitting done') if show_fitting: # Visualize landmark fitting mv = MeshViewer() mv.set_static_meshes( create_lmk_spheres(target_3d_lmks, 0.001, [255.0, 0.0, 0.0])) mv.set_dynamic_meshes( [Mesh(session.run(tf_model), template_mesh.f)] + create_lmk_spheres(session.run(lmks), 0.001, [0.0, 0.0, 255.0]), blocking=True) six.moves.input('Press key to continue') return Mesh(session.run(tf_model), template_mesh.f)