def calculate_errors_4DMaja_real(path_fits, path_gt, model): # Parameters: # -------------- # path_fits: 'string' of the directory that contains your reconstructed meshes. # Meshes' filenames should be the same with the # corresponding ground truth meshes filenames + the suffix of # the mesh type (e.g <frame_number>.obj, <frame_number>.ply) # path_gt : 'string' of the full path of the ground truth mesh. # model : 'string' of the model template you are using. Should be one of the # following: 'LSFM', 'Basel', 'Surrey' # Returns: # -------------- # errors : python list that contains the error per vertex between the ground # truth mesh and a mean mesh calculated from your reconstructed meshes path_fits = Path(path_fits) path_gt = Path(path_gt) # load meshes' filenames filenames = [p.name for p in path_fits.glob('*')] filenames.sort() # load gt_mesh (it is olny one, maja's neautral face) gt_mesh = m3io.import_mesh(path_gt, texture_resolver=None) gt_mesh.landmarks['ibug49'] = PointCloud(gt_mesh.points[lms_indexes][19:]) errors = [0] # accumulate fits acc_points = np.zeros((gt_mesh.n_points, 3)) for i, filename in enumerate(print_progress(filenames)): fit_3d = m3io.import_mesh(path_fits / filename, texture_resolver=None) if model == 'Surrey': lms = face_ibug_49_to_face_ibug_49( PointCloud(fit_3d.points[eos.load_eos_low_res_lm_index()])) fit_3d = eos.upsample_eos_low_res_to_fw_no_texture(fit_3d) fit_3d.landmarks['ibug49'] = lms elif model == 'LSFM' or model == 'Basel': fit_3d.landmarks['ibug49'] = PointCloud( fit_3d.points[lms_indexes][19:]) else: print('Error: Not supported model template') return acc_points += fit_3d.points # create mean_fit_3d mean_fit_3d = TriMesh(acc_points / len(filenames), gt_mesh.trilist) mean_fit_3d.landmarks['ibug49'] = PointCloud( mean_fit_3d.points[lms_indexes][19:]) # calculate per vertex errors between the neutral gt_mesh and the mean_fit_3d gt_mesh, eval_mask = landmark_and_mask_gt_mesh(gt_mesh, distance=1) errors[0], _, _ = mask_align_and_calculate_dense_error( mean_fit_3d, gt_mesh, eval_mask) return errors
def menpo3d_non_rigid_icp(fitted_obj, gt_obj, fitted_imp_3d_points, gt_imp_3d_points, output_obj): import sys #sys.path.append("/user/HS204/m09113/scripts/menpo_playground/src/lib/python3.5/site-packages") #sys.path.append("/user/HS204/m09113/miniconda2/envs/menpo/lib/python2.7/site-packages/") from menpo3d.correspond import non_rigid_icp from menpo3d.io.output.base import export_mesh import menpo3d.io as m3io import menpo # try something # lm_weights = [5, 2, .5, 0, 0, 0, 0, 0] # default weights # lm_weights = [10, 8, 5, 3, 2, 0.5, 0, 0] lm_weights = [25, 20, 15, 10, 8, 5, 3, 1] # lm_weights = [2, 1, 0, 0, 0, 0, 0, 0] # lm_weights = [25, 20, 15, 10, 5, 2, 1, 0] # lm_weights = [100, 0, 0, 0, 0, 0, 0, 0] # lm_weights = [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] stiff_weights = [50, 20, 5, 2, 0.8, 0.5, 0.35, 0.2] # default weights # stiff_weights = [50, 20, 15, 10, 3, 1, 0.35, 0.2] # stiff_weights = [50, 40, 30, 20, 10, 8, 5, 2] # stiff_weights = [50, 20, 10, 5, 2, 1, 0.5, 0.2] # stiff_weights = [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] # load pointcloud surrey model as src src = m3io.import_mesh(fitted_obj) # load scan mesh as dest dest = m3io.import_mesh(gt_obj) #print('destination loaded') # add landmarks to mesh src.landmarks['myLM'] = menpo.shape.PointCloud(fitted_imp_3d_points) dest.landmarks['myLM'] = menpo.shape.PointCloud(gt_imp_3d_points) #print('landmarks loaded') # non rigid icp pointcloud as result #marc org result = non_rigid_icp(src, dest, eps=1e-3, landmark_group='myLM', stiffness_weights=stiff_weights, data_weights=None, landmark_weights=lm_weights, generate_instances=False, verbose=False) # export the result mesh export_mesh(result, output_obj, extension='.obj', overwrite=True)
def calculate_errors_3dMDLab(path_fits, path_gt, model): r""" Parameters ---------- path_fits: 'string' of the directory that contains your reconstructed meshes. Meshes' filenames should be the same with the corresponding images/ground truth meshes filenames + the suffix of the mesh type (e.g .obj, .ply) path_gt: 'string' of the directory that contains the ground truth meshes. model: 'string' of the model template you are using. Should be one of the following: 'LSFM', 'Basel', 'Surrey' Returns ------- errors: python list that contains the error per vertex for all meshes """ path_fits = Path(path_fits) path_gt = Path(path_gt) # load meshes' filenames filenames = [p.name for p in path_fits.glob('*')] filenames.sort() errors = [0] * len(filenames) for i, filename in enumerate(print_progress(filenames)): # load gt_mesh, fit_3d gt_mesh = m3io.import_mesh(path_gt / filename, texture_resolver=None) gt_mesh.landmarks['ibug49'] = PointCloud( gt_mesh.points[lms_indexes][19:]) fit_3d = m3io.import_mesh(path_fits / filename, texture_resolver=None) if model == 'Surrey': lms = face_ibug_49_to_face_ibug_49( PointCloud(fit_3d.points[load_eos_low_res_lm_index()])) fit_3d = upsample_eos_low_res_to_fw_no_texture(fit_3d) fit_3d.landmarks['ibug49'] = lms elif model == 'LSFM' or model == 'Basel': fit_3d.landmarks['ibug49'] = PointCloud( fit_3d.points[lms_indexes][19:]) else: print('Error: Not supported model template') return # calculate per vertex errors gt_mesh, eval_mask = landmark_and_mask_gt_mesh(gt_mesh, distance=1) errors[i], _, _ = mask_align_and_calculate_dense_error( fit_3d, gt_mesh, eval_mask) return errors
def test_custom_landmark_logic_bunny(): def f(path): lmark_dict = { 'no_nose': path.with_name('bunny_no_nose.ljson'), 'full_set': path.with_name('bunny.ljson') } return menpo.io.input.resolve_from_paths(lmark_dict) mesh = mio.import_mesh(mio.data_path_to('bunny.obj'), landmark_resolver=f) assert ('no_nose' in mesh.landmarks.group_labels) lms = mesh.landmarks['no_nose'] labels = {'reye', 'mouth', 'leye'} assert (len(set(lms.labels) - labels) == 0) assert_allclose(lms.with_labels('leye').points, bunny_leye, atol=1e-7) assert_allclose(lms.with_labels('reye').points, bunny_reye, atol=1e-7) assert_allclose(lms.with_labels('mouth').points, bunny_mouth, atol=1e-7) assert ('full_set' in mesh.landmarks.group_labels) lms = mesh.landmarks['full_set'] labels = {'reye', 'mouth', 'nose', 'leye'} assert (len(set(lms.labels) - labels) == 0) assert_allclose(lms.with_labels('leye').points, bunny_leye, atol=1e-7) assert_allclose(lms.with_labels('reye').points, bunny_reye, atol=1e-7) assert_allclose(lms.with_labels('nose').points, bunny_nose, atol=1e-7) assert_allclose(lms.with_labels('mouth').points, bunny_mouth, atol=1e-7)
def test_custom_landmark_logic_bunny(): def f(path): lmark_dict = { "no_nose": path.with_name("bunny_no_nose.ljson"), "full_set": path.with_name("bunny.ljson"), } return menpo.io.input.resolve_from_paths(lmark_dict) mesh = mio.import_mesh(mio.data_path_to("bunny.obj"), landmark_resolver=f) assert "no_nose" in mesh.landmarks.group_labels lms = mesh.landmarks["no_nose"] labels = {"reye", "mouth", "leye"} assert len(set(lms.labels) - labels) == 0 assert_allclose(lms.with_labels("leye").points, bunny_leye, atol=1e-7) assert_allclose(lms.with_labels("reye").points, bunny_reye, atol=1e-7) assert_allclose(lms.with_labels("mouth").points, bunny_mouth, atol=1e-7) assert "full_set" in mesh.landmarks.group_labels lms = mesh.landmarks["full_set"] labels = {"reye", "mouth", "nose", "leye"} assert len(set(lms.labels) - labels) == 0 assert_allclose(lms.with_labels("leye").points, bunny_leye, atol=1e-7) assert_allclose(lms.with_labels("reye").points, bunny_reye, atol=1e-7) assert_allclose(lms.with_labels("nose").points, bunny_nose, atol=1e-7) assert_allclose(lms.with_labels("mouth").points, bunny_mouth, atol=1e-7)
def import_mesh(path): if path.suffix == '.pkl' or path.suffix == '.gz': mesh = import_pickle(path) else: mesh = m3io.import_mesh(path) if hasattr(mesh, "texture") and mesh.texture.pixels.dtype != np.float64: mesh.texture.pixels = normalize_pixels_range(mesh.texture.pixels) return mesh
def load_eos_output(image_path, output_dir): image_stem = Path(image_path).stem mesh = m3io.import_mesh(Path(output_dir) / (image_stem + '.obj')) lms = face_ibug_49_to_face_ibug_49( PointCloud(mesh.points[load_eos_low_res_lm_index()])) mesh_upsampled = upsample_eos_low_res_to_fw(mesh) mesh_upsampled.landmarks['ibug49'] = lms return {'raw_fit': mesh, 'corresponded_fit': mesh_upsampled}
def __init__(self, batch_size=1): super().__init__(batch_size) from menpo.transform import Translation, scale_about_centre import menpo3d.io as m3dio self.name = 'LFPW' self.batch_size = batch_size self.root = Path('/vol/atlas/databases/lfpw/trainset') template = m3dio.import_mesh( '/vol/construct3dmm/regression/src/template.obj') template = Translation(-template.centre()).apply(template) self.template = scale_about_centre(template, 1. / 1000.).apply(template) self.image_extension = '.png' self.lms_extension = '.pts'
def __init__(self, batch_size=1): super().__init__(batch_size) from menpo.transform import Translation, scale_about_centre import menpo3d.io as m3dio self.name = 'FDDB' self.root = Path('/vol/atlas/databases/fddb_ibug') template = m3dio.import_mesh( '/vol/construct3dmm/regression/src/template.obj') template = Translation(-template.centre()).apply(template) self.template = scale_about_centre(template, 1. / 1000.).apply(template) pca_path = '/homes/gt108/Projects/ibugface/pose_settings/pca_params.pkl' self.eigenvectors, self.eigenvalues, self.h_mean, self.h_max = mio.import_pickle( pca_path) self.image_extension = '.jpg' self.lms_extension = '.ljson'
def test_custom_landmark_logic_bunny(): def f(mesh): return {"no_nose": mesh.path.with_name("bunny_no_nose.ljson"), "full_set": mesh.path.with_name("bunny.ljson")} mesh = mio.import_mesh(mio.data_path_to("bunny.obj"), landmark_resolver=f) assert "no_nose" in mesh.landmarks.group_labels lms = mesh.landmarks["no_nose"] labels = {"reye", "mouth", "leye"} assert len(set(lms.labels) - labels) == 0 assert_allclose(lms["leye"].points, bunny_leye, atol=1e-7) assert_allclose(lms["reye"].points, bunny_reye, atol=1e-7) assert_allclose(lms["mouth"].points, bunny_mouth, atol=1e-7) assert "full_set" in mesh.landmarks.group_labels lms = mesh.landmarks["full_set"] labels = {"reye", "mouth", "nose", "leye"} assert len(set(lms.labels) - labels) == 0 assert_allclose(lms["leye"].points, bunny_leye, atol=1e-7) assert_allclose(lms["reye"].points, bunny_reye, atol=1e-7) assert_allclose(lms["nose"].points, bunny_nose, atol=1e-7) assert_allclose(lms["mouth"].points, bunny_mouth, atol=1e-7)
def import_mesh(path, hasTexture=False, landmark_type='ibug68'): if path.suffix == '.pkl' or path.suffix == '.gz': mesh = import_pickle(path) else: mesh = m3io.import_mesh(path) if hasTexture: if mesh.texture.pixels.dtype != np.float64: mesh.texture.pixels = normalize_pixels_range(mesh.texture.pixels) else: landmark = [] count = 0 with open(str(path) + '.landmark') as pp_file: pp_file = csv.reader(pp_file, delimiter=' ') for row in pp_file: count = count + 1 if landmark_type == 'ibug100': if count >= 1 and count <= 100: landmark.append( [float(row[0]), float(row[1]), float(row[2])]) if landmark_type == 'ibug68': if count < 69: landmark.append( [float(row[0]), float(row[1]), float(row[2])]) if landmark_type == 'ibugEar': if count >= 78 and count <= 88: landmark.append( [float(row[0]), float(row[1]), float(row[2])]) if count >= 90 and count <= 100: landmark.append( [float(row[0]), float(row[1]), float(row[2])]) mesh.landmarks[landmark_type] = PointCloud(np.array(landmark)) return mesh
def test_custom_landmark_logic_bunny(): def f(path): return { 'no_nose': path.with_name('bunny_no_nose.ljson'), 'full_set': path.with_name('bunny.ljson') } mesh = mio.import_mesh(mio.data_path_to('bunny.obj'), landmark_resolver=f) assert('no_nose' in mesh.landmarks.group_labels) lms = mesh.landmarks['no_nose'] labels = {'reye', 'mouth', 'leye'} assert(len(set(lms.labels) - labels) == 0) assert_allclose(lms['leye'].points, bunny_leye, atol=1e-7) assert_allclose(lms['reye'].points, bunny_reye, atol=1e-7) assert_allclose(lms['mouth'].points, bunny_mouth, atol=1e-7) assert('full_set' in mesh.landmarks.group_labels) lms = mesh.landmarks['full_set'] labels = {'reye', 'mouth', 'nose', 'leye'} assert(len(set(lms.labels) - labels) == 0) assert_allclose(lms['leye'].points, bunny_leye, atol=1e-7) assert_allclose(lms['reye'].points, bunny_reye, atol=1e-7) assert_allclose(lms['nose'].points, bunny_nose, atol=1e-7) assert_allclose(lms['mouth'].points, bunny_mouth, atol=1e-7)
def test_custom_landmark_logic_bunny(): def f(mesh): return { 'no_nose': mesh.path.with_name('bunny_no_nose.ljson'), 'full_set': mesh.path.with_name('bunny.ljson') } mesh = mio.import_mesh(mio.data_path_to('bunny.obj'), landmark_resolver=f) assert ('no_nose' in mesh.landmarks.group_labels) lms = mesh.landmarks['no_nose'] labels = {'reye', 'mouth', 'leye'} assert (len(set(lms.labels) - labels) == 0) assert_allclose(lms['leye'].points, bunny_leye, atol=1e-7) assert_allclose(lms['reye'].points, bunny_reye, atol=1e-7) assert_allclose(lms['mouth'].points, bunny_mouth, atol=1e-7) assert ('full_set' in mesh.landmarks.group_labels) lms = mesh.landmarks['full_set'] labels = {'reye', 'mouth', 'nose', 'leye'} assert (len(set(lms.labels) - labels) == 0) assert_allclose(lms['leye'].points, bunny_leye, atol=1e-7) assert_allclose(lms['reye'].points, bunny_reye, atol=1e-7) assert_allclose(lms['nose'].points, bunny_nose, atol=1e-7) assert_allclose(lms['mouth'].points, bunny_mouth, atol=1e-7)
def test_custom_landmark_logic_None_bunny(): def f(mesh): return None mesh = mio.import_mesh(mio.data_path_to('bunny.obj'), landmark_resolver=f) assert (mesh.landmarks.n_groups == 0)
def test_custom_landmark_logic_None_bunny(): def f(mesh): return None mesh = mio.import_mesh(mio.data_path_to('bunny.obj'), landmark_resolver=f) assert(mesh.landmarks.n_groups == 0)
def main(): test_data_directory = ('./test_data_face/render') # load obj face_mesh = m3io.import_mesh('./test_data_face/mesh.obj') texture_index = (face_mesh.tcoords.points[:, ::-1] * face_mesh.texture.shape).astype(np.int32) vertex_color = face_mesh.texture.pixels[:, 1 - texture_index[:, 0], texture_index[:, 1]].T tf.reset_default_graph() # Set up a basic cube centered at the origin, with vertex normals pointing # outwards along the line from the origin to the cube vertices: face_vertices = tf.constant(face_mesh.points, dtype=tf.float32) face_normals = tf.nn.l2_normalize(face_vertices, dim=1) face_triangles = tf.constant(face_mesh.trilist, dtype=tf.int32) # testRendersSimpleCube: """Renders a simple cube to test the full forward pass. Verifies the functionality of both the custom kernel and the python wrapper. """ n_randering = 16 model_transforms = camera_utils.euler_matrices( tf.random_uniform([n_randering, 3]) * np.pi / 2 - np.pi / 4.)[:, :3, :3] vertices_world_space = tf.matmul(tf.stack( [face_vertices for _ in range(n_randering)]), model_transforms, transpose_b=True) normals_world_space = tf.matmul(tf.stack( [face_normals for _ in range(n_randering)]), model_transforms, transpose_b=True) # camera position: eye = tf.constant(n_randering * [[0.0, 0.0, 6.0]], dtype=tf.float32) center = tf.constant(n_randering * [[0.0, 0.0, 0.0]], dtype=tf.float32) world_up = tf.constant(n_randering * [[0.0, 1.0, 0.0]], dtype=tf.float32) ambient_colors = tf.constant(n_randering * [[0.2, 0.2, 0.2]], dtype=tf.float32) image_width = 256 image_height = 256 light_positions = tf.constant(n_randering * [[[6.0, 6.0, 6.0], [-6.0, -6.0, 6.0]]]) light_intensities = tf.ones([n_randering, 1, 3], dtype=tf.float32) vertex_diffuse_colors = tf.constant(np.stack( [vertex_color for _ in range(n_randering)]), dtype=tf.float32) rendered = mesh_renderer.mesh_renderer( vertices_world_space, triangles=face_triangles, normals=normals_world_space, diffuse_colors=vertex_diffuse_colors, camera_position=eye, camera_lookat=center, camera_up=world_up, light_positions=light_positions, light_intensities=light_intensities, image_width=image_width, image_height=image_height, ambient_color=ambient_colors) image_id = 0 with tf.Session() as sess: fps_list = [] while (image_id < 100): start_time = time.time() images = sess.run(rendered, feed_dict={}) for image in images: target_image_name = 'Gray_face_%i.png' % image_id image_id += 1 baseline_image_path = os.path.join(test_data_directory, target_image_name) mio.export_image(Image.init_from_channels_at_back( image[..., :3].clip(0, 1)), baseline_image_path, overwrite=True) end_time = time.time() fps = n_randering / (end_time - start_time) fps_list.append(fps) if len(fps_list) > 5: fps_list.pop(0) print(np.mean(fps_list))
def main(): # hyperparameters BATCH_SIZE = FLAGS.batch_size N_VERTICES = 5023 EMBEDING = 8 INPUT_CHANNEL = 6 if FLAGS.ref_model == 'lsfm_rgb' or FLAGS.ref_model.startswith( 'mein3d') else 3 LR = FLAGS.lr FILTERS = [16, 16, 16, 32] LOGDIR = FLAGS.logdir if 'model_' in FLAGS.logdir else "{}/model_{}".format( FLAGS.logdir, int(time.time())) # referece model if FLAGS.ref_model.startswith('lsfm') or FLAGS.ref_model == 'mein3d': N_VERTICES = 53215 shape_model = mio.import_pickle(FLAGS.meta_path + '/all_all_all.pkl') trilist = shape_model.instance([]).trilist EMBEDING = 128 FILTERS = [16, 32, 32, 64] elif FLAGS.ref_model == 'mein3dcrop': N_VERTICES = 28431 face_mean_crop = m3io.import_mesh(FLAGS.meta_path + '/face_mean_mesh_crop.obj') trilist = face_mean_crop.trilist EMBEDING = 256 FILTERS = [16, 32, 32, 64] elif FLAGS.ref_model == 'mein3dcropmesh': N_VERTICES = 28431 face_mean_crop = m3io.import_mesh(FLAGS.meta_path + '/face_mean_mesh_crop.obj') trilist = face_mean_crop.trilist EMBEDING = 128 FILTERS = [16, 32, 32, 64] INPUT_CHANNEL = 3 elif FLAGS.ref_model == 'coma': N_VERTICES = 5023 trilist = mio.import_pickle(FLAGS.meta_path + '/coma_f.pkl', encoding='latin1') elif FLAGS.ref_model == '4dfab': N_VERTICES = 2064 template_mesh = m3io.import_mesh( '/vol/atlas/homes/Shiyang/CVPR18/code/animation/COMA/coma/data/afm_l1_cropped_final.obj' ) trilist = template_mesh.trilist else: raise Exception('Undefined ref_model: {}'.format(FLAGS.ref_model)) graph_laplacians, downsampling_matrices, upsamling_matrices, adj_matrices = mio.import_pickle( FLAGS.meta_path + '/{}_LDUA.pkl'.format(FLAGS.ref_model), encoding='latin1') def build_data(): class MeshRandomSample(dm.utils.Sequence): def __init__(self, batch_size=BATCH_SIZE): self.batch_size = batch_size self.size = 10000 // BATCH_SIZE super().__init__() def __len__(self): return self.size def __getitem__(self, idx): batch_sample_mesh = np.array([ shape_model.instance(np.random.sample([50]) * 3, normalized_weights=True).points for _ in range(self.batch_size) ]) return [batch_sample_mesh], [batch_sample_mesh] class NumpyMesh(dm.utils.Sequence): def __init__(self, fp, batch_size=BATCH_SIZE, scale=1, normalize=False): self.train_mesh = np.load(fp) if normalize: self.train_mesh /= np.max( np.abs([self.train_mesh.max(), self.train_mesh.min()])) + 1 self.batch_size = batch_size self.size = self.train_mesh.shape[0] self.indexes = list(range(self.size)) self.scale = scale np.random.shuffle(self.indexes) super().__init__() def __len__(self): return self.size // self.batch_size def __getitem__(self, idx): indexes = self.indexes[idx * self.batch_size:(idx + 1) * self.batch_size] batch_sample_mesh = np.array( [self.train_mesh[i] * self.scale for i in indexes]) return [batch_sample_mesh], [batch_sample_mesh] def on_epoch_end(self, *args, **kwargs): np.random.shuffle(self.indexes) return super().on_epoch_end() class H5Mesh(dm.utils.Sequence): def __init__(self, fp, dataset, batch_size=BATCH_SIZE, with_rgb=True): self.train_mesh = h5py.File(fp, 'r')[dataset] self.batch_size = batch_size self.size = self.train_mesh.len() self.indexes = list(range(self.size)) self.with_rgb = with_rgb np.random.shuffle(self.indexes) super().__init__() def __len__(self): return self.size // self.batch_size def __getitem__(self, idx): batch_sample_mesh = np.array([ self.train_mesh[self.indexes[i]] for i in range(idx * self.batch_size, idx * self.batch_size + self.batch_size) ]) if not self.with_rgb: batch_sample_mesh = batch_sample_mesh[..., :3] return [batch_sample_mesh], [batch_sample_mesh] def on_epoch_end(self, *args, **kwargs): np.random.shuffle(self.indexes) return super().on_epoch_end() if FLAGS.ref_model == 'lsfm': return MeshRandomSample(batch_size=BATCH_SIZE) elif FLAGS.ref_model == 'lsfm_rgb': return H5Mesh(FLAGS.meta_path + '/../lsfm_texture_train.h5', 'lsfm_colour', batch_size=BATCH_SIZE) elif FLAGS.ref_model == 'mein3d': return H5Mesh(FLAGS.meta_path + '/../mein3d.h5', 'colour_mesh', batch_size=BATCH_SIZE) elif FLAGS.ref_model == 'mein3dcrop': return H5Mesh(FLAGS.meta_path + '/../mein3dcrop.h5', 'colour_mesh', batch_size=BATCH_SIZE) elif FLAGS.ref_model == 'mein3dcropmesh': return H5Mesh(FLAGS.meta_path + '/../mein3dcropmesh.h5', 'mesh', batch_size=BATCH_SIZE) elif FLAGS.ref_model == 'coma': return NumpyMesh( '/homes/yz4009/wd/gitdev/coma/data/bareteeth/train.npy', batch_size=BATCH_SIZE, scale=5) elif FLAGS.ref_model == '4dfab': return NumpyMesh( '/vol/atlas/homes/Shiyang/CVPR18/code/animation/COMA/coma/data/RECON4DFAB/AFM/train.npy', batch_size=BATCH_SIZE, normalize=True) return None def build_model(inputs_channels=3): input_mesh = dm.layers.Input(shape=[N_VERTICES, inputs_channels], name='input_mesh') mesh_embedding = dm.networks.MeshEncoder(input_mesh, EMBEDING, graph_laplacians, downsampling_matrices, filter_list=FILTERS) output_mesh = dm.networks.MeshDecoder(mesh_embedding, inputs_channels, graph_laplacians, adj_matrices, upsamling_matrices, polynomial_order=6, filter_list=FILTERS) # wrapping input and output mesh_ae = dm.DeepMachine(inputs=input_mesh, outputs=[output_mesh]) n_gpu = len(FLAGS.gpu.split(',')) if n_gpu > 1: mesh_ae = multi_gpu_model(mesh_ae, gpus=n_gpu) # compile model with optimizer mesh_ae.compile(optimizer=dm.optimizers.Adam(lr=LR), loss=['mae']) # ---------------- rendering layer ------------ mesh_to_render = dm.layers.Input(shape=[N_VERTICES, 3], name='mesh_to_render') mesh_to_render.set_shape([BATCH_SIZE, N_VERTICES, 3]) vertex_color = dm.layers.Input(shape=[N_VERTICES, 3], name='vertex_color') vertex_color.set_shape([BATCH_SIZE, N_VERTICES, 3]) # Build vertices and normals mesh_vertices = mesh_to_render mesh_vertices.set_shape([BATCH_SIZE, N_VERTICES, 3]) mesh_normals = tf.nn.l2_normalize(mesh_vertices, axis=2) mesh_normals.set_shape([BATCH_SIZE, N_VERTICES, 3]) # rendering output mesh_triangles = tf.constant(trilist, dtype=tf.int32) # camera position: eye = tf.constant(BATCH_SIZE * [[0.0, 0.0, -2.0]], dtype=tf.float32) center = tf.constant(BATCH_SIZE * [[0.0, 0.0, 0.0]], dtype=tf.float32) world_up = tf.constant(BATCH_SIZE * [[1.0, 0.0, 0.0]], dtype=tf.float32) ambient_colors = tf.constant(BATCH_SIZE * [[1., 1., 1.]], dtype=tf.float32) * 0.1 light_positions = tf.constant(BATCH_SIZE * [[[2.0, 2.0, 2.0]]]) * 3. light_intensities = tf.ones([BATCH_SIZE, 1, 3], dtype=tf.float32) render_mesh = dm.layers.Renderer( # image size image_width=256, image_height=256, # mesh definition triangles=mesh_triangles, normals=mesh_normals, # colour definition diffuse_colors=vertex_color, ambient_color=ambient_colors, # camera definition camera_position=eye, camera_lookat=center, camera_up=world_up, # light definition light_positions=light_positions, light_intensities=light_intensities, )(mesh_vertices) mesh_render = dm.DeepMachine(inputs=[mesh_to_render, vertex_color], outputs=[render_mesh]) # ---------------------- return mesh_ae, mesh_render def custom_summary(train_x, train_y, predict_y, epoch): def render_mesh(sample_mesh, res=256, scale=1): if sample_mesh.shape[-1] != 3: sample_colours = sample_mesh[..., 3:] else: sample_colours = np.ones_like(sample_mesh) * [0, 0, 1] sample_mesh = sample_mesh[..., :3] sample_mesh = Homogeneous( dm.utils.rotation_matrix(np.deg2rad(90), [0, 0, -1])).apply(sample_mesh) sample_mesh = ColouredTriMesh(sample_mesh * scale * res / 2 + res / 2, trilist=trilist, colours=sample_colours) sample_mesh = lambertian_shading(sample_mesh, ambient_colour=0) store_path = Path(LOGDIR) / str(epoch) if not store_path.exists(): store_path.mkdir() m3io.export_mesh(sample_mesh, store_path / '{}.obj'.format(time.time())) mesh_img = rasterize_mesh(sample_mesh, [res, res]) mesh_img = mesh_img.rotate_ccw_about_centre(180) return mesh_img.pixels_with_channels_at_back() def dr_render_mesh(sample_mesh): if sample_mesh.shape[-1] != 3: sample_colours = sample_mesh[..., 3:] else: sample_colours = np.ones_like(sample_mesh) * [0, 0, 1] sample_mesh = sample_mesh[..., :3] return renderer.predict([sample_mesh, sample_colours])[..., :3] return { 'target/mesh': np.array(list(map(render_mesh, train_y[0][:4]))), 'output/mesh': np.array(list(map(render_mesh, predict_y[0][:4]))), 'target/dr_mesh': dr_render_mesh(train_y[0]), 'output/dr_mesh': dr_render_mesh(predict_y[0]), } training_generator = build_data() auto_encoder, renderer = build_model(inputs_channels=INPUT_CHANNEL) results = auto_encoder.fit(training_generator, epochs=1000, lr_decay=FLAGS.lr_decay, logdir=LOGDIR, verbose=2, workers=FLAGS.no_thread, summary_ops=[custom_summary])
# lm_weights = [5, 2, .5, 0, 0, 0, 0, 0] # default weights # lm_weights = [10, 8, 5, 3, 2, 0.5, 0, 0] lm_weights = [25, 20, 15, 10, 8, 5, 3, 1] # lm_weights = [2, 1, 0, 0, 0, 0, 0, 0] # lm_weights = [25, 20, 15, 10, 5, 2, 1, 0] # lm_weights = [100, 0, 0, 0, 0, 0, 0, 0] # lm_weights = [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] stiff_weights = [50, 20, 5, 2, 0.8, 0.5, 0.35, 0.2] # default weights # stiff_weights = [50, 20, 15, 10, 3, 1, 0.35, 0.2] # stiff_weights = [50, 40, 30, 20, 10, 8, 5, 2] # stiff_weights = [50, 20, 10, 5, 2, 1, 0.5, 0.2] # stiff_weights = [1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000] # load pointcloud surrey model as src src = m3io.import_mesh(src_path) print('source loaded') # load scan mesh as dest #dest = m3io.import_mesh(dest_path) #print('destination loaded') # load landmark pointcloud as lm src_lm_file = open(src_lm_path, 'r') #dest_lm_file = open(dest_lm_path, 'r') src_lm = np.loadtxt(src_lm_file) #dest_lm = np.loadtxt(dest_lm_file) print (src_lm)
def main(): dm.K.clear_session() dm.K.set_learning_phase(1) #set learning phase # hyperparameters BATCH_SIZE = FLAGS.batch_size LR = FLAGS.lr LOGDIR = FLAGS.logdir if 'model_' in FLAGS.logdir else "{}/model_{}".format( FLAGS.logdir, int(time.time())) N_VERTICES = 28431 EMBEDING = 128 CAMERA_PARAM = 12 INPUT_SHAPE = 112 FILTERS = [16, 32, 32, 64] # globel constant n_gpu = len(FLAGS.gpu.split(',')) face_mean_crop = m3io.import_mesh(FLAGS.meta_path + '/face_mean_mesh_crop.obj') trilist = face_mean_crop.trilist graph_laplacians, downsampling_matrices, upsamling_matrices, adj_matrices = mio.import_pickle( FLAGS.meta_path + '/mein3dcrop_LDUA.pkl', encoding='latin1') def build_data(): class H5Mesh(dm.utils.Sequence): def __init__(self, fp, dataset, batch_size=BATCH_SIZE): self.train_mesh = h5py.File(fp, 'r')[dataset] self.batch_size = batch_size self.size = self.train_mesh.len() self.indexes = list(range(self.size)) np.random.shuffle(self.indexes) super().__init__() def __len__(self): return self.size // self.batch_size def __getitem__(self, idx): indexes = self.indexes[idx * self.batch_size: (idx + 1) * self.batch_size] batch_sample_mesh = np.array([ self.train_mesh[i] for i in indexes ]) return [batch_sample_mesh], [batch_sample_mesh] def on_epoch_end(self, *args, **kwargs): np.random.shuffle(self.indexes) return super().on_epoch_end() class ImageSequence(dm.utils.Sequence): def __init__(self, dirpath, batch_size=BATCH_SIZE): self.detection = pd.read_csv('/homes/yz4009/db/face/loose_landmark_test.csv') self.size = self.detection.shape[0] self.image_path = Path(dirpath) self.batch_size = BATCH_SIZE self.indexes = list(range(self.size)) np.random.shuffle(self.indexes) def on_epoch_end(self, *args, **kwargs): np.random.shuffle(self.indexes) def __len__(self): return self.size // self.batch_size def _preprocess(self, idx): name, *lms5pt = self.detection.loc[idx] lms5pt = PointCloud(np.array(lms5pt).reshape([-1,2])[:,::-1]) img = mio.import_image((self.image_path/name).with_suffix('.jpg')) cimg, _, _ = dm.utils.crop_image_bounding_box(img, lms5pt.bounding_box(), [112, 112], base=186) return cimg.pixels_with_channels_at_back() * 2 - 1 def __getitem__(self, idx): image_indexes = self.indexes[ idx * self.batch_size: (idx + 1) * self.batch_size] batch_img = [self._preprocess(i) for i in image_indexes] return [np.array(batch_img)], [np.array(batch_img)] return H5Mesh('/homes/yz4009/wd/gitdev/coma/data/mein3dcrop.h5', 'colour_mesh', batch_size=BATCH_SIZE), ImageSequence(FLAGS.dataset_path) def build_model(inputs_channels=6, n_gpu=n_gpu): # define components ## image encoder def build_img_encoder(): input_img = dm.layers.Input(shape=[INPUT_SHAPE, INPUT_SHAPE, 3], name='input_img') img_embedding = dm.networks.Encoder2D( input_img, EMBEDING + CAMERA_PARAM, depth=4, nf=64) mesh_rec_embeding = dm.layers.Lambda(lambda x: x[..., :EMBEDING])(img_embedding) cam_rec_embeding = dm.layers.Lambda(lambda x: dm.K.tanh(x[..., EMBEDING:]) * 3)(img_embedding) return dm.Model(input_img, [mesh_rec_embeding, cam_rec_embeding], name='image_encoder') ## mesh encoder def build_mesh_encoder(): input_mesh = dm.layers.Input(shape=[N_VERTICES, inputs_channels], name='input_mesh') mesh_embedding = dm.networks.MeshEncoder( input_mesh, EMBEDING, graph_laplacians, downsampling_matrices, filter_list=FILTERS) return dm.Model(input_mesh, mesh_embedding, name='mesh_encoder') ## common decoder def build_decoder(): input_embeding = dm.layers.Input(shape=[EMBEDING], name='input_embeding') output_mesh = dm.networks.MeshDecoder( input_embeding, inputs_channels, graph_laplacians, adj_matrices, upsamling_matrices, polynomial_order=6, filter_list=FILTERS) return dm.Model(input_embeding, output_mesh, name='decoder') ## renderer def build_renderer(mesh_vertices, vertex_color, cam_parameter): # mesh_vertices = dm.layers.Input(shape=[N_VERTICES, 3], name='mesh_vertices') mesh_vertices.set_shape([BATCH_SIZE, N_VERTICES, 3]) # vertex_color = dm.layers.Input(shape=[N_VERTICES, 3], name='vertex_color') vertex_color.set_shape([BATCH_SIZE, N_VERTICES, 3]) # cam_parameter = dm.layers.Input(shape=[CAMERA_PARAM], name='cam_parameter') cam_parameter.set_shape([BATCH_SIZE, CAMERA_PARAM]) # Build vertices and normals mesh_normals = tf.nn.l2_normalize(mesh_vertices, axis=2) # rendering output mesh_triangles = tf.constant(trilist, dtype=tf.int32) # camera position: eye = cam_parameter[...,:3] center = cam_parameter[...,3:6] world_up = cam_parameter[...,6:9] light_positions = cam_parameter[:,None,9:12] ambient_colors = tf.ones([BATCH_SIZE, 3], dtype=tf.float32) * 0.1 light_intensities = tf.ones([BATCH_SIZE, 1, 3], dtype=tf.float32) render_mesh = dm.layers.Renderer( # image size image_width=INPUT_SHAPE, image_height=INPUT_SHAPE, # mesh definition triangles=mesh_triangles, normals=mesh_normals, # colour definition diffuse_colors=vertex_color, ambient_color=ambient_colors, # camera definition camera_position=eye, camera_lookat=center, camera_up=world_up, # light definition light_positions=light_positions, light_intensities=light_intensities, )(mesh_vertices) render_mesh = dm.layers.Lambda(lambda x: x[..., :3])(render_mesh) return render_mesh # Mesh AE stream ## define inputs input_mesh_stream = dm.layers.Input(shape=[N_VERTICES, 6], name='input_mesh_stream') ## define components mesh_encoder_model = build_mesh_encoder() decoder_model = build_decoder() ## define connections output_mesh = decoder_model(mesh_encoder_model(input_mesh_stream)) mesh_ae_model = dm.DeepMachine( inputs=input_mesh_stream, outputs=output_mesh, name='MeshStream' ) ## multi gpu support if n_gpu > 1: mesh_ae_model = multi_gpu_model(mesh_ae_model, gpus=n_gpu) ## compile mesh stream mesh_ae_model.compile( optimizer=dm.optimizers.Adam(lr=LR), loss=['mae'] ) ## set trainable mesh_ae_model.trainable = False decoder_model.trainable = False mesh_encoder_model.trainable = False # Render Stream ## define inputs input_image_stream = dm.layers.Input(shape=[INPUT_SHAPE, INPUT_SHAPE, 3], name='input_image_stream') ## define components img_encoder_model = build_img_encoder() ## define connections rec_mesh_emb, rec_cam_emb = img_encoder_model(input_image_stream) mesh_with_colour = decoder_model(rec_mesh_emb) mesh_vert = dm.layers.Lambda(lambda x: x[..., :3])(mesh_with_colour) mesh_vert.set_shape([BATCH_SIZE, N_VERTICES, 3]) mesh_colour = dm.layers.Lambda(lambda x: x[..., 3:])(mesh_with_colour) mesh_colour.set_shape([BATCH_SIZE, N_VERTICES, 3]) rec_render = build_renderer( mesh_vert, mesh_colour, rec_cam_emb ) render_model = dm.DeepMachine( inputs=input_image_stream, outputs=[rec_render, mesh_with_colour], name='ImageStream' ) ## multi gpu support if n_gpu > 1: render_model = multi_gpu_model(render_model, gpus=n_gpu) ## compile render stream render_model.compile( optimizer=dm.optimizers.Adam(lr=LR), loss=['mae', dm.losses.dummy] ) return render_model, mesh_ae_model, img_encoder_model def train_op(models, data, i_epoch, i_batch, epoch_end, training_history=None, **kwargs): sess = dm.K.get_session() image_stream, mesh_stream, img_encoder = models [train_mesh, train_image], _ = dm.engine.training.generator_adapter(data) # ---------------------- # Train Mesh Stream # ---------------------- loss_mesh = mesh_stream.train_on_batch([train_mesh], [train_mesh]) # ------------------ # Train Render Stream # ------------------ loss_img = image_stream.train_on_batch([train_image], [train_image, train_mesh]) logs = dm.utils.Summary( { "losses/loss_mesh": loss_mesh, "losses/loss_img": loss_img[0], "learning_rate/mesh": mesh_stream.optimizer.lr.eval(sess), "learning_rate/img": image_stream.optimizer.lr.eval(sess), } ) if epoch_end: ae_mesh = mesh_stream.predict(train_mesh) rec_imgs, rec_mesh = image_stream.predict(train_image) _, cam_params = img_encoder.predict(train_image) logs.update_images({ 'image/input': train_image, 'image/render': rec_imgs, 'image/mesh': dm.utils.mesh.render_meshes(rec_mesh[:4], trilist, res=INPUT_SHAPE), 'mesh/input': dm.utils.mesh.render_meshes(train_mesh[:4], trilist, res=INPUT_SHAPE), 'mesh/ae': dm.utils.mesh.render_meshes(ae_mesh[:4], trilist, res=INPUT_SHAPE), }) logs.update_scalars( {'cam_params/{}'.format(idx_p): p for idx_p, p in enumerate(cam_params[0])} ) return logs # prepare data train_generator = dm.data.generator.MergeGenerators(*build_data()) train_queue = enqueue_generator( train_generator, workers=FLAGS.no_thread) # prepare model image_stream, mesh_stream, img_encoder = build_model() mesh_lr_decay = dm.callbacks.LearningRateScheduler( schedule=lambda epoch: LR * FLAGS.lr_decay ** epoch) mesh_lr_decay.set_model(mesh_stream) image_lr_decay = dm.callbacks.LearningRateScheduler( schedule=lambda epoch: LR * FLAGS.lr_decay ** epoch) image_lr_decay.set_model(image_stream) # training history = dm.engine.training.train_monitor( [image_stream, mesh_stream, img_encoder], train_queue, train_op, epochs=200, step_per_epoch=len(train_generator), callbacks=[ train_generator, mesh_lr_decay, image_lr_decay ], verbose=FLAGS.verbose, logdir=LOGDIR, )
def save_customize_template_from_basel(path): import menpo3d.io as m3io import lsfm template = m3io.import_mesh(path) lsfm.landmark.landmark_template(template,verbose=True) save_template(template,overwrite=True)