def test_mean_pointcloud_type(): points = np.array([[1, 2, 3], [1, 1, 1]]) trilist = np.array([0, 1, 2]) pcs = [TriMesh(points, trilist), TriMesh(points + 2, trilist)] mean_pc = mean_pointcloud(pcs) assert isinstance(mean_pc, TriMesh) assert_allclose(mean_pc.points, points + 1)
def calculate_errors_4DMaja_real(path_fits, path_gt, model): # Parameters: # -------------- # path_fits: 'string' of the directory that contains your reconstructed meshes. # Meshes' filenames should be the same with the # corresponding ground truth meshes filenames + the suffix of # the mesh type (e.g <frame_number>.obj, <frame_number>.ply) # path_gt : 'string' of the full path of the ground truth mesh. # model : 'string' of the model template you are using. Should be one of the # following: 'LSFM', 'Basel', 'Surrey' # Returns: # -------------- # errors : python list that contains the error per vertex between the ground # truth mesh and a mean mesh calculated from your reconstructed meshes path_fits = Path(path_fits) path_gt = Path(path_gt) # load meshes' filenames filenames = [p.name for p in path_fits.glob('*')] filenames.sort() # load gt_mesh (it is olny one, maja's neautral face) gt_mesh = m3io.import_mesh(path_gt, texture_resolver=None) gt_mesh.landmarks['ibug49'] = PointCloud(gt_mesh.points[lms_indexes][19:]) errors = [0] # accumulate fits acc_points = np.zeros((gt_mesh.n_points, 3)) for i, filename in enumerate(print_progress(filenames)): fit_3d = m3io.import_mesh(path_fits / filename, texture_resolver=None) if model == 'Surrey': lms = face_ibug_49_to_face_ibug_49( PointCloud(fit_3d.points[eos.load_eos_low_res_lm_index()])) fit_3d = eos.upsample_eos_low_res_to_fw_no_texture(fit_3d) fit_3d.landmarks['ibug49'] = lms elif model == 'LSFM' or model == 'Basel': fit_3d.landmarks['ibug49'] = PointCloud( fit_3d.points[lms_indexes][19:]) else: print('Error: Not supported model template') return acc_points += fit_3d.points # create mean_fit_3d mean_fit_3d = TriMesh(acc_points / len(filenames), gt_mesh.trilist) mean_fit_3d.landmarks['ibug49'] = PointCloud( mean_fit_3d.points[lms_indexes][19:]) # calculate per vertex errors between the neutral gt_mesh and the mean_fit_3d gt_mesh, eval_mask = landmark_and_mask_gt_mesh(gt_mesh, distance=1) errors[0], _, _ = mask_align_and_calculate_dense_error( mean_fit_3d, gt_mesh, eval_mask) return errors
def test_trimesh_face_normals(): points = np.array([[0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [0.0, 1.0, 0.0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) expected_normals = np.array( [[-np.sqrt(3) / 3, -np.sqrt(3) / 3, np.sqrt(3) / 3], [-0, -0, 1]]) trimesh = TriMesh(points, trilist) face_normals = trimesh.tri_normals() assert_allclose(face_normals, expected_normals)
def test_trimesh_boundary_tri_index(): points = np.array([[0.0, 0.0, 0.0], [0.5, 0.5, 0.0], [0.0, 0.5, 0.0], [-0.5, 0.5, 0.0], [-0.5, -0.5, 0.0], [0.5, -0.5, 0.0], [0.0, -1.0, 0.0]]) trilist = np.array([[0, 2, 3], [2, 0, 1], [4, 0, 3], [0, 5, 1], [4, 5, 0], [5, 4, 6]]) trimesh = TriMesh(points, trilist) boundary_tri_index = trimesh.boundary_tri_index() # The "middle" triangle is [4, 5, 0] which is surrounded on all sides # [5, 4, 6] has two edges that have no neighbours assert_allclose(boundary_tri_index, [True, True, True, True, False, True])
def test_trimesh_face_normals(): points = np.array([[0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [0.0, 1.0, 0.0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) expected_normals = np.array([[-np.sqrt(3)/3, -np.sqrt(3)/3, np.sqrt(3)/3], [-0, -0, 1]]) trimesh = TriMesh(points, trilist) face_normals = trimesh.tri_normals() assert_allclose(face_normals, expected_normals)
def test_trimesh_copy(): points = np.ones([10, 3]) trilist = np.ones([10, 3]) landmarks = PointCloud(np.ones([3, 3]), copy=False) tmesh = TriMesh(points, trilist=trilist, copy=False) tmesh.landmarks['test'] = landmarks tmesh_copy = tmesh.copy() assert (not is_same_array(tmesh_copy.points, tmesh.points)) assert (not is_same_array(tmesh_copy.trilist, tmesh.trilist)) assert (not is_same_array(tmesh_copy.landmarks['test'].lms.points, tmesh.landmarks['test'].lms.points))
def test_trimesh_vertex_normals(): points = np.array([[0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [0.0, 1.0, 0.0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) # 0 and 2 are the corner of the triangles and so the maintain the # face normals. The other two are the re-normalised vertices: # normalise(n0 + n2) expected_normals = np.array( [[-np.sqrt(3) / 3, -np.sqrt(3) / 3, np.sqrt(3) / 3], [-0.32505758, -0.32505758, 0.88807383], [0, 0, 1], [-0.32505758, -0.32505758, 0.88807383]]) trimesh = TriMesh(points, trilist) vertex_normals = trimesh.vertex_normals() assert_allclose(vertex_normals, expected_normals)
def duplicate_vertices(mesh): # generate a new mesh with unique vertices per triangle # (i.e. duplicate verts so that each triangle is unique) old_to_new = mesh.trilist.ravel() old_to_new = mesh.trilist.ravel() new_trilist = np.arange(old_to_new.shape[0]).reshape([-1, 3]) new_points = mesh.points[old_to_new] return TriMesh(new_points, trilist=new_trilist), old_to_new
def test_trimesh__init_2d_grid(): tm = TriMesh.init_2d_grid([10, 10]) assert tm.n_points == 100 assert tm.n_dims == 2 # 162 = 9 * 9 * 2 assert_allclose(tm.trilist.shape, (162, 3)) assert_allclose(tm.range(), [9, 9])
def test_chain_pwa_before_tps(): a_tm = TriMesh(np.random.random([10, 2])) b = PointCloud(np.random.random([10, 2])) pwa = PiecewiseAffine(a_tm, b) tps = ThinPlateSplines(b, a_tm) chain = pwa.compose_before(tps) assert (isinstance(chain, TransformChain))
def constrain_to_pointcloud(self, pointcloud, trilist=None): r""" Restricts this mask to be equal to the convex hull around a point cloud Parameters ---------- pointcloud : :map:`PointCloud` The pointcloud of points that should be constrained to trilist: (t, 3) ndarray, Optional Triangle list to be used on the points in selecting the mask region. If None defaults to performing Delaunay triangulation on the points. Default: None """ from menpo.transform.piecewiseaffine import PiecewiseAffine from menpo.transform.piecewiseaffine import TriangleContainmentError if self.n_dims != 2: raise ValueError("can only constrain mask on 2D images.") if trilist is not None: from menpo.shape import TriMesh pointcloud = TriMesh(pointcloud.points, trilist) pwa = PiecewiseAffine(pointcloud, pointcloud) try: pwa.apply(self.indices) except TriangleContainmentError as e: self.from_vector_inplace(~e.points_outside_source_domain)
def test_trimesh_creation_copy_warning(): points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]], order='F') with warnings.catch_warnings(record=True) as w: warnings.simplefilter("always") TriMesh(points, trilist=trilist, copy=False) assert len(w) == 1
def face_ibug_68_to_face_ibug_68_trimesh(pcloud): r""" Apply the IBUG 68-point semantic labels, with trimesh connectivity. The semantic labels applied are as follows: - tri References ---------- .. [1] http://www.multipie.org/ .. [2] http://ibug.doc.ic.ac.uk/resources/300-W/ """ from menpo.shape import TriMesh n_expected_points = 68 validate_input(pcloud, n_expected_points) tri_list = np.array([[47, 29, 28], [44, 43, 23], [38, 20, 21], [47, 28, 42], [49, 61, 60], [40, 41, 37], [37, 19, 20], [28, 40, 39], [38, 21, 39], [36, 1, 0], [48, 59, 4], [49, 60, 48], [67, 59, 60], [13, 53, 14], [61, 51, 62], [57, 8, 7], [52, 51, 33], [61, 67, 60], [52, 63, 51], [66, 56, 57], [35, 30, 29], [53, 52, 35], [37, 36, 17], [18, 37, 17], [37, 38, 40], [38, 37, 20], [19, 37, 18], [38, 39, 40], [28, 29, 40], [41, 36, 37], [27, 39, 21], [41, 31, 1], [30, 32, 31], [33, 51, 50], [33, 30, 34], [31, 40, 29], [36, 0, 17], [31, 2, 1], [31, 41, 40], [ 1, 36, 41], [31, 49, 2], [ 2, 49, 3], [60, 59, 48], [ 3, 49, 48], [31, 32, 50], [48, 4, 3], [59, 5, 4], [58, 67, 66], [ 5, 59, 58], [58, 59, 67], [ 7, 6, 58], [66, 57, 58], [13, 54, 53], [ 7, 58, 57], [ 6, 5, 58], [50, 61, 49], [62, 67, 61], [31, 50, 49], [32, 33, 50], [30, 33, 32], [34, 52, 33], [35, 52, 34], [53, 63, 52], [62, 63, 65], [62, 51, 63], [66, 65, 56], [63, 53, 64], [62, 66, 67], [62, 65, 66], [57, 56, 9], [65, 63, 64], [ 8, 57, 9], [ 9, 56, 10], [10, 56, 11], [11, 56, 55], [11, 55, 12], [56, 65, 55], [55, 64, 54], [55, 65, 64], [55, 54, 12], [64, 53, 54], [12, 54, 13], [45, 46, 44], [35, 34, 30], [14, 53, 35], [15, 46, 45], [27, 28, 39], [27, 42, 28], [35, 29, 47], [30, 31, 29], [15, 35, 46], [15, 14, 35], [43, 22, 23], [27, 21, 22], [24, 44, 23], [44, 47, 43], [43, 47, 42], [46, 35, 47], [26, 45, 44], [46, 47, 44], [25, 44, 24], [25, 26, 44], [16, 15, 45], [16, 45, 26], [22, 42, 43], [50, 51, 61], [27, 22, 42]]) new_pcloud = TriMesh(pcloud.points, tri_list) mapping = OrderedDict() mapping['tri'] = np.arange(new_pcloud.n_points) return new_pcloud, mapping
def eye_ibug_close_17_to_eye_ibug_close_17_trimesh(pcloud): r""" Apply the IBUG 17-point close eye semantic labels, with trimesh connectivity. The semantic labels applied are as follows: - tri """ from menpo.shape import TriMesh n_expected_points = 17 validate_input(pcloud, n_expected_points) tri_list = np.array([[10, 11, 13], [ 3, 13, 2], [ 4, 14, 3], [15, 5, 16], [12, 11, 0], [13, 14, 10], [13, 12, 2], [14, 13, 3], [ 0, 1, 12], [ 2, 12, 1], [13, 11, 12], [ 9, 10, 14], [15, 9, 14], [ 7, 8, 15], [ 5, 6, 16], [15, 14, 4], [ 7, 15, 16], [ 8, 9, 15], [15, 4, 5], [16, 6, 7]]) new_pcloud = TriMesh(pcloud.points, tri_list, copy=False) mapping = OrderedDict() mapping['tri'] = np.arange(new_pcloud.n_points) return new_pcloud, mapping
def _construct_shape_type(points, trilist, tcoords, texture, colour_per_vertex): # Four different outcomes - either we have a textured mesh, a coloured # mesh or just a plain mesh or we fall back to a plain pointcloud. if trilist is None: obj = PointCloud(points, copy=False) elif tcoords is not None and texture is not None: obj = TexturedTriMesh(points, tcoords, texture, trilist=trilist, copy=False) elif colour_per_vertex is not None: obj = ColouredTriMesh(points, trilist=trilist, colours=colour_per_vertex, copy=False) else: # TriMesh fall through obj = TriMesh(points, trilist=trilist, copy=False) if tcoords is not None and texture is None: warnings.warn('tcoords were found, but no texture was recovered, ' 'reverting to an untextured mesh.') if texture is not None and tcoords is None: warnings.warn('texture was found, but no tcoords were recovered, ' 'reverting to an untextured mesh.') return obj
def getTriMeshfromPly(path): data = [] with open(path) as csv_file: csv_reader = csv.reader(csv_file, delimiter=' ') for row in csv_reader: data.append(row) flag = False points = [] trilist = [] count = 0 for row in range(len(data)): if (data[row][0] == 'element') and (data[row][1] == 'vertex'): numOfVertices = int(data[row][2]) if flag and count < numOfVertices: data[row][0] = "{0:.6f}".format(float(data[row][0])) data[row][1] = "{0:.6f}".format(float(data[row][1])) data[row][2] = "{0:.6f}".format(float(data[row][2])) points.append([ float(data[row][0]), float(data[row][1]), float(data[row][2]) ]) count = count + 1 elif flag and count >= numOfVertices: if data[row][0] == '3': trilist.append( [int(data[row][1]), int(data[row][2]), int(data[row][3])]) if (data[row][0] == 'end_header'): flag = True points_np = np.array(points) trilist_np = np.array(trilist) return TriMesh(points_np, trilist_np)
def chain_tps_after_pwa_test(): a_tm = TriMesh(np.random.random([10, 2])) b = PointCloud(np.random.random([10, 2])) pwa = PiecewiseAffine(a_tm, b) tps = ThinPlateSplines(b, a_tm) chain = tps.compose_after(pwa) assert(isinstance(chain, TransformChain))
def test_trimesh_init_2d_grid(): tm = TriMesh.init_2d_grid([10, 10]) assert tm.n_points == 100 assert tm.n_dims == 2 # 162 = 9 * 9 * 2 assert_allclose(tm.trilist.shape, (162, 3)) assert_allclose(tm.range(), [9, 9])
def __init__(self, source, target): from menpo.shape import TriMesh # to avoid circular import if not isinstance(source, TriMesh): source = TriMesh(source.points) Alignment.__init__(self, source, target) if self.n_dims != 2: raise ValueError("source and target must be 2 " "dimensional")
def test_trimesh_init_from_depth_image(): fake_z = np.random.uniform(size=(10, 10)) tm = TriMesh.init_from_depth_image(Image(fake_z)) assert tm.n_points == 100 assert tm.n_dims == 3 assert_allclose(tm.range()[:2], [9, 9]) assert tm.points[:, -1].max() <= 1.0 assert tm.points[:, -1].min() >= 0.0
def test_trimesh_vertex_normals(): points = np.array([[0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [0.0, 1.0, 0.0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) # 0 and 2 are the corner of the triangles and so the maintain the # face normals. The other two are the re-normalised vertices: # normalise(n0 + n2) expected_normals = np.array([[-np.sqrt(3)/3, -np.sqrt(3)/3, np.sqrt(3)/3], [-0.32505758, -0.32505758, 0.88807383], [0, 0, 1], [-0.32505758, -0.32505758, 0.88807383]]) trimesh = TriMesh(points, trilist) vertex_normals = trimesh.vertex_normals() assert_allclose(vertex_normals, expected_normals)
def test_trimesh_creation(): points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) TriMesh(points, trilist)
def test_trimesh_n_tris(): points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) trimesh = TriMesh(points, trilist) assert(trimesh.n_tris == 2)
def test_trimesh_from_tri_mask(): points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) mask = np.zeros(2, dtype=np.bool) mask[0] = True trimesh = TriMesh(points, trilist=trilist).from_tri_mask(mask) assert (trimesh.n_tris == 1) assert (trimesh.n_points == 3) assert_allclose(trimesh.points, points[trilist[0]])
def load_tassos_lsfm_combined_model(path): m = loadmat(str(path)) mean = TriMesh(m['mean'].reshape([-1, 3]), trilist=m['trilist']) return { 'shape_model': PCAModel.init_from_components( m['components'].T, m['eigenvalues'].ravel(), mean, 8888, True), 'n_id_comps': int(m['n_trunc_ids'][0][0]), 'n_exp_comps': int(m['n_trunc_expressions'][0][0]) }
def test_trimesh_creation_copy_false(): points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) tm = TriMesh(points, trilist, copy=False) assert (is_same_array(tm.points, points)) assert (is_same_array(tm.trilist, trilist))
def generative_construct(DB, fitter, trilist, label=None, fit_group='mean', train_group='final', feature=igo, diagonal=200, scales=(0.5, 1.0), n_processes=24, model_class=HolisticAAM, increament_model=None, original_shape_model=None, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, max_iters=10, ): # fix appearance optimize shape error = [] # Multi Processs Computation frs = mp_fit(DB, fitter, group=fit_group, n_processes=n_processes, max_iters=max_iters) for fr, img in zip(frs, DB): img.landmarks[train_group] = TriMesh(fr.final_shape.points, trilist=trilist) error.append(fr.final_error(alignment_error)) if label: img.landmarks[label] = TriMesh(fr.final_shape.points, trilist=trilist) del fitter if increament_model: pdm = copy.deepcopy(increament_model) pdm.increment(DB, verbose=True, group=train_group, shape_forgetting_factor=shape_forgetting_factor, appearance_forgetting_factor=appearance_forgetting_factor) else: pdm = model_class(DB, holistic_features=feature, diagonal=diagonal, scales=scales, verbose=True, group=train_group) if original_shape_model: pdm.shape_models = original_shape_model if increament_model: del increament_model return pdm, np.mean(error)
def from_UV_2_3D(uv, uv_layout='oval', topology='full', plot=False): res = uv.shape[0] info_dict = import_uv_info(uv, res, uv_layout=uv_layout, topology=topology) tmask = info_dict['tmask'] tc_ps = info_dict['tcoords_pixel_scaled'] tmask_im = info_dict['tmask_image'] trilist = info_dict['trilist'] #uv = interpolaton_of_uv_xyz(uv,tmask).as_unmasked() x = uv.pixels[0][(tc_ps.points.astype(int).T[0, :], tc_ps.points.astype(int).T[1, :])] y = uv.pixels[1][(tc_ps.points.astype(int).T[0, :], tc_ps.points.astype(int).T[1, :])] z = uv.pixels[2][(tc_ps.points.astype(int).T[0, :], tc_ps.points.astype(int).T[1, :])] points = np.hstack((x.T[:, None], y.T[:, None], z.T[:, None])) if plot is True: TriMesh(points, trilist).view() return TriMesh(points, trilist)
def test_trimesh_init_from_depth_image_masked(): fake_z = np.random.uniform(size=(10, 10)) mask = np.zeros(fake_z.shape, dtype=np.bool) mask[2:6, 2:6] = True im = MaskedImage(fake_z, mask=mask) tm = TriMesh.init_from_depth_image(im) assert tm.n_points == 16 assert tm.n_dims == 3 assert_allclose(tm.range()[:2], [3, 3]) assert tm.points[:, -1].max() <= 1.0 assert tm.points[:, -1].min() >= 0.0
def _construct_shape_type(points, trilist, tcoords, texture, colour_per_vertex): r""" Construct the correct Shape subclass given the inputs. TexturedTriMesh can only be created when tcoords and texture are available. ColouredTriMesh can only be created when colour_per_vertex is non None and TriMesh can only be created when trilist is non None. Worst case fall back is PointCloud. Parameters ---------- points : ``(N, D)`` `ndarray` The N-D points. trilist : ``(N, 3)`` `ndarray`` or ``None`` Triangle list or None. tcoords : ``(N, 2)`` `ndarray` or ``None`` Texture coordinates. texture : :map:`Image` or ``None`` Texture. colour_per_vertex : ``(N, 1)`` or ``(N, 3)`` `ndarray` or ``None`` The colour per vertex. Returns ------- shape : :map:`PointCloud` or subclass The correct shape for the given inputs. """ # Four different outcomes - either we have a textured mesh, a coloured # mesh or just a plain mesh or we fall back to a plain pointcloud. if trilist is None: obj = PointCloud(points, copy=False) elif tcoords is not None and texture is not None: obj = TexturedTriMesh(points, tcoords, texture, trilist=trilist, copy=False) elif colour_per_vertex is not None: obj = ColouredTriMesh(points, trilist=trilist, colours=colour_per_vertex, copy=False) else: # TriMesh fall through obj = TriMesh(points, trilist=trilist, copy=False) if tcoords is not None and texture is None: warnings.warn('tcoords were found, but no texture was recovered, ' 'reverting to an untextured mesh.') if texture is not None and tcoords is None: warnings.warn('texture was found, but no tcoords were recovered, ' 'reverting to an untextured mesh.') return obj
def _get_mean_model(model): """ Get mean BFM mesh. Parameters: model (BFM dict model): BFM model Returns: mean_mesh (menpo.shape.mesh.base.TriMesh): mean mesh model """ return TriMesh(points=np.reshape(model['shapeMU'], (-1, 3)), trilist=model['tri'])
def face_ibug_68_to_face_ibug_51_trimesh(pcloud): r""" Apply the IBUG 51-point semantic labels, with trimesh connectivity.. The semantic labels applied are as follows: - tri References ---------- .. [1] http://www.multipie.org/ .. [2] http://ibug.doc.ic.ac.uk/resources/300-W/ """ from menpo.shape import TriMesh # Apply face_ibug_68_to_face_ibug_51 new_pcloud = face_ibug_68_to_face_ibug_51(pcloud) # This is in terms of the 51 points tri_list = np.array([[30, 12, 11], [27, 26, 6], [21, 3, 4], [30, 11, 25], [32, 44, 43], [23, 24, 20], [20, 2, 3], [11, 23, 22], [21, 4, 22], [32, 43, 31], [50, 42, 43], [44, 34, 45], [35, 34, 16], [44, 50, 43], [35, 46, 34], [49, 39, 40], [18, 13, 12], [36, 35, 18], [20, 19, 0], [1, 20, 0], [20, 21, 23], [21, 20, 3], [2, 20, 1], [21, 22, 23], [11, 12, 23], [24, 19, 20], [10, 22, 4], [13, 15, 14], [16, 34, 33], [16, 13, 17], [14, 23, 12], [14, 24, 23], [43, 42, 31], [14, 15, 33], [41, 50, 49], [41, 42, 50], [49, 40, 41], [33, 44, 32], [45, 50, 44], [14, 33, 32], [15, 16, 33], [13, 16, 15], [17, 35, 16], [18, 35, 17], [36, 46, 35], [45, 46, 48], [45, 34, 46], [49, 48, 39], [46, 36, 47], [45, 49, 50], [45, 48, 49], [48, 46, 47], [39, 48, 38], [38, 47, 37], [38, 48, 47], [47, 36, 37], [28, 29, 27], [18, 17, 13], [10, 11, 22], [10, 25, 11], [18, 12, 30], [13, 14, 12], [26, 5, 6], [10, 4, 5], [7, 27, 6], [27, 30, 26], [26, 30, 25], [29, 18, 30], [9, 28, 27], [29, 30, 27], [8, 27, 7], [8, 9, 27], [5, 25, 26], [33, 34, 44], [10, 5, 25]]) new_pcloud = TriMesh(new_pcloud.points, trilist=tri_list, copy=False) mapping = OrderedDict() mapping['tri'] = np.arange(new_pcloud.n_points) return new_pcloud, mapping
def pseudoinverse(self): r""" The pseudoinverse of the transform - that is, the transform that results from swapping `source` and `target`, or more formally, negating the transforms parameters. If the transform has a true inverse this is returned instead. :type: ``type(self)`` """ from menpo.shape import PointCloud, TriMesh # to avoid circular import new_source = TriMesh(self.target.points, self.source.trilist) new_target = PointCloud(self.source.points) return type(self)(new_source, new_target)
def non_rigid_icp(source, target, eps=1e-3, stiffness_values=None, verbose=False, landmarks=None, lm_weight=None): r""" Deforms the source trimesh to align with to optimally the target. """ # Scale factors completely change the behavior of the algorithm - always # rescale the source down to a sensible size (so it fits inside box of # diagonal 1) and is centred on the origin. We'll undo this after the fit # so the user can use whatever scale they prefer. tr = Translation(-1 * source.centre()) sc = UniformScale(1.0 / np.sqrt(np.sum(source.range() ** 2)), 3) prepare = tr.compose_before(sc) source = prepare.apply(source) target = prepare.apply(target) # store how to undo the similarity transform restore = prepare.pseudoinverse() n_dims = source.n_dims # Homogeneous dimension (1 extra for translation effects) h_dims = n_dims + 1 points, trilist = source.points, source.trilist n = points.shape[0] # record number of points edge_tris = source.boundary_tri_index() M_s = node_arc_incidence_matrix(source) # weight matrix G = np.identity(n_dims + 1) M_kron_G_s = sp.kron(M_s, G) # build octree for finding closest points on target. target_vtk = trimesh_to_vtk(target) closest_points_on_target = VTKClosestPointLocator(target_vtk) # save out the target normals. We need them for the weight matrix. target_tri_normals = target.tri_normals() # init transformation X_prev = np.tile(np.zeros((n_dims, h_dims)), n).T v_i = points if stiffness_values is not None: stiffness = stiffness_values if verbose: print('using user defined stiffness values: {}'.format(stiffness)) else: # these values have been empirically found to perform well for well # rigidly aligned facial meshes stiffness = [50, 20, 5, 2, 0.8, 0.5, 0.35, 0.2] if verbose: print('using default stiffness values: {}'.format(stiffness)) if lm_weight is not None: lm_weight = lm_weight if verbose: print('using user defined lm_weight values: {}'.format(lm_weight)) else: # these values have been empirically found to perform well for well # rigidly aligned facial meshes lm_weight = [5, 2, .5, 0, 0, 0, 0, 0] if verbose: print('using default lm_weight values: {}'.format(lm_weight)) # to store per iteration information info = [] # we need to prepare some indices for efficient construction of the D # sparse matrix. row = np.hstack((np.repeat(np.arange(n)[:, None], n_dims, axis=1).ravel(), np.arange(n))) x = np.arange(n * h_dims).reshape((n, h_dims)) col = np.hstack((x[:, :n_dims].ravel(), x[:, n_dims])) if landmarks is not None: if verbose: print("'{}' landmarks will be used as a landmark constraint.".format(landmarks)) source_lm_index = source.distance_to( source.landmarks[landmarks].lms).argmin(axis=0) target_lms = target.landmarks[landmarks].lms U_L = target_lms.points n_landmarks = target_lms.n_points lm_mask = np.in1d(row, source_lm_index) col_lm = col[lm_mask] # pull out the rows for the lms - but the values are # all wrong! need to map them back to the order of the landmarks row_lm_to_fix = row[lm_mask] source_lm_index_l = list(source_lm_index) row_lm = np.array([source_lm_index_l.index(r) for r in row_lm_to_fix]) o = np.ones(n) for alpha, beta in zip(stiffness, lm_weight): alpha_M_kron_G_s = alpha * M_kron_G_s # get the term for stiffness j = 0 while True: # iterate until convergence # find nearest neighbour and the normals U, tri_indices = closest_points_on_target(v_i) # ---- WEIGHTS ---- # 1. Edges # Are any of the corresponding tris on the edge of the target? # Where they are we return a false weight (we *don't* want to # include these points in the solve) w_i_e = np.in1d(tri_indices, edge_tris, invert=True) # 2. Normals # Calculate the normals of the current v_i v_i_tm = TriMesh(v_i, trilist=trilist, copy=False) v_i_n = v_i_tm.vertex_normals() # Extract the corresponding normals from the target u_i_n = target_tri_normals[tri_indices] # If the dot of the normals is lt 0.9 don't contrib to deformation w_i_n = (u_i_n * v_i_n).sum(axis=1) > 0.9 # 3. Self-intersection # This adds approximately 12% to the running cost and doesn't seem # to be very critical in helping mesh fitting performance so for # now it's removed. Revisit later. # # Build an intersector for the current deformed target # intersect = build_intersector(to_vtk(v_i_tm)) # # budge the source points 1% closer to the target # source = v_i + ((U - v_i) * 0.5) # # if the vector from source to target intersects the deformed # # template we don't want to include it in the optimisation. # problematic = [i for i, (s, t) in enumerate(zip(source, U)) # if len(intersect(s, t)[0]) > 0] # print(len(problematic) * 1.0 / n) # w_i_i = np.ones(v_i_tm.n_points, dtype=np.bool) # w_i_i[problematic] = False # Form the overall w_i from the normals, edge case w_i = np.logical_and(w_i_n, w_i_e) # we could add self intersection at a later date too... # w_i = np.logical_and(np.logical_and(w_i_n, w_i_e), w_i_i) prop_w_i = (n - w_i.sum() * 1.0) / n prop_w_i_n = (n - w_i_n.sum() * 1.0) / n prop_w_i_e = (n - w_i_e.sum() * 1.0) / n j = j + 1 # Build the sparse diagonal weight matrix W_s = sp.diags(w_i.astype(np.float)[None, :], [0]) data = np.hstack((v_i.ravel(), o)) D_s = sp.coo_matrix((data, (row, col))) # nullify the masked U values U[~w_i] = 0 to_stack_A = [alpha_M_kron_G_s, W_s.dot(D_s)] to_stack_B = [np.zeros((alpha_M_kron_G_s.shape[0], n_dims)), U] if landmarks: D_L = sp.coo_matrix((data[lm_mask], (row_lm, col_lm)), shape=(n_landmarks, D_s.shape[1])) to_stack_A.append(beta * D_L) to_stack_B.append(beta * U_L) A_s = sp.vstack(to_stack_A).tocsr() B_s = sp.vstack(to_stack_B).tocsr() X = spsolve(A_s, B_s) # deform template v_i = D_s.dot(X) err = np.linalg.norm(X_prev - X, ord='fro') if landmarks is not None: src_lms = v_i[source_lm_index] lm_err = np.sqrt((src_lms - U_L) ** 2).sum(axis=1).mean() if verbose: v_str = ('a: {}, ({}) - total : {:.0%} norms: {:.0%} ' 'edges: {:.0%}'.format(alpha, j, prop_w_i, prop_w_i_n, prop_w_i_e)) if landmarks is not None: v_str += ' beta: {}, lm_err: {:.5f}'.format(beta, lm_err) print(v_str) info_dict = { 'alpha': alpha, 'iteration': j + 1, 'prop_omitted': prop_w_i, 'prop_omitted_norms': prop_w_i_n, 'prop_omitted_edges': prop_w_i_e, 'delta': err } if landmarks: info_dict['beta'] = beta info_dict['lm_err'] = lm_err info.append(info_dict) X_prev = X if err / np.sqrt(np.size(X_prev)) < eps: break # final result if we choose closest points point_corr = closest_points_on_target(v_i)[0] result = { 'deformed_source': restore.apply(v_i), 'matched_target': restore.apply(point_corr), 'matched_tri_indices': tri_indices, 'info': info } if landmarks is not None: result['source_lm_index'] = source_lm_index return result
def non_rigid_icp_generator(source, target, eps=1e-3, stiffness_weights=None, data_weights=None, landmark_group=None, landmark_weights=None, v_i_update_func=None, verbose=False): r""" Deforms the source trimesh to align with to optimally the target. """ # If landmarks are provided, we should always start with a simple # AlignmentSimilarity between the landmarks to initialize optimally. if landmark_group is not None: if verbose: print("'{}' landmarks will be used as " "a landmark constraint.".format(landmark_group)) print("performing similarity alignment using landmarks") lm_align = AlignmentSimilarity(source.landmarks[landmark_group], target.landmarks[landmark_group]).as_non_alignment() source = lm_align.apply(source) # Scale factors completely change the behavior of the algorithm - always # rescale the source down to a sensible size (so it fits inside box of # diagonal 1) and is centred on the origin. We'll undo this after the fit # so the user can use whatever scale they prefer. tr = Translation(-1 * source.centre()) sc = UniformScale(1.0 / np.sqrt(np.sum(source.range() ** 2)), 3) prepare = tr.compose_before(sc) source = prepare.apply(source) target = prepare.apply(target) # store how to undo the similarity transform restore = prepare.pseudoinverse() n_dims = source.n_dims # Homogeneous dimension (1 extra for translation effects) h_dims = n_dims + 1 points, trilist = source.points, source.trilist n = points.shape[0] # record number of points edge_tris = source.boundary_tri_index() M_s, unique_edge_pairs = node_arc_incidence_matrix(source) # weight matrix G = np.identity(n_dims + 1) M_kron_G_s = sp.kron(M_s, G) # build octree for finding closest points on target. target_vtk = trimesh_to_vtk(target) closest_points_on_target = VTKClosestPointLocator(target_vtk) # save out the target normals. We need them for the weight matrix. target_tri_normals = target.tri_normals() # init transformation X_prev = np.tile(np.zeros((n_dims, h_dims)), n).T v_i = points if stiffness_weights is not None: if verbose: print('using user-defined stiffness_weights') validate_weights('stiffness_weights', stiffness_weights, source.n_points, verbose=verbose) else: # these values have been empirically found to perform well for well # rigidly aligned facial meshes stiffness_weights = [50, 20, 5, 2, 0.8, 0.5, 0.35, 0.2] if verbose: print('using default ' 'stiffness_weights: {}'.format(stiffness_weights)) n_iterations = len(stiffness_weights) if landmark_weights is not None: if verbose: print('using user defined ' 'landmark_weights: {}'.format(landmark_weights)) elif landmark_group is not None: # these values have been empirically found to perform well for well # rigidly aligned facial meshes landmark_weights = [5, 2, .5, 0, 0, 0, 0, 0] if verbose: print('using default ' 'landmark_weights: {}'.format(landmark_weights)) else: # no landmark_weights provided - no landmark_group in use. We still # need a landmark group for the iterator landmark_weights = [None] * n_iterations # We should definitely have some landmark weights set now - check the # number is correct. # Note we say verbose=False, as we have done custom reporting above, and # per-vertex landmarks are not supported. validate_weights('landmark_weights', landmark_weights, source.n_points, n_iterations=n_iterations, verbose=False) if data_weights is not None: if verbose: print('using user-defined data_weights') validate_weights('data_weights', data_weights, source.n_points, n_iterations=n_iterations, verbose=verbose) else: data_weights = [None] * n_iterations if verbose: print('Not customising data_weights') # we need to prepare some indices for efficient construction of the D # sparse matrix. row = np.hstack((np.repeat(np.arange(n)[:, None], n_dims, axis=1).ravel(), np.arange(n))) x = np.arange(n * h_dims).reshape((n, h_dims)) col = np.hstack((x[:, :n_dims].ravel(), x[:, n_dims])) o = np.ones(n) if landmark_group is not None: source_lm_index = source.distance_to( source.landmarks[landmark_group]).argmin(axis=0) target_lms = target.landmarks[landmark_group] U_L = target_lms.points n_landmarks = target_lms.n_points lm_mask = np.in1d(row, source_lm_index) col_lm = col[lm_mask] # pull out the rows for the lms - but the values are # all wrong! need to map them back to the order of the landmarks row_lm_to_fix = row[lm_mask] source_lm_index_l = list(source_lm_index) row_lm = np.array([source_lm_index_l.index(r) for r in row_lm_to_fix]) for i, (alpha, beta, gamma) in enumerate(zip(stiffness_weights, landmark_weights, data_weights), 1): alpha_is_per_vertex = isinstance(alpha, np.ndarray) if alpha_is_per_vertex: # stiffness is provided per-vertex if alpha.shape[0] != source.n_points: raise ValueError() alpha_per_edge = alpha[unique_edge_pairs].mean(axis=1) alpha_M_s = sp.diags(alpha_per_edge).dot(M_s) alpha_M_kron_G_s = sp.kron(alpha_M_s, G) else: # stiffness is global - just a scalar multiply. Note that here # we don't have to recalculate M_kron_G_s alpha_M_kron_G_s = alpha * M_kron_G_s if verbose: a_str = (alpha if not alpha_is_per_vertex else 'min: {:.2f}, max: {:.2f}'.format(alpha.min(), alpha.max())) i_str = '{}/{}: stiffness: {}'.format(i, len(stiffness_weights), a_str) if landmark_group is not None: i_str += ' lm_weight: {}'.format(beta) print(i_str) j = 0 while True: # iterate until convergence j += 1 # track the iterations for this alpha/landmark weight # find nearest neighbour and the normals U, tri_indices = closest_points_on_target(v_i) # ---- WEIGHTS ---- # 1. Edges # Are any of the corresponding tris on the edge of the target? # Where they are we return a false weight (we *don't* want to # include these points in the solve) w_i_e = np.in1d(tri_indices, edge_tris, invert=True) # 2. Normals # Calculate the normals of the current v_i v_i_tm = TriMesh(v_i, trilist=trilist) v_i_n = v_i_tm.vertex_normals() # Extract the corresponding normals from the target u_i_n = target_tri_normals[tri_indices] # If the dot of the normals is lt 0.9 don't contrib to deformation w_i_n = (u_i_n * v_i_n).sum(axis=1) > 0.9 # 3. Self-intersection # This adds approximately 12% to the running cost and doesn't seem # to be very critical in helping mesh fitting performance so for # now it's removed. Revisit later. # # Build an intersector for the current deformed target # intersect = build_intersector(to_vtk(v_i_tm)) # # budge the source points 1% closer to the target # source = v_i + ((U - v_i) * 0.5) # # if the vector from source to target intersects the deformed # # template we don't want to include it in the optimisation. # problematic = [i for i, (s, t) in enumerate(zip(source, U)) # if len(intersect(s, t)[0]) > 0] # print(len(problematic) * 1.0 / n) # w_i_i = np.ones(v_i_tm.n_points, dtype=np.bool) # w_i_i[problematic] = False # Form the overall w_i from the normals, edge case # for now disable the edge constraint (it was noisy anyway) w_i = w_i_n # w_i = np.logical_and(w_i_n, w_i_e).astype(np.float) # we could add self intersection at a later date too... # w_i = np.logical_and(np.logical_and(w_i_n, # w_i_e, # w_i_i).astype(np.float) prop_w_i = (n - w_i.sum() * 1.0) / n prop_w_i_n = (n - w_i_n.sum() * 1.0) / n prop_w_i_e = (n - w_i_e.sum() * 1.0) / n if data_weights is not None: w_i = w_i * gamma # Build the sparse diagonal weight matrix W_s = sp.diags(w_i.astype(np.float)[None, :], [0]) data = np.hstack((v_i.ravel(), o)) D_s = sp.coo_matrix((data, (row, col))) to_stack_A = [alpha_M_kron_G_s, W_s.dot(D_s)] to_stack_B = [np.zeros((alpha_M_kron_G_s.shape[0], n_dims)), U * w_i[:, None]] # nullify nearest points by w_i if landmark_group is not None: D_L = sp.coo_matrix((data[lm_mask], (row_lm, col_lm)), shape=(n_landmarks, D_s.shape[1])) to_stack_A.append(beta * D_L) to_stack_B.append(beta * U_L) A_s = sp.vstack(to_stack_A).tocsr() B_s = sp.vstack(to_stack_B).tocsr() X = spsolve(A_s, B_s) # deform template v_i_prev = v_i v_i = D_s.dot(X) delta_v_i = v_i - v_i_prev if v_i_update_func: # custom logic is provided to update the current template # deformation. This is typically used by Active NICP. # take the v_i points matrix and convert back to a TriMesh in # the original space def_template = restore.apply(source.from_vector(v_i.ravel())) # perform the update updated_def_template = v_i_update_func(def_template) # convert back to points in the NICP space v_i = prepare.apply(updated_def_template.points) err = np.linalg.norm(X_prev - X, ord='fro') stop_criterion = err / np.sqrt(np.size(X_prev)) if landmark_group is not None: src_lms = v_i[source_lm_index] lm_err = np.sqrt((src_lms - U_L) ** 2).sum(axis=1).mean() if verbose: v_str = (' - {} stop crit: {:.3f} ' 'total: {:.0%} norms: {:.0%} ' 'edges: {:.0%}'.format(j, stop_criterion, prop_w_i, prop_w_i_n, prop_w_i_e)) if landmark_group is not None: v_str += ' lm_err: {:.4f}'.format(lm_err) print(v_str) X_prev = X # track the progress of the algorithm per-iteration info_dict = { 'alpha': alpha, 'iteration': j, 'prop_omitted': prop_w_i, 'prop_omitted_norms': prop_w_i_n, 'prop_omitted_edges': prop_w_i_e, 'delta': err, 'mask_normals': w_i_n, 'mask_edges': w_i_e, 'mask_all': w_i, 'nearest_points': restore.apply(U), 'deformation_per_step': delta_v_i } current_instance = source.copy() current_instance.points = v_i.copy() if landmark_group: info_dict['beta'] = beta info_dict['lm_err'] = lm_err current_instance.landmarks[landmark_group] = PointCloud(src_lms) yield restore.apply(current_instance), info_dict if stop_criterion < eps: break
def test_2d_trimesh_2d_positive_areas(): t = TriMesh(np.array([[0, 0], [0, 1], [1, 1], [1, 0]], dtype=np.float), trilist=np.array([[0, 2, 3], [0, 2, 1]])) assert np.all(t.tri_areas() > 0)