def instance(self, model_type='bfm', alpha=None, beta=None, landmark_group='ibug68'): if alpha is None: alpha = np.zeros(len(self.shape_model.eigenvalues)) if beta is None: beta = np.zeros(len(self.texture_model.eigenvalues)) # Generate instance shape = self.shape_model.instance(alpha, normalized_weights=True) texture = self.texture_model.instance(beta, normalized_weights=True) if model_type == 'bfm': tex_scale = UniformScale(1. / 255, 3) lms_scale = UniformScale(1e-5, 3) texture = tex_scale.apply(texture.reshape([-1, 3])) landmarks = lms_scale.apply(self.landmarks) elif model_type == 'lsfm': pass trimesh = ColouredTriMesh(shape.points, trilist=shape.trilist, colours=texture) trimesh.landmarks[landmark_group] = landmarks return trimesh
def generate_template_from_basel_and_metadata(basel, meta): template = ColouredTriMesh( basel.points[meta['map_tddfa_to_basel']], trilist=meta['tddfa_trilist'], colours=basel.colours[meta['map_tddfa_to_basel']]) template.landmarks['ibug68'] = meta['landmarks']['ibug68'] template.landmarks['nosetip'] = meta['landmarks']['nosetip'] return template
def _instance(self, shape_instance, texture_instance, landmark_group): # Reshape the texture instance texture_instance = texture_instance.reshape([-1, self.n_channels]) # restrict the texture to 0-1 # texture_instance = np.clip(texture_instance, 0, 1) # Create trimesh trimesh = ColouredTriMesh(shape_instance.points, trilist=shape_instance.trilist, colours=texture_instance) # Attach landmarks to trimesh trimesh.landmarks[landmark_group] = self.landmarks # Return trimesh return trimesh
def test_colouredtrimesh_copy(): points = np.ones([10, 3]) colours = np.ones([10, 3]) trilist = np.ones([10, 3]) landmarks = PointCloud(np.ones([3, 3]), copy=False) ctmesh = ColouredTriMesh(points, trilist=trilist, colours=colours, copy=False) ctmesh.landmarks['test'] = landmarks ctmesh_copy = ctmesh.copy() assert (not is_same_array(ctmesh_copy.points, ctmesh.points)) assert (not is_same_array(ctmesh_copy.trilist, ctmesh.trilist)) assert (not is_same_array(ctmesh_copy.colours, ctmesh.colours)) assert (not is_same_array(ctmesh_copy.landmarks['test'].lms.points, ctmesh.landmarks['test'].lms.points))
def test_colouredtrimesh_init_2d_grid(): tm = ColouredTriMesh.init_2d_grid([10, 10]) assert tm.n_points == 100 assert tm.n_dims == 2 # 162 = 9 * 9 * 2 assert_allclose(tm.trilist.shape, (162, 3)) assert_allclose(tm.range(), [9, 9])
def render_mesh(sample_mesh, res=256, scale=1): if sample_mesh.shape[-1] != 3: sample_colours = sample_mesh[..., 3:] else: sample_colours = np.ones_like(sample_mesh) * [0, 0, 1] sample_mesh = sample_mesh[..., :3] sample_mesh = Homogeneous( dm.utils.rotation_matrix(np.deg2rad(90), [0, 0, -1])).apply(sample_mesh) sample_mesh = ColouredTriMesh(sample_mesh * scale * res / 2 + res / 2, trilist=trilist, colours=sample_colours) sample_mesh = lambertian_shading(sample_mesh, ambient_colour=0) store_path = Path(LOGDIR) / str(epoch) if not store_path.exists(): store_path.mkdir() m3io.export_mesh(sample_mesh, store_path / '{}.obj'.format(time.time())) mesh_img = rasterize_mesh(sample_mesh, [res, res]) mesh_img = mesh_img.rotate_ccw_about_centre(180) return mesh_img.pixels_with_channels_at_back()
def test_colouredtrimesh_creation_copy_true(): points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]]) trilist = np.array([[0, 1, 3], [1, 2, 3]]) colours = np.ones([4, 13]) ttm = ColouredTriMesh(points, trilist=trilist, colours=colours, copy=True) assert (not is_same_array(ttm.points, points)) assert (not is_same_array(ttm.trilist, trilist)) assert (not is_same_array(ttm.colours, colours))
def test_colouredtrimesh_init_from_depth_image(): fake_z = np.random.uniform(size=(10, 10)) tm = ColouredTriMesh.init_from_depth_image(Image(fake_z)) assert tm.n_points == 100 assert tm.n_dims == 3 assert_allclose(tm.range()[:2], [9, 9]) assert tm.points[:, -1].max() <= 1.0 assert tm.points[:, -1].min() >= 0.0
def upsample_eos_low_res_to_fw_no_texture(eos_mesh_low_res): bc_fw_on_eos_low_res, tri_index_fw_on_eos_low_res = load_fw_on_eos_low_res_settings( ) effective_fw_pc = eos_mesh_low_res.project_barycentric_coordinates( bc_fw_on_eos_low_res, tri_index_fw_on_eos_low_res) effective_fw = ColouredTriMesh(effective_fw_pc.points, trilist=load_basel_kf_trilist()) return effective_fw
def uv_in_img(image, mesh, uv_template): from menpo3d.rasterize import rasterize_mesh shape = image.shape u_mesh = ColouredTriMesh( mesh.points, trilist=mesh.trilist, colours=uv_template.points[:, 0, None]) u_image = rasterize_mesh(u_mesh, shape) v_mesh = ColouredTriMesh( mesh.points, trilist=mesh.trilist, colours=uv_template.points[:, 1, None]) v_image = rasterize_mesh(v_mesh, shape) IUV_image = Image(np.concatenate( [u_image.mask.pixels, u_image.pixels / 255., v_image.pixels / 255.]).clip(0, 1)) return IUV_image
def render_mesh(sample_mesh, trilist=None, scale=128, offset=128, shape=[256, 256], store_path=None, return_image=True, **kwargs): if isinstance(sample_mesh, ColouredTriMesh): sample_mesh = sample_mesh elif isinstance(sample_mesh, TriMesh): sample_mesh = ColouredTriMesh(sample_mesh.points, trilist=sample_mesh.trilist) sample_mesh = lambertian_shading(sample_mesh, **kwargs) else: if sample_mesh.shape[-1] != 3: sample_colours = sample_mesh[..., 3:] else: sample_colours = np.ones_like(sample_mesh) * [0, 0, 1] sample_mesh = sample_mesh[..., :3] sample_mesh = ColouredTriMesh(sample_mesh * scale + offset, trilist=trilist, colours=sample_colours) sample_mesh = lambertian_shading(sample_mesh, **kwargs) mesh_img = rasterize_mesh(sample_mesh, shape) # mesh_img = mesh_img.rotate_ccw_about_centre(180) if store_path: if not store_path.exists(): store_path.mkdir() m3io.export_mesh(sample_mesh, store_path / '{}.obj'.format(time.time())) if return_image: return mesh_img return mesh_img.pixels_with_channels_at_back()
def test_colouredtrimesh_init_from_depth_image_masked(): fake_z = np.random.uniform(size=(10, 10)) mask = np.zeros(fake_z.shape, dtype=np.bool) mask[2:6, 2:6] = True im = MaskedImage(fake_z, mask=mask) tm = ColouredTriMesh.init_from_depth_image(im) assert tm.n_points == 16 assert tm.n_dims == 3 assert_allclose(tm.range()[:2], [3, 3]) assert tm.points[:, -1].max() <= 1.0 assert tm.points[:, -1].min() >= 0.0
def _construct_shape_type(points, trilist, tcoords, texture, colour_per_vertex): r""" Construct the correct Shape subclass given the inputs. TexturedTriMesh can only be created when tcoords and texture are available. ColouredTriMesh can only be created when colour_per_vertex is non None and TriMesh can only be created when trilist is non None. Worst case fall back is PointCloud. Parameters ---------- points : ``(N, D)`` `ndarray` The N-D points. trilist : ``(N, 3)`` `ndarray`` or ``None`` Triangle list or None. tcoords : ``(N, 2)`` `ndarray` or ``None`` Texture coordinates. texture : :map:`Image` or ``None`` Texture. colour_per_vertex : ``(N, 1)`` or ``(N, 3)`` `ndarray` or ``None`` The colour per vertex. Returns ------- shape : :map:`PointCloud` or subclass The correct shape for the given inputs. """ # Four different outcomes - either we have a textured mesh, a coloured # mesh or just a plain mesh or we fall back to a plain pointcloud. if trilist is None: obj = PointCloud(points, copy=False) elif tcoords is not None and texture is not None: obj = TexturedTriMesh(points, tcoords, texture, trilist=trilist, copy=False) elif colour_per_vertex is not None: obj = ColouredTriMesh(points, trilist=trilist, colours=colour_per_vertex, copy=False) else: # TriMesh fall through obj = TriMesh(points, trilist=trilist, copy=False) if tcoords is not None and texture is None: warnings.warn('tcoords were found, but no texture was recovered, ' 'reverting to an untextured mesh.') if texture is not None and tcoords is None: warnings.warn('texture was found, but no tcoords were recovered, ' 'reverting to an untextured mesh.') return obj
def save_template_from_mesh(path): points, trilist = lio.getTriMeshfromPly(path) template = ColouredTriMesh(points, trilist) #to obtain landmark landmark_68 = [] landmark_100 = [] landmark_ear = [] nosetip = [] count = 0 with open(str(DATA_DIR / 'ibug100.pp')) as pp_file: pp_file = csv.reader(pp_file, delimiter='"') for row in pp_file: count = count + 1 if len(row) >= 10: for i in range(0, 10, 2): if row[i] == " x=": lm_x = row[i + 1] if row[i] == " y=": lm_y = row[i + 1] if row[i] == " z=": lm_z = row[i + 1] if count < 107: landmark_100.append( [float(lm_x), float(lm_y), float(lm_z)]) if count < 75: landmark_68.append([float(lm_x), float(lm_y), float(lm_z)]) if count >= 84 and count <= 94: landmark_ear.append( [float(lm_x), float(lm_y), float(lm_z)]) if count >= 96 and count <= 106: landmark_ear.append( [float(lm_x), float(lm_y), float(lm_z)]) count = 0 with open(str(DATA_DIR / 'nosetip.pp')) as pp_file: pp_file = csv.reader(pp_file, delimiter='"') for row in pp_file: count = count + 1 if len(row) >= 10: for i in range(0, 10, 2): if row[i] == " x=": lm_x = row[i + 1] if row[i] == " y=": lm_y = row[i + 1] if row[i] == " z=": lm_z = row[i + 1] if count < 8: nosetip.append([float(lm_x), float(lm_y), float(lm_z)]) template.landmarks['ibug68'] = PointCloud(np.array(landmark_68)) template.landmarks['ibug100'] = PointCloud(np.array(landmark_100)) template.landmarks['ibugEar'] = PointCloud(np.array(landmark_ear)) template.landmarks['nosetip'] = PointCloud(np.array(nosetip)) save_template(template, overwrite=True)
def import_mesh_as_color_trimesh(path): img = None h, w = 0,0 mean_points = [] trilist = [] vt_colors = [] map3dto2d = {} with open(path, "r") as f: for line in f: if line.startswith("mtllib"): # Get texture mtl_path = path.parent / line.strip().split()[-1] mtl = open(mtl_path, "r") for l in mtl: if l.startswith("map_Kd"): img_path = str(mtl_path.parent / l.strip().split()[-1]) print("Read image at", img_path) img = cv2.imread(img_path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) h,w, _ = img.shape mtl.close() if line.startswith("v "): tokens = line.strip().split() x, y, z = float(tokens[1]), float(tokens[2]), float(tokens[3]) mean_points.append([x,y,z]) if line.startswith("vt "): tokens = line.strip().split() x, y = float(tokens[1]), float(tokens[2]) vt_colors.append(img[int((1-y)*h), int(x*w)].tolist()) if line.startswith("f "): tokens = line.strip().split() tripoints = [] for comp in tokens[1:]: pts = comp.split("/") tripoints.append(int(pts[0]) - 1) map3dto2d[int(pts[0])-1] = int(pts[1])-1 trilist.append(tripoints[:3]) mean_colour = [None] * len(mean_points) for k, v in map3dto2d.items(): mean_colour[k] = vt_colors[v] mean_points = np.array(mean_points) trilist = np.array(trilist) mean_colour = np.array(mean_colour) / 255 return ColouredTriMesh(mean_points, trilist=trilist, colours=mean_colour)
def import_coloured_mesh(fp): points = [] colours = [] trilist = [] with open(fp, 'r') as f: for l in f.readlines(): if l.startswith('v '): pts = list(map(float, l.strip().split(' ')[1:7])) points.append(pts[:3]) colours.append(pts[3:]) elif l.startswith('f '): faces = [ int(f.split('/')[0]) - 1 for f in l.strip().split(' ')[1:4] ] trilist.append(faces) return ColouredTriMesh(np.array(points), trilist=np.array(trilist), colours=np.array(colours))
def as_colouredtrimesh(self, colours=None, copy=True): """ Converts this to a :map:`ColouredTriMesh`. Parameters ---------- colours : ``(N, 3)`` `ndarray`, optional The floating point RGB colour per vertex. If not given, grey will be assigned to each vertex. copy : `bool`, optional If ``True``, the graph will be a copy. Returns ------- coloured : :map:`ColouredTriMesh` A version of this mesh with per-vertex colour assigned. """ ctm = ColouredTriMesh(self.points, trilist=self.trilist, colours=colours, copy=copy) return copy_landmarks_and_path(self, ctm)
def recover_mesh_from_uvxyz(uvxyz, img, template_mask, trilist=None): mu = np.mean(uvxyz[template_mask], axis=0) std = np.std(uvxyz[template_mask], axis=0) nd = norm(mu, std) nd_mask = (nd.cdf(uvxyz[template_mask]) < [0.05, 0.05, 0.01]).any(axis=-1) plan_points = np.stack(np.meshgrid(*map(range, [256, 256])) + [np.zeros([256, 256])], axis=-1)[template_mask][~nd_mask] if trilist is not None: plan_trilist = trilist else: plan_trilist = TriMesh(plan_points[:, :2]).trilist masked_points = uvxyz[template_mask][~nd_mask].clip(0, 1) * 256 color_sampling = masked_points[:, :2].astype(np.uint8) rec_shape = ColouredTriMesh( masked_points * [1, 1, -1], trilist=plan_trilist, colours=img.pixels_with_channels_at_back() [color_sampling[:, 0], color_sampling[:, 1], :] # np.ones_like(masked_points) * [0,0,1] # ) return rec_shape
def fit(imagepath): image = mio.import_image(imagepath, normalize=False) if len(image.pixels.shape) == 2: image.pixels = np.stack([image.pixels, image.pixels, image.pixels]) if image.pixels.shape[0] == 1: image.pixels = np.concatenate( [image.pixels, image.pixels, image.pixels], axis=0) print(image.pixels_with_channels_at_back().shape) bb = detect(image.pixels_with_channels_at_back())[0] initial_shape = aam_fitter.fit_from_bb(image, bb).final_shape result = fitter.fit_from_shape(image, initial_shape, max_iters=40, camera_update=True, focal_length_update=False, reconstruction_weight=1, shape_prior_weight=.4e8, texture_prior_weight=1., landmarks_prior_weight=1e5, return_costs=True, init_shape_params_from_lms=False) mesh = ColouredTriMesh(result.final_mesh.points, result.final_mesh.trilist) def transform(mesh): return result._affine_transforms[-1].apply( result.camera_transforms[-1].apply(mesh)) mesh_in_img = transform(lambertian_shading(mesh)) expr_dir = image.path.parent p = image.path.stem raster = rasterize_mesh(mesh_in_img, image.shape) uv_shape = (600, 1000) template = shape_model.mean() unwrapped_template = optimal_cylindrical_unwrap(template).apply(template) minimum = unwrapped_template.bounds(boundary=0)[0] unwrapped_template = Translation(-minimum).apply(unwrapped_template) unwrapped_template.points = unwrapped_template.points[:, [1, 0]] unwrapped_template.points[:, 0] = unwrapped_template.points[:, 0].max( ) - unwrapped_template.points[:, 0] unwrapped_template.points *= np.array([.40, .31]) unwrapped_template.points *= np.array([uv_shape]) bcoords_img, tri_index_img = rasterize_barycentric_coordinate_images( unwrapped_template, uv_shape) TI = tri_index_img.as_vector() BC = bcoords_img.as_vector(keep_channels=True).T def masked_texture(mesh_in_image, background): sample_points_3d = mesh_in_image.project_barycentric_coordinates( BC, TI) texture = bcoords_img.from_vector( background.sample(sample_points_3d.points[:, :2])) return texture uv = masked_texture(mesh_in_img, image) t = TexturedTriMesh( result.final_mesh.points, image_coords_to_tcoords(uv.shape).apply(unwrapped_template).points, uv, mesh_in_img.trilist) m3io.export_textured_mesh(t, str(expr_dir / Path(p).with_suffix('.mesh.obj')), overwrite=True) mio.export_image(raster, str(expr_dir / Path(p).with_suffix('.render.jpg')), overwrite=True)
def visualize_nicp_weighting(template, weighting): colours = ((weighting[:, None] * np.array([1, 0, 0])) + ((1 - weighting[:, None]) * np.array([1, 1, 1]))) print('min: {}, max: {}'.format(weighting.min(), weighting.max())) ColouredTriMesh(template.points, trilist=template.trilist, colours=colours).view()
def load_mean_from_basel(path): mm = loadmat(str(path)) trilist = mm['tl'][:, [0, 2, 1]] - 1 mean_points = mm['shapeMU'].reshape(-1, 3) mean_colour = mm['texMU'].reshape(-1, 3) / 255 return ColouredTriMesh(mean_points, trilist=trilist, colours=mean_colour)
def coloured_trimesh_viewer_test(): ColouredTriMesh(fake_triangle, colours=fake_tcoords, trilist=fake_trilist, copy=False).view()
def test_coloured_trimesh_viewer(): with raises(Menpo3dMissingError): ColouredTriMesh(fake_triangle, colours=fake_tcoords, trilist=fake_trilist, copy=False).view()