def per_vertex_occlusion(mesh_in_img, err_proportion=0.0001, render_diag=600): [x_r, y_r, z_r] = mesh_in_img.range() av_xy_r = (x_r + y_r) / 2.0 rescale = render_diag / np.sqrt((mesh_in_img.range()[:2] ** 2).sum()) rescale_z = av_xy_r / z_r mesh = Scale([rescale, rescale, rescale * rescale_z]).apply(mesh_in_img) mesh.points[...] = mesh.points - mesh.points.min(axis=0) mesh.points[:, :2] = mesh.points[:, :2] + 2 shape = np.around(mesh.points.max(axis=0)[:2] + 2) print("shape!!!:", shape) bc, ti = rasterize_barycentric_coordinate_images(mesh, shape) si = rasterize_shape_image_from_barycentric_coordinate_images( as_colouredtrimesh(mesh), bc, ti) # err_proportion=0.01 is 1% deviation of total range of 3D shape threshold = render_diag * err_proportion xyz_found = si.as_unmasked().sample(mesh.with_dims([0, 1]), order=1).T err = np.sum((xyz_found - mesh.points) ** 2, axis=1) visible = err < threshold return visible
def get_landmark_points(mesh, img_shape=(320, 240), verbose=False): fitter = load_balanced_frontal_face_fitter() detector = load_dlib_frontal_face_detector() camera = perspective_camera_for_template(img_shape) # Pre-process - align the mesh roughly with the template aligned_mesh = align_mesh_to_template(mesh, load_template()).apply(mesh) mesh_in_img = camera.apply(aligned_mesh) bcs = rasterize_barycentric_coordinate_images(mesh_in_img, img_shape) img = rasterize_mesh_from_barycentric_coordinate_images(mesh_in_img, *bcs) shape_img = rasterize_shape_image_from_barycentric_coordinate_images( mesh, *bcs) # 2. Find the one bounding box in the rendered image bboxes = detector(img) if len(bboxes) != 1: raise ValueError("Expected to find one face - found {}".format( len(bboxes))) else: if verbose: print('Detected 1 face') # 3. Fit from the bounding box fr = fitter.fit_from_bb(img, bboxes[0]) if verbose: print('AMM fitting successfully completed') # 4. Sample the XYZ image to build back the landmarks img_lms = fr.final_shape.from_mask(LANDMARK_MASK) # test to see if the landmark fell on the 3D surface or not occlusion_mask = img.mask.sample(img_lms).ravel() img.landmarks['__lsfm_on_surface'] = img_lms.from_mask(occlusion_mask) img.landmarks['__lsfm_off_surface'] = img_lms.from_mask(~occlusion_mask) return PointCloud(shape_img.sample(img.landmarks['__lsfm_on_surface']).T)
def landmark_template(mesh, img_shape=(320, 240), verbose=False): fitter = load_balanced_frontal_face_fitter() detector = load_dlib_frontal_face_detector() camera = perspective_camera_for_template(img_shape) # Pre-process - align the mesh roughly with the template aligned_mesh = prepare_template_reference_space(mesh) mesh_in_img = camera.apply(aligned_mesh) bcs = rasterize_barycentric_coordinate_images(mesh_in_img, img_shape) img = rasterize_mesh_from_barycentric_coordinate_images(mesh_in_img, *bcs) shape_img = rasterize_shape_image_from_barycentric_coordinate_images( mesh, *bcs) # 2. Find the one bounding box in the rendered image bboxes = detector(img) if len(bboxes) != 1: raise ValueError("Expected to find one face - found {}".format( len(bboxes))) else: if verbose: print('Detected 1 face') # 3. Fit from the bounding box fr = fitter.fit_from_bb(img, bboxes[0]) if verbose: print('AMM fitting successfully completed') # 4. Sample the XYZ image to build back the landmarks img_lms = fr.final_shape # test to see if the landmark fell on the 3D surface or not mesh.landmarks["ibug68"] = PointCloud(Image.sample(shape_img, img_lms).T) mask = np.zeros(68, dtype=np.bool) mask[30] = True mesh.landmarks["nosetip"] = mesh.landmarks["ibug68"].lms.from_mask(mask)
def landmark_mesh(mesh, img_shape=(320, 240), verbose=False, template_fn=None): fitter = load_balanced_frontal_face_fitter() detector = load_dlib_frontal_face_detector() camera = perspective_camera_for_template(img_shape) # Pre-process - align the mesh roughly with the template aligned_mesh = align_mesh_to_template(mesh, load_template(template_fn)).apply(mesh) mesh_in_img = camera.apply(aligned_mesh) bcs = rasterize_barycentric_coordinate_images(mesh_in_img, img_shape) img = rasterize_mesh_from_barycentric_coordinate_images(mesh_in_img, *bcs) shape_img = rasterize_shape_image_from_barycentric_coordinate_images( mesh, *bcs) # 2. Find the one bounding box in the rendered image bboxes = detector(img) if len(bboxes) != 1: raise ValueError( "Expected to find one face - found {}".format(len(bboxes))) else: if verbose: print('Detected 1 face') # 3. Fit from the bounding box fr = fitter.fit_from_bb(img, bboxes[0]) if verbose: print('AMM fitting successfully completed') # 4. Sample the XYZ image to build back the landmarks img_lms = fr.final_shape.from_mask(LANDMARK_MASK) # test to see if the landmark fell on the 3D surface or not occlusion_mask = img.mask.sample(img_lms).ravel() img.landmarks['__lsfm_on_surface'] = img_lms.from_mask(occlusion_mask) img.landmarks['__lsfm_off_surface'] = img_lms.from_mask(~occlusion_mask) return_dict = { 'landmarks_2d': img_lms, 'occlusion_mask': occlusion_mask, 'landmarks_3d_masked': PointCloud(shape_img.sample( img.landmarks['__lsfm_on_surface']).T) } if (~occlusion_mask).sum() != 0: groups = ['dlib_0', '__lsfm_on_surface', '__lsfm_off_surface'] marker_edge_colours = ['blue', 'yellow', 'red'] else: groups = ['dlib_0', '__lsfm_on_surface'] marker_edge_colours = ['blue', 'yellow'] lm_img = img.rasterize_landmarks(group=groups, line_colour='blue', marker_edge_colour=marker_edge_colours) return_dict['landmarked_image'] = lm_img return return_dict
def shape_mask(mesh): bcoords_img_inst, tri_index_img_inst = rasterize_barycentric_coordinate_images( mesh, [256, 256]) TI_inst = tri_index_img_inst.as_vector() BC_inst = bcoords_img_inst.as_vector(keep_channels=True).T sample_index = tri_index_img_inst.as_unmasked().sample( mesh.with_dims([0, 1])).squeeze() shape_mask_idx = np.unique(mesh.trilist[sample_index].flatten()) shape_mask_arr = np.zeros([mesh.n_points]) shape_mask_arr[shape_mask_idx] += 1 return shape_mask_arr > 0
def fit(imagepath): image = mio.import_image(imagepath, normalize=False) if len(image.pixels.shape) == 2: image.pixels = np.stack([image.pixels, image.pixels, image.pixels]) if image.pixels.shape[0] == 1: image.pixels = np.concatenate( [image.pixels, image.pixels, image.pixels], axis=0) print(image.pixels_with_channels_at_back().shape) bb = detect(image.pixels_with_channels_at_back())[0] initial_shape = aam_fitter.fit_from_bb(image, bb).final_shape result = fitter.fit_from_shape(image, initial_shape, max_iters=40, camera_update=True, focal_length_update=False, reconstruction_weight=1, shape_prior_weight=.4e8, texture_prior_weight=1., landmarks_prior_weight=1e5, return_costs=True, init_shape_params_from_lms=False) mesh = ColouredTriMesh(result.final_mesh.points, result.final_mesh.trilist) def transform(mesh): return result._affine_transforms[-1].apply( result.camera_transforms[-1].apply(mesh)) mesh_in_img = transform(lambertian_shading(mesh)) expr_dir = image.path.parent p = image.path.stem raster = rasterize_mesh(mesh_in_img, image.shape) uv_shape = (600, 1000) template = shape_model.mean() unwrapped_template = optimal_cylindrical_unwrap(template).apply(template) minimum = unwrapped_template.bounds(boundary=0)[0] unwrapped_template = Translation(-minimum).apply(unwrapped_template) unwrapped_template.points = unwrapped_template.points[:, [1, 0]] unwrapped_template.points[:, 0] = unwrapped_template.points[:, 0].max( ) - unwrapped_template.points[:, 0] unwrapped_template.points *= np.array([.40, .31]) unwrapped_template.points *= np.array([uv_shape]) bcoords_img, tri_index_img = rasterize_barycentric_coordinate_images( unwrapped_template, uv_shape) TI = tri_index_img.as_vector() BC = bcoords_img.as_vector(keep_channels=True).T def masked_texture(mesh_in_image, background): sample_points_3d = mesh_in_image.project_barycentric_coordinates( BC, TI) texture = bcoords_img.from_vector( background.sample(sample_points_3d.points[:, :2])) return texture uv = masked_texture(mesh_in_img, image) t = TexturedTriMesh( result.final_mesh.points, image_coords_to_tcoords(uv.shape).apply(unwrapped_template).points, uv, mesh_in_img.trilist) m3io.export_textured_mesh(t, str(expr_dir / Path(p).with_suffix('.mesh.obj')), overwrite=True) mio.export_image(raster, str(expr_dir / Path(p).with_suffix('.render.jpg')), overwrite=True)