def per_vertex_occlusion(mesh_in_img, err_proportion=0.0001, render_diag=600): [x_r, y_r, z_r] = mesh_in_img.range() av_xy_r = (x_r + y_r) / 2.0 rescale = render_diag / np.sqrt((mesh_in_img.range()[:2] ** 2).sum()) rescale_z = av_xy_r / z_r mesh = Scale([rescale, rescale, rescale * rescale_z]).apply(mesh_in_img) mesh.points[...] = mesh.points - mesh.points.min(axis=0) mesh.points[:, :2] = mesh.points[:, :2] + 2 shape = np.around(mesh.points.max(axis=0)[:2] + 2) print("shape!!!:", shape) bc, ti = rasterize_barycentric_coordinate_images(mesh, shape) si = rasterize_shape_image_from_barycentric_coordinate_images( as_colouredtrimesh(mesh), bc, ti) # err_proportion=0.01 is 1% deviation of total range of 3D shape threshold = render_diag * err_proportion xyz_found = si.as_unmasked().sample(mesh.with_dims([0, 1]), order=1).T err = np.sum((xyz_found - mesh.points) ** 2, axis=1) visible = err < threshold return visible
def get_landmark_points(mesh, img_shape=(320, 240), verbose=False): fitter = load_balanced_frontal_face_fitter() detector = load_dlib_frontal_face_detector() camera = perspective_camera_for_template(img_shape) # Pre-process - align the mesh roughly with the template aligned_mesh = align_mesh_to_template(mesh, load_template()).apply(mesh) mesh_in_img = camera.apply(aligned_mesh) bcs = rasterize_barycentric_coordinate_images(mesh_in_img, img_shape) img = rasterize_mesh_from_barycentric_coordinate_images(mesh_in_img, *bcs) shape_img = rasterize_shape_image_from_barycentric_coordinate_images( mesh, *bcs) # 2. Find the one bounding box in the rendered image bboxes = detector(img) if len(bboxes) != 1: raise ValueError("Expected to find one face - found {}".format( len(bboxes))) else: if verbose: print('Detected 1 face') # 3. Fit from the bounding box fr = fitter.fit_from_bb(img, bboxes[0]) if verbose: print('AMM fitting successfully completed') # 4. Sample the XYZ image to build back the landmarks img_lms = fr.final_shape.from_mask(LANDMARK_MASK) # test to see if the landmark fell on the 3D surface or not occlusion_mask = img.mask.sample(img_lms).ravel() img.landmarks['__lsfm_on_surface'] = img_lms.from_mask(occlusion_mask) img.landmarks['__lsfm_off_surface'] = img_lms.from_mask(~occlusion_mask) return PointCloud(shape_img.sample(img.landmarks['__lsfm_on_surface']).T)
def landmark_template(mesh, img_shape=(320, 240), verbose=False): fitter = load_balanced_frontal_face_fitter() detector = load_dlib_frontal_face_detector() camera = perspective_camera_for_template(img_shape) # Pre-process - align the mesh roughly with the template aligned_mesh = prepare_template_reference_space(mesh) mesh_in_img = camera.apply(aligned_mesh) bcs = rasterize_barycentric_coordinate_images(mesh_in_img, img_shape) img = rasterize_mesh_from_barycentric_coordinate_images(mesh_in_img, *bcs) shape_img = rasterize_shape_image_from_barycentric_coordinate_images( mesh, *bcs) # 2. Find the one bounding box in the rendered image bboxes = detector(img) if len(bboxes) != 1: raise ValueError("Expected to find one face - found {}".format( len(bboxes))) else: if verbose: print('Detected 1 face') # 3. Fit from the bounding box fr = fitter.fit_from_bb(img, bboxes[0]) if verbose: print('AMM fitting successfully completed') # 4. Sample the XYZ image to build back the landmarks img_lms = fr.final_shape # test to see if the landmark fell on the 3D surface or not mesh.landmarks["ibug68"] = PointCloud(Image.sample(shape_img, img_lms).T) mask = np.zeros(68, dtype=np.bool) mask[30] = True mesh.landmarks["nosetip"] = mesh.landmarks["ibug68"].lms.from_mask(mask)
def landmark_mesh(mesh, img_shape=(320, 240), verbose=False, template_fn=None): fitter = load_balanced_frontal_face_fitter() detector = load_dlib_frontal_face_detector() camera = perspective_camera_for_template(img_shape) # Pre-process - align the mesh roughly with the template aligned_mesh = align_mesh_to_template(mesh, load_template(template_fn)).apply(mesh) mesh_in_img = camera.apply(aligned_mesh) bcs = rasterize_barycentric_coordinate_images(mesh_in_img, img_shape) img = rasterize_mesh_from_barycentric_coordinate_images(mesh_in_img, *bcs) shape_img = rasterize_shape_image_from_barycentric_coordinate_images( mesh, *bcs) # 2. Find the one bounding box in the rendered image bboxes = detector(img) if len(bboxes) != 1: raise ValueError( "Expected to find one face - found {}".format(len(bboxes))) else: if verbose: print('Detected 1 face') # 3. Fit from the bounding box fr = fitter.fit_from_bb(img, bboxes[0]) if verbose: print('AMM fitting successfully completed') # 4. Sample the XYZ image to build back the landmarks img_lms = fr.final_shape.from_mask(LANDMARK_MASK) # test to see if the landmark fell on the 3D surface or not occlusion_mask = img.mask.sample(img_lms).ravel() img.landmarks['__lsfm_on_surface'] = img_lms.from_mask(occlusion_mask) img.landmarks['__lsfm_off_surface'] = img_lms.from_mask(~occlusion_mask) return_dict = { 'landmarks_2d': img_lms, 'occlusion_mask': occlusion_mask, 'landmarks_3d_masked': PointCloud(shape_img.sample( img.landmarks['__lsfm_on_surface']).T) } if (~occlusion_mask).sum() != 0: groups = ['dlib_0', '__lsfm_on_surface', '__lsfm_off_surface'] marker_edge_colours = ['blue', 'yellow', 'red'] else: groups = ['dlib_0', '__lsfm_on_surface'] marker_edge_colours = ['blue', 'yellow'] lm_img = img.rasterize_landmarks(group=groups, line_colour='blue', marker_edge_colour=marker_edge_colours) return_dict['landmarked_image'] = lm_img return return_dict