def instance(self,
model_type='bfm',
alpha=None,
beta=None,
landmark_group='ibug68'):
if alpha is None:
alpha = np.zeros(len(self.shape_model.eigenvalues))
if beta is None:
beta = np.zeros(len(self.texture_model.eigenvalues))
# Generate instance
shape = self.shape_model.instance(alpha, normalized_weights=True)
texture = self.texture_model.instance(beta, normalized_weights=True)
if model_type == 'bfm':
tex_scale = UniformScale(1. / 255, 3)
lms_scale = UniformScale(1e-5, 3)
texture = tex_scale.apply(texture.reshape([-1, 3]))
landmarks = lms_scale.apply(self.landmarks)
elif model_type == 'lsfm':
pass
trimesh = ColouredTriMesh(shape.points,
trilist=shape.trilist,
colours=texture)
trimesh.landmarks[landmark_group] = landmarks
return trimesh
def generate_template_from_basel_and_metadata(basel, meta):
template = ColouredTriMesh(
basel.points[meta['map_tddfa_to_basel']],
trilist=meta['tddfa_trilist'],
colours=basel.colours[meta['map_tddfa_to_basel']])
template.landmarks['ibug68'] = meta['landmarks']['ibug68']
template.landmarks['nosetip'] = meta['landmarks']['nosetip']
return template
def _instance(self, shape_instance, texture_instance, landmark_group):
# Reshape the texture instance
texture_instance = texture_instance.reshape([-1, self.n_channels])
# restrict the texture to 0-1
# texture_instance = np.clip(texture_instance, 0, 1)
# Create trimesh
trimesh = ColouredTriMesh(shape_instance.points,
trilist=shape_instance.trilist,
colours=texture_instance)
# Attach landmarks to trimesh
trimesh.landmarks[landmark_group] = self.landmarks
# Return trimesh
return trimesh
def _instance(self, shape_instance, texture_instance, landmark_group):
# Reshape the texture instance
texture_instance = texture_instance.reshape([-1, self.n_channels])
# restrict the texture to 0-1
# texture_instance = np.clip(texture_instance, 0, 1)
# Create trimesh
trimesh = ColouredTriMesh(shape_instance.points,
trilist=shape_instance.trilist,
colours=texture_instance)
# Attach landmarks to trimesh
trimesh.landmarks[landmark_group] = self.landmarks
# Return trimesh
return trimesh
def test_colouredtrimesh_copy():
points = np.ones([10, 3])
colours = np.ones([10, 3])
trilist = np.ones([10, 3])
landmarks = PointCloud(np.ones([3, 3]), copy=False)
ctmesh = ColouredTriMesh(points, trilist=trilist, colours=colours,
copy=False)
ctmesh.landmarks['test'] = landmarks
ctmesh_copy = ctmesh.copy()
assert (not is_same_array(ctmesh_copy.points, ctmesh.points))
assert (not is_same_array(ctmesh_copy.trilist, ctmesh.trilist))
assert (not is_same_array(ctmesh_copy.colours, ctmesh.colours))
assert (not is_same_array(ctmesh_copy.landmarks['test'].lms.points,
ctmesh.landmarks['test'].lms.points))
def test_colouredtrimesh_init_2d_grid():
tm = ColouredTriMesh.init_2d_grid([10, 10])
assert tm.n_points == 100
assert tm.n_dims == 2
# 162 = 9 * 9 * 2
assert_allclose(tm.trilist.shape, (162, 3))
assert_allclose(tm.range(), [9, 9])
def render_mesh(sample_mesh, res=256, scale=1):
if sample_mesh.shape[-1] != 3:
sample_colours = sample_mesh[..., 3:]
else:
sample_colours = np.ones_like(sample_mesh) * [0, 0, 1]
sample_mesh = sample_mesh[..., :3]
sample_mesh = Homogeneous(
dm.utils.rotation_matrix(np.deg2rad(90),
[0, 0, -1])).apply(sample_mesh)
sample_mesh = ColouredTriMesh(sample_mesh * scale * res / 2 +
res / 2,
trilist=trilist,
colours=sample_colours)
sample_mesh = lambertian_shading(sample_mesh, ambient_colour=0)
store_path = Path(LOGDIR) / str(epoch)
if not store_path.exists():
store_path.mkdir()
m3io.export_mesh(sample_mesh,
store_path / '{}.obj'.format(time.time()))
mesh_img = rasterize_mesh(sample_mesh, [res, res])
mesh_img = mesh_img.rotate_ccw_about_centre(180)
return mesh_img.pixels_with_channels_at_back()
def test_colouredtrimesh_init_2d_grid():
tm = ColouredTriMesh.init_2d_grid([10, 10])
assert tm.n_points == 100
assert tm.n_dims == 2
# 162 = 9 * 9 * 2
assert_allclose(tm.trilist.shape, (162, 3))
assert_allclose(tm.range(), [9, 9])
def test_colouredtrimesh_creation_copy_true():
points = np.array([[0, 0, 0], [1, 0, 0], [1, 1, 0], [0, 1, 0]])
trilist = np.array([[0, 1, 3], [1, 2, 3]])
colours = np.ones([4, 13])
ttm = ColouredTriMesh(points, trilist=trilist, colours=colours, copy=True)
assert (not is_same_array(ttm.points, points))
assert (not is_same_array(ttm.trilist, trilist))
assert (not is_same_array(ttm.colours, colours))
def test_colouredtrimesh_init_from_depth_image():
fake_z = np.random.uniform(size=(10, 10))
tm = ColouredTriMesh.init_from_depth_image(Image(fake_z))
assert tm.n_points == 100
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [9, 9])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def test_colouredtrimesh_init_from_depth_image():
fake_z = np.random.uniform(size=(10, 10))
tm = ColouredTriMesh.init_from_depth_image(Image(fake_z))
assert tm.n_points == 100
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [9, 9])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def upsample_eos_low_res_to_fw_no_texture(eos_mesh_low_res):
bc_fw_on_eos_low_res, tri_index_fw_on_eos_low_res = load_fw_on_eos_low_res_settings(
)
effective_fw_pc = eos_mesh_low_res.project_barycentric_coordinates(
bc_fw_on_eos_low_res, tri_index_fw_on_eos_low_res)
effective_fw = ColouredTriMesh(effective_fw_pc.points,
trilist=load_basel_kf_trilist())
return effective_fw
def uv_in_img(image, mesh, uv_template):
from menpo3d.rasterize import rasterize_mesh
shape = image.shape
u_mesh = ColouredTriMesh(
mesh.points,
trilist=mesh.trilist,
colours=uv_template.points[:, 0, None])
u_image = rasterize_mesh(u_mesh, shape)
v_mesh = ColouredTriMesh(
mesh.points,
trilist=mesh.trilist,
colours=uv_template.points[:, 1, None])
v_image = rasterize_mesh(v_mesh, shape)
IUV_image = Image(np.concatenate(
[u_image.mask.pixels, u_image.pixels / 255., v_image.pixels / 255.]).clip(0, 1))
return IUV_image
def render_mesh(sample_mesh,
trilist=None,
scale=128,
offset=128,
shape=[256, 256],
store_path=None,
return_image=True,
**kwargs):
if isinstance(sample_mesh, ColouredTriMesh):
sample_mesh = sample_mesh
elif isinstance(sample_mesh, TriMesh):
sample_mesh = ColouredTriMesh(sample_mesh.points,
trilist=sample_mesh.trilist)
sample_mesh = lambertian_shading(sample_mesh, **kwargs)
else:
if sample_mesh.shape[-1] != 3:
sample_colours = sample_mesh[..., 3:]
else:
sample_colours = np.ones_like(sample_mesh) * [0, 0, 1]
sample_mesh = sample_mesh[..., :3]
sample_mesh = ColouredTriMesh(sample_mesh * scale + offset,
trilist=trilist,
colours=sample_colours)
sample_mesh = lambertian_shading(sample_mesh, **kwargs)
mesh_img = rasterize_mesh(sample_mesh, shape)
# mesh_img = mesh_img.rotate_ccw_about_centre(180)
if store_path:
if not store_path.exists():
store_path.mkdir()
m3io.export_mesh(sample_mesh,
store_path / '{}.obj'.format(time.time()))
if return_image:
return mesh_img
return mesh_img.pixels_with_channels_at_back()
def test_colouredtrimesh_init_from_depth_image_masked():
fake_z = np.random.uniform(size=(10, 10))
mask = np.zeros(fake_z.shape, dtype=np.bool)
mask[2:6, 2:6] = True
im = MaskedImage(fake_z, mask=mask)
tm = ColouredTriMesh.init_from_depth_image(im)
assert tm.n_points == 16
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [3, 3])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def _construct_shape_type(points, trilist, tcoords, texture,
colour_per_vertex):
r"""
Construct the correct Shape subclass given the inputs. TexturedTriMesh
can only be created when tcoords and texture are available. ColouredTriMesh
can only be created when colour_per_vertex is non None and TriMesh
can only be created when trilist is non None. Worst case fall back is
PointCloud.
Parameters
----------
points : ``(N, D)`` `ndarray`
The N-D points.
trilist : ``(N, 3)`` `ndarray`` or ``None``
Triangle list or None.
tcoords : ``(N, 2)`` `ndarray` or ``None``
Texture coordinates.
texture : :map:`Image` or ``None``
Texture.
colour_per_vertex : ``(N, 1)`` or ``(N, 3)`` `ndarray` or ``None``
The colour per vertex.
Returns
-------
shape : :map:`PointCloud` or subclass
The correct shape for the given inputs.
"""
# Four different outcomes - either we have a textured mesh, a coloured
# mesh or just a plain mesh or we fall back to a plain pointcloud.
if trilist is None:
obj = PointCloud(points, copy=False)
elif tcoords is not None and texture is not None:
obj = TexturedTriMesh(points,
tcoords,
texture,
trilist=trilist,
copy=False)
elif colour_per_vertex is not None:
obj = ColouredTriMesh(points,
trilist=trilist,
colours=colour_per_vertex,
copy=False)
else:
# TriMesh fall through
obj = TriMesh(points, trilist=trilist, copy=False)
if tcoords is not None and texture is None:
warnings.warn('tcoords were found, but no texture was recovered, '
'reverting to an untextured mesh.')
if texture is not None and tcoords is None:
warnings.warn('texture was found, but no tcoords were recovered, '
'reverting to an untextured mesh.')
return obj
def test_colouredtrimesh_init_from_depth_image_masked():
fake_z = np.random.uniform(size=(10, 10))
mask = np.zeros(fake_z.shape, dtype=np.bool)
mask[2:6, 2:6] = True
im = MaskedImage(fake_z, mask=mask)
tm = ColouredTriMesh.init_from_depth_image(im)
assert tm.n_points == 16
assert tm.n_dims == 3
assert_allclose(tm.range()[:2], [3, 3])
assert tm.points[:, -1].max() <= 1.0
assert tm.points[:, -1].min() >= 0.0
def save_template_from_mesh(path):
points, trilist = lio.getTriMeshfromPly(path)
template = ColouredTriMesh(points, trilist)
#to obtain landmark
landmark_68 = []
landmark_100 = []
landmark_ear = []
nosetip = []
count = 0
with open(str(DATA_DIR / 'ibug100.pp')) as pp_file:
pp_file = csv.reader(pp_file, delimiter='"')
for row in pp_file:
count = count + 1
if len(row) >= 10:
for i in range(0, 10, 2):
if row[i] == " x=":
lm_x = row[i + 1]
if row[i] == " y=":
lm_y = row[i + 1]
if row[i] == " z=":
lm_z = row[i + 1]
if count < 107:
landmark_100.append(
[float(lm_x), float(lm_y),
float(lm_z)])
if count < 75:
landmark_68.append([float(lm_x), float(lm_y), float(lm_z)])
if count >= 84 and count <= 94:
landmark_ear.append(
[float(lm_x), float(lm_y),
float(lm_z)])
if count >= 96 and count <= 106:
landmark_ear.append(
[float(lm_x), float(lm_y),
float(lm_z)])
count = 0
with open(str(DATA_DIR / 'nosetip.pp')) as pp_file:
pp_file = csv.reader(pp_file, delimiter='"')
for row in pp_file:
count = count + 1
if len(row) >= 10:
for i in range(0, 10, 2):
if row[i] == " x=":
lm_x = row[i + 1]
if row[i] == " y=":
lm_y = row[i + 1]
if row[i] == " z=":
lm_z = row[i + 1]
if count < 8:
nosetip.append([float(lm_x), float(lm_y), float(lm_z)])
template.landmarks['ibug68'] = PointCloud(np.array(landmark_68))
template.landmarks['ibug100'] = PointCloud(np.array(landmark_100))
template.landmarks['ibugEar'] = PointCloud(np.array(landmark_ear))
template.landmarks['nosetip'] = PointCloud(np.array(nosetip))
save_template(template, overwrite=True)
def import_mesh_as_color_trimesh(path):
img = None
h, w = 0,0
mean_points = []
trilist = []
vt_colors = []
map3dto2d = {}
with open(path, "r") as f:
for line in f:
if line.startswith("mtllib"):
# Get texture
mtl_path = path.parent / line.strip().split()[-1]
mtl = open(mtl_path, "r")
for l in mtl:
if l.startswith("map_Kd"):
img_path = str(mtl_path.parent / l.strip().split()[-1])
print("Read image at", img_path)
img = cv2.imread(img_path)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
h,w, _ = img.shape
mtl.close()
if line.startswith("v "):
tokens = line.strip().split()
x, y, z = float(tokens[1]), float(tokens[2]), float(tokens[3])
mean_points.append([x,y,z])
if line.startswith("vt "):
tokens = line.strip().split()
x, y = float(tokens[1]), float(tokens[2])
vt_colors.append(img[int((1-y)*h), int(x*w)].tolist())
if line.startswith("f "):
tokens = line.strip().split()
tripoints = []
for comp in tokens[1:]:
pts = comp.split("/")
tripoints.append(int(pts[0]) - 1)
map3dto2d[int(pts[0])-1] = int(pts[1])-1
trilist.append(tripoints[:3])
mean_colour = [None] * len(mean_points)
for k, v in map3dto2d.items():
mean_colour[k] = vt_colors[v]
mean_points = np.array(mean_points)
trilist = np.array(trilist)
mean_colour = np.array(mean_colour) / 255
return ColouredTriMesh(mean_points, trilist=trilist, colours=mean_colour)
def import_coloured_mesh(fp):
points = []
colours = []
trilist = []
with open(fp, 'r') as f:
for l in f.readlines():
if l.startswith('v '):
pts = list(map(float, l.strip().split(' ')[1:7]))
points.append(pts[:3])
colours.append(pts[3:])
elif l.startswith('f '):
faces = [
int(f.split('/')[0]) - 1 for f in l.strip().split(' ')[1:4]
]
trilist.append(faces)
return ColouredTriMesh(np.array(points),
trilist=np.array(trilist),
colours=np.array(colours))
def as_colouredtrimesh(self, colours=None, copy=True):
"""
Converts this to a :map:`ColouredTriMesh`.
Parameters
----------
colours : ``(N, 3)`` `ndarray`, optional
The floating point RGB colour per vertex. If not given, grey will be
assigned to each vertex.
copy : `bool`, optional
If ``True``, the graph will be a copy.
Returns
-------
coloured : :map:`ColouredTriMesh`
A version of this mesh with per-vertex colour assigned.
"""
ctm = ColouredTriMesh(self.points,
trilist=self.trilist,
colours=colours,
copy=copy)
return copy_landmarks_and_path(self, ctm)
def recover_mesh_from_uvxyz(uvxyz, img, template_mask, trilist=None):
mu = np.mean(uvxyz[template_mask], axis=0)
std = np.std(uvxyz[template_mask], axis=0)
nd = norm(mu, std)
nd_mask = (nd.cdf(uvxyz[template_mask]) < [0.05, 0.05, 0.01]).any(axis=-1)
plan_points = np.stack(np.meshgrid(*map(range, [256, 256])) +
[np.zeros([256, 256])],
axis=-1)[template_mask][~nd_mask]
if trilist is not None:
plan_trilist = trilist
else:
plan_trilist = TriMesh(plan_points[:, :2]).trilist
masked_points = uvxyz[template_mask][~nd_mask].clip(0, 1) * 256
color_sampling = masked_points[:, :2].astype(np.uint8)
rec_shape = ColouredTriMesh(
masked_points * [1, 1, -1],
trilist=plan_trilist,
colours=img.pixels_with_channels_at_back()
[color_sampling[:, 0],
color_sampling[:, 1], :] # np.ones_like(masked_points) * [0,0,1] #
)
return rec_shape
def fit(imagepath):
image = mio.import_image(imagepath, normalize=False)
if len(image.pixels.shape) == 2:
image.pixels = np.stack([image.pixels, image.pixels, image.pixels])
if image.pixels.shape[0] == 1:
image.pixels = np.concatenate(
[image.pixels, image.pixels, image.pixels], axis=0)
print(image.pixels_with_channels_at_back().shape)
bb = detect(image.pixels_with_channels_at_back())[0]
initial_shape = aam_fitter.fit_from_bb(image, bb).final_shape
result = fitter.fit_from_shape(image,
initial_shape,
max_iters=40,
camera_update=True,
focal_length_update=False,
reconstruction_weight=1,
shape_prior_weight=.4e8,
texture_prior_weight=1.,
landmarks_prior_weight=1e5,
return_costs=True,
init_shape_params_from_lms=False)
mesh = ColouredTriMesh(result.final_mesh.points, result.final_mesh.trilist)
def transform(mesh):
return result._affine_transforms[-1].apply(
result.camera_transforms[-1].apply(mesh))
mesh_in_img = transform(lambertian_shading(mesh))
expr_dir = image.path.parent
p = image.path.stem
raster = rasterize_mesh(mesh_in_img, image.shape)
uv_shape = (600, 1000)
template = shape_model.mean()
unwrapped_template = optimal_cylindrical_unwrap(template).apply(template)
minimum = unwrapped_template.bounds(boundary=0)[0]
unwrapped_template = Translation(-minimum).apply(unwrapped_template)
unwrapped_template.points = unwrapped_template.points[:, [1, 0]]
unwrapped_template.points[:, 0] = unwrapped_template.points[:, 0].max(
) - unwrapped_template.points[:, 0]
unwrapped_template.points *= np.array([.40, .31])
unwrapped_template.points *= np.array([uv_shape])
bcoords_img, tri_index_img = rasterize_barycentric_coordinate_images(
unwrapped_template, uv_shape)
TI = tri_index_img.as_vector()
BC = bcoords_img.as_vector(keep_channels=True).T
def masked_texture(mesh_in_image, background):
sample_points_3d = mesh_in_image.project_barycentric_coordinates(
BC, TI)
texture = bcoords_img.from_vector(
background.sample(sample_points_3d.points[:, :2]))
return texture
uv = masked_texture(mesh_in_img, image)
t = TexturedTriMesh(
result.final_mesh.points,
image_coords_to_tcoords(uv.shape).apply(unwrapped_template).points, uv,
mesh_in_img.trilist)
m3io.export_textured_mesh(t,
str(expr_dir / Path(p).with_suffix('.mesh.obj')),
overwrite=True)
mio.export_image(raster,
str(expr_dir / Path(p).with_suffix('.render.jpg')),
overwrite=True)
def visualize_nicp_weighting(template, weighting):
colours = ((weighting[:, None] * np.array([1, 0, 0])) +
((1 - weighting[:, None]) * np.array([1, 1, 1])))
print('min: {}, max: {}'.format(weighting.min(), weighting.max()))
ColouredTriMesh(template.points, trilist=template.trilist,
colours=colours).view()
def load_mean_from_basel(path):
mm = loadmat(str(path))
trilist = mm['tl'][:, [0, 2, 1]] - 1
mean_points = mm['shapeMU'].reshape(-1, 3)
mean_colour = mm['texMU'].reshape(-1, 3) / 255
return ColouredTriMesh(mean_points, trilist=trilist, colours=mean_colour)
def coloured_trimesh_viewer_test():
ColouredTriMesh(fake_triangle,
colours=fake_tcoords,
trilist=fake_trilist,
copy=False).view()
def test_coloured_trimesh_viewer():
with raises(Menpo3dMissingError):
ColouredTriMesh(fake_triangle,
colours=fake_tcoords,
trilist=fake_trilist,
copy=False).view()