def vis_verts(mean_shape, verts, face, mvs=None, textures=None):
"""
mean_shape: N x 3
verts: B x N x 3
face: numpy F x 3
textures: B x F x T x T (x T) x 3
"""
from psbody.mesh.mesh import Mesh
from psbody.mesh.meshviewer import MeshViewers
if mvs is None:
mvs = MeshViewers((2, 3))
num_row = len(mvs)
num_col = len(mvs[0])
mean_shape = convert2np(mean_shape)
verts = convert2np(verts)
num_show = min(num_row * num_col, verts.shape[0] + 1)
mvs[0][0].set_dynamic_meshes([Mesh(mean_shape, face)])
# 0th is mean shape:
if textures is not None:
tex = convert2np(textures)
for k in np.arange(1, num_show):
vert_here = verts[k - 1]
if textures is not None:
tex_here = tex[k - 1]
fc = tex_here.reshape(tex_here.shape[0], -1, 3).mean(axis=1)
mesh = Mesh(vert_here, face, fc=fc)
else:
mesh = Mesh(vert_here, face)
mvs[int(k % num_row)][int(k / num_row)].set_dynamic_meshes([mesh])
def test_aabb_tree(self):
v_src = np.array([[-36, 37, 8], [5, -36, 35], [12, -15, 1],
[-10, -42, -26], [-38, -32, -26], [-8, -45, 40],
[44, -1, -1], [-16, 40, -13], [-39, 28, -11],
[-26, -10, -40], [-37, 44, 46], [8, -44, -27],
[-15, 32, -48], [-46, -33, 15], [23, 15, -5],
[5, -20, 24], [-31, 19, -32], [-13, 13, 28],
[-42, 43, 28], [-1, -6, -5]])
f_src = np.array([[12, 16, 17], [5, 10, 1], [13, 19, 7], [13, 1, 5],
[14, 8, 16], [9, 2, 8], [1, 19, 18], [4, 0, 3],
[18, 15, 5], [3, 16, 2]])
m = Mesh(v=v_src, f=f_src)
t = m.compute_aabb_tree()
v_query = np.array([[-19, 1, 1], [32, 29, 14], [-12, 31, 3],
[-15, 44, 38], [5, 12, 9]])
v_expected = np.array([[-19.678178, 0.364208, -1.384218],
[23.000000, 15.000000, -5.000000],
[-13.729523, 19.930467, 0.278131],
[-31.869765, 34.228123, 44.656367],
[7.794764, 18.188195, -6.471474]])
f_expected = np.array([2, 4, 0, 1, 4])
f_est, v_est = t.nearest(v_query)
diff1 = abs(f_est - f_expected)
diff2 = abs(v_est - v_expected)
self.assertTrue(max(diff1.flatten()) < 1e-6)
self.assertTrue(max(diff2.flatten()) < 1e-6)
def test_estimate_vertex_normals(self):
# normals of a sphere should be scaled versions of the vertices
m = Mesh(filename=self.test_sphere_path)
m.v -= np.mean(m.v, axis=0)
rad = np.linalg.norm(m.v[0])
vn = np.array(m.estimate_vertex_normals())
mse = np.mean(np.sqrt(np.sum((vn - m.v / rad)**2, axis=1)))
self.assertTrue(mse < 0.05)
def test_writing_bin_ply(self):
m = Mesh(filename=self.test_ply_path)
(_, tempname) = tempfile.mkstemp()
m.write_ply(tempname)
with open(tempname, 'rb') as f:
candidate = f.read()
os.remove(tempname)
with open(self.test_bin_ply_path, 'rb') as f:
truth = f.read()
self.assertEqual(candidate, truth)
def test_vert_normals(self):
from psbody.mesh.geometry.vert_normals import VertNormals
from psbody.mesh.mesh import Mesh
mesh = Mesh(filename=os.path.join(test_data_folder, 'sphere.ply'))
pred = VertNormals(mesh.v, mesh.f)
vn_obs = mesh.estimate_vertex_normals().reshape((-1, 3))
vn_pred = pred.reshape((-1, 3))
self.assertTrue(np.max(np.abs(vn_pred.flatten() - vn_obs.flatten())) < 1e-15)
def setUp(self):
simpleobjpath = os.path.join(test_data_folder, 'test_doublebox.obj')
self.simple_m = Mesh(filename=simpleobjpath)
cylinderpath = os.path.join(test_data_folder, 'cylinder.obj')
self.cylinder_m = Mesh(filename=cylinderpath)
cylinder_trans_path = os.path.join(test_data_folder,
'cylinder_trans.obj')
self.cylinder_trans_m = Mesh(filename=cylinder_trans_path)
self_int_cyl_path = os.path.join(test_data_folder,
'self_intersecting_cyl.obj')
self.self_int_cyl_m = Mesh(filename=self_int_cyl_path)
def test_ascii_bad_ply(self):
"""Ensure that the proper exception is raised when a file fails to be read."""
with self.assertRaisesRegex(SerializationError, 'Unable to open file'):
Mesh(filename=self.test_bad_ply_path)
# The next two tests are unnecessary,
# just demonstrating the exception hierarchy:
with self.assertRaises(MeshError):
Mesh(filename=self.test_bad_ply_path)
with self.assertRaises(Exception):
Mesh(filename=self.test_bad_ply_path)
def test_qslim_smoke_test(self):
from psbody.mesh.mesh import Mesh
from psbody.mesh.topology.decimation import qslim_decimator
from psbody.mesh.geometry.triangle_area import triangle_area
m = Mesh(
filename=os.path.join(test_data_folder, 'female_template.ply'))
ta = triangle_area(m.v, m.f)
m.set_face_colors(ta / np.max(ta))
qslimmer = qslim_decimator(m, factor=0.1)
m2 = qslimmer(m)
ta = triangle_area(m2.v, m2.f)
m2.set_face_colors(ta / np.max(ta))
def test_load_obj(self):
m = Mesh(filename=self.test_obj_path)
self.assertTrue((m.v == self.box_v).all())
self.assertTrue((m.f == self.box_f).all())
self.assertDictOfArraysEqual(m.segm, self.box_segm)
self.assertEqual(m.landm, self.landm)
self.assertDictOfArraysEqual(m.landm_xyz, self.landm_xyz)
def setup_silhouette_obj(silhs, rends, f):
n_model = [ch.sum(rend[:, :, 0] > 0) for rend in rends]
dist_tsf = [cv2.distanceTransform(np.uint8(1 - silh), cv2.DIST_L2, cv2.DIST_MASK_PRECISE) for silh in silhs]
# Make sigma proportional to image area.
# This gives radius 400 for 960 x 1920 image.
sigma_ratio = 0.2727
sigma = [np.sqrt(sigma_ratio * (silh.shape[0] * silh.shape[1]) / np.pi) for silh in silhs]
# Model2silhouette ==> Consistency (contraction)
def obj_m2s(w, i):
return w * (GMOf(rends[i][:, :, 0] * dist_tsf[i], sigma[i]) / np.sqrt(n_model[i]))
# silhouette error term (scan-to-model) ==> Coverage (expansion)
coords = [np.fliplr(np.array(np.where(silh > 0)).T) + 0.5 for silh in silhs]# is this offset needed?
scan_flat_v = [np.hstack((coord, ch.zeros(len(coord)).reshape((-1, 1)))) for coord in coords]
scan_flat = [Mesh(v=sflat_v, f=[]) for sflat_v in scan_flat_v]
# 2d + all 0.
sv_flat = [ch.hstack((rend.camera, ch.zeros(len(rend.v)).reshape((-1, 1)))) for rend in rends]
for i in range(len(rends)):
sv_flat[i].f = f
def obj_s2m(w, i):
from sbody.mesh_distance import ScanToMesh
return w * ch.sqrt(GMOf(ScanToMesh(scan_flat[i], sv_flat[i], f), sigma[i]))
# For vis
for i in range(len(rends)):
scan_flat[i].vc = np.tile(np.array([0, 0, 1]), (len(scan_flat[i].v), 1))
return obj_m2s, obj_s2m, dist_tsf
def render_and_show(sv):
for i in range(len(rends)):
img = imgs[i]
img_res = render_mesh(Mesh(sv[i].r, sv[i].model.f), img.shape[1], img.shape[0], kp_camera[i], near=0.5, far=20)
plt.figure()
plt.imshow(img[:, :, ::-1])
plt.imshow(img_res)
plt.axis('off')
def test_connectivity_smoke_test(self):
from psbody.mesh import Mesh
from psbody.mesh.topology.connectivity import get_vert_connectivity, get_faces_per_edge
m = Mesh(
filename=os.path.join(test_data_folder, 'female_template.ply'))
vconn = get_vert_connectivity(m)
fpe = get_faces_per_edge(m)
self.assertIsNotNone(vconn)
self.assertIsNotNone(fpe)
def on_step(_):
global c
mesh = [None] * len(rends)
for i in range(len(rends)):
rend = rends[i]
img = imgs[i]
edges = cv2.Canny(np.uint8(rend[:, :, 0].r * 255), 100, 200)
coords = np.array(np.where(edges > 0)).T
plt.figure(200 + i, facecolor='w')
plt.clf()
plt.subplot(2, 2, 1)
plt.imshow(dist_tsf[i])
plt.plot(coords[:, 1], coords[:, 0], 'w.')
plt.axis('off')
plt.subplot(2, 2, 2)
plt.imshow(rend[:, :, 0].r)
plt.title('fitted silhouette')
plt.draw()
plt.axis('off')
if img is not None and kp_camera is not None and j2d is not None:
plt.subplot(2, 2, 3)
plt.imshow(img[:, :, ::-1])
plt.scatter(kp_camera[i].r[:, 0], kp_camera[i].r[:, 1])
plt.scatter(j2d[i][:, 0], j2d[i][:, 1], c='w')
plt.axis('off')
plt.subplot(2, 2, 4)
plt.imshow((silhs[i] + rend[:, :, 0].r) / 2.0)
plt.axis('off')
plt.draw()
plt.show(block=False)
plt.pause(1e-5)
#mv[i].set_static_meshes([Mesh(shape_model[i].r, shape_model[i].f)])
if False: #vc is not None:
vc1 = vc[i].r.copy()
vc1[:, 0] = vc[i].r.copy()[:, 2]
vc1[:, 2] = vc[i].r.copy()[:, 0]
mv[i].set_dynamic_meshes(
[Mesh(shape_model[i].r, shape_model[i].f, vc=vc1)])
vc2 = vc[i].r.copy()[symIdx, :]
vc2[:, 0] = vc[i].r.copy()[symIdx, 2]
vc2[:, 2] = vc[i].r.copy()[symIdx, 0]
mv2[i].set_dynamic_meshes(
[Mesh(shape_model[i].r, shape_model[i].f, vc=vc2)])
else:
vc[:, 0] = 0.8
vc[:, 1] = 0.76
vc[:, 2] = 0.77
mesh[i] = Mesh(shape_model[i].r, shape_model[i].f, vc=vc)
v = mesh[i].v.copy()
mesh[i].v[:, 1] = -v[:, 1]
mesh[i].v[:, 2] = -v[:, 2]
#mv[i].set_static_meshes([mesh])
mv2[0][i].set_static_meshes([mesh[i]])
#if c==0:
# import pdb; pdb.set_trace()
#mv2[0][i].save_snapshot('tmp_%.4d.png' % (c))
c = c + 1
def on_step(_):
for i in range(len(rends)):
rend = rends[i]
img = imgs[i]
edges = cv2.Canny(np.uint8(rend[:, :, 0].r * 255), 100, 200)
coords = np.array(np.where(edges > 0)).T
plt.figure(200+i, facecolor='w')
plt.clf()
plt.subplot(2, 2, 1)
plt.imshow(dist_tsf[i])
plt.plot(coords[:, 1], coords[:, 0], 'w.')
plt.axis('off')
plt.subplot(2, 2, 2)
plt.imshow(rend[:, :, 0].r)
plt.title('fitted silhouette')
plt.draw()
plt.axis('off')
if img is not None and kp_camera is not None:
plt.subplot(2, 2, 3)
plt.imshow(img[:, :, ::-1])
plt.scatter(kp_camera[i].r[:, 0], kp_camera[i].r[:, 1])
plt.axis('off')
plt.subplot(2, 2, 4)
plt.imshow((silhs[i]+rend[:, :, 0].r)/2.0)
plt.axis('off')
plt.draw()
plt.show(block=False)
plt.pause(1e-5)
if vc is not None:
vc1 = vc[i].r.copy()
vc1[:,0] = vc[i].r.copy()[:,2]
vc1[:,2] = vc[i].r.copy()[:,0]
mv[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].model.f, vc=vc1)])
vc2 = vc[i].r.copy()[symIdx,:]
vc2[:,0] = vc[i].r.copy()[symIdx,2]
vc2[:,2] = vc[i].r.copy()[symIdx,0]
mv2[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].model.f, vc=vc2)])
else:
mv[i].set_dynamic_meshes([Mesh(sv[i].r, sv[i].f)])
def test_launch_smoke_test(self):
"""this test just opens a mesh window, waits, and kills the window"""
sphere = Mesh(filename=os.path.join(test_data_folder, 'sphere.ply'))
sphere.v = sphere.v / 10.
print('keeping MeshViewer alive for 10 seconds..')
time.sleep(10)
print('killing MeshViewer and exiting...')
if 0:
# this cannot be unit tested
from psbody.mesh.utils import row
click = self.mvs[0][0].get_mouseclick()
subwin_row = click['which_subwindow'][0]
subwin_col = click['which_subwindow'][1]
sphere.v = sphere.v - row(np.mean(sphere.v, axis=0)) + row(
np.array([click['x'], click['y'], click['z']]))
self.mvs[subwin_row][subwin_col].set_dynamic_meshes([sphere])
print('items in mouseclick dict are as follows:')
print(click)
def setUp(self):
fnames = [
os.path.join(test_data_folder, i)
for i in os.listdir(test_data_folder)
if os.path.splitext(i)[1].lower() == '.ply'
]
self.meshes = [Mesh(filename=fname) for fname in fnames]
self.mvs = MeshViewers(shape=[2, 2])
self.mvs[0][0].set_static_meshes([self.meshes[0]])
self.mvs[0][1].set_static_meshes([self.meshes[1]])
self.mvs[1][0].set_static_meshes([self.meshes[2]])
self.mvs[1][1].set_static_meshes(
[self.meshes[0]]) # only 2 .ply files left in the GitHub version
def vis_vert2kp(verts, vert2kp, face, mvs=None):
"""
verts: N x 3
vert2kp: K x N
For each keypoint, visualize its weights on each vertex.
Base color is white, pick a color for each kp.
Using the weights, interpolate between base and color.
"""
from psbody.mesh.mesh import Mesh
from psbody.mesh.meshviewer import MeshViewer, MeshViewers
from psbody.mesh.sphere import Sphere
num_kp = vert2kp.shape[0]
if mvs is None:
mvs = MeshViewers((4, 4))
# mv = MeshViewer()
# Generate colors
import pylab
cm = pylab.get_cmap('gist_rainbow')
cms = 255 * np.array([cm(1. * i / num_kp)[:3] for i in range(num_kp)])
base = np.zeros((1, 3)) * 255
# base = np.ones((1, 3)) * 255
verts = convert2np(verts)
vert2kp = convert2np(vert2kp)
num_row = len(mvs)
num_col = len(mvs[0])
colors = []
for k in range(num_kp):
# Nx1 for this kp.
weights = vert2kp[k].reshape(-1, 1)
# So we can see it,,
weights = weights / weights.max()
cm = cms[k, None]
# Simple linear interpolation,,
# cs = np.uint8((1-weights) * base + weights * cm)
# In [0, 1]
cs = ((1 - weights) * base + weights * cm) / 255.
colors.append(cs)
# sph = [Sphere(center=jc, radius=.03).to_mesh(c/255.) for jc, c in zip(vert,cs)]
# mvs[int(k/4)][k%4].set_dynamic_meshes(sph)
mvs[int(k % num_row)][int(k / num_row)].set_dynamic_meshes(
[Mesh(verts, face, vc=cs)])
def setUp(self):
fnames = [
os.path.join(test_data_folder, i)
for i in os.listdir(test_data_folder)
if os.path.splitext(i)[1].lower() == '.ply'
]
# We build a cycle to make sure we have enough meshes
self.meshes = itertools.cycle(Mesh(filename=fname) for fname in fnames)
self.mvs = MeshViewers(shape=[2, 2])
self.mvs[0][0].set_static_meshes([next(self.meshes)])
self.mvs[0][1].set_static_meshes([next(self.meshes)])
self.mvs[1][0].set_static_meshes([next(self.meshes)])
self.mvs[1][1].set_static_meshes([next(self.meshes)])
def test_trinormal(self):
from psbody.mesh.mesh import Mesh
from psbody.mesh.geometry.tri_normals import TriNormals, TriToScaledNormal, TriNormalsScaled, NormalizeRows
m = Mesh(
filename=os.path.join(test_data_folder, 'female_template.ply'))
# Raffi: I do not know what this thing is supposed to test, maybe stability over some noise...
tn = TriNormals(m.v, m.f)
tn2 = NormalizeRows(TriToScaledNormal(m.v, m.f))
eps = 1e-8
mvc = m.v.copy()
mvc[0] += eps
tn_b = TriNormals(mvc, m.f)
tn2_b = NormalizeRows(TriToScaledNormal(mvc, m.f))
# our TriNormals empirical: sp.csc_matrix(tn_b.flatten() - tn.flatten()) / eps
# old TriToScaledNormals empirical': sp.csc_matrix(tn2_b.flatten() - tn2.flatten()) / eps
# apparently just for printing sparsly
# import scipy.sparse as sp
# print sp.csc_matrix(tn_b.flatten() - tn.flatten()) / eps
np.testing.assert_almost_equal(tn_b.flatten() - tn.flatten(),
tn2_b.flatten() - tn2.flatten())
tn = TriNormalsScaled(m.v, m.f)
tn2 = TriToScaledNormal(m.v, m.f)
eps = 1e-8
mvc = m.v.copy()
mvc[0] += eps
tn_b = TriNormalsScaled(mvc, m.f)
tn2_b = TriToScaledNormal(mvc, m.f)
np.testing.assert_almost_equal(tn_b.flatten() - tn.flatten(),
tn2_b.flatten() - tn2.flatten())
def test_raw_initialization(self):
m = Mesh(v=self.box_v, f=self.box_f)
self.assertTrue((m.v == self.box_v).all())
self.assertTrue((m.f == self.box_f).all())
def test_loop_subdivision_smoke_test(self):
from psbody.mesh import Mesh
from psbody.mesh.topology.subdivision import loop_subdivider
m1 = Mesh(
filename=os.path.join(test_data_folder, 'female_template.ply'))
sdv = loop_subdivider(m1)
self.assertIsNotNone(sdv)
self.assertTrue(hasattr(sdv, "faces"))
f_new = sdv.faces
v_new = sdv(m1.v)
self.assertIsNotNone(v_new)
v_new = v_new.reshape((-1, 3))
v_new_want_edge = sdv(m1.v, want_edges=True)
self.assertIsNotNone(v_new_want_edge)
v_new_want_edge = v_new_want_edge.reshape((-1, 3))
m2 = Mesh(v=v_new, f=f_new)
m1.reset_normals()
m2.reset_normals()
m1.write_ply(os.path.join(temporary_files_folder, 'lowres.ply'))
m2.write_ply(os.path.join(temporary_files_folder, 'highres.ply'))
if 0:
from psbody.mesh import MeshViewers
mvs = MeshViewers(shape=(2, 2))
mvs[0][0].set_static_meshes([m1])
m1.f = []
mvs[0][1].set_static_meshes([m1])
mvs[1][0].set_static_meshes([m2])
m2.f = []
mvs[1][1].set_static_meshes([m2])
def test_landmark_loader(self):
scan_fname = pjoin(test_data_folder, 'csr0001a.ply')
scan_lmrk = pjoin(test_data_folder, 'csr0001a.lmrk')
template_fname = pjoin(test_data_folder,
'textured_mean_scape_female.obj')
template_pp = pjoin(test_data_folder,
'template_caesar_picked_points.pp')
scan = Mesh(filename=scan_fname, lmrkfilename=scan_lmrk)
template = Mesh(filename=template_fname, ppfilename=template_pp)
# Detecting CAESAR lmrk files:
m = Mesh(filename=scan_fname, landmarks=scan_lmrk)
self.assertEqual(m.landm, scan.landm)
self.assertDictOfArraysEqual(m.landm_xyz, scan.landm_xyz)
# Detecting Meshlab pp file
m = Mesh(filename=template_fname, landmarks=template_pp)
self.assertEqual(m.landm, template.landm)
self.assertDictOfArraysAlmostEqual(m.landm_xyz, template.landm_xyz)
del template.landm_regressors
def test(landmarks):
m = Mesh(filename=template_fname, landmarks=landmarks)
self.assertEqual(m.landm, template.landm)
self.assertDictOfArraysAlmostEqual(m.landm_xyz, template.landm_xyz)
import json
import yaml
import pickle
tmp_dir = tempfile.mkdtemp('bodylabs-test')
test_files = [
(yaml, os.path.join(tmp_dir, 'landmarks.yaml'), 'w'),
(yaml, os.path.join(tmp_dir, 'landmarks.yml'), 'w'),
(json, os.path.join(tmp_dir, 'landmarks.json'), 'w'),
(pickle, os.path.join(tmp_dir, 'landmarks.pkl'), 'wb'),
]
test_data_ind = dict((n, int(v)) for n, v in template.landm.items())
test_data_xyz = dict(
(n, v.tolist()) for n, v in template.landm_xyz.items())
for loader, filename, mode in test_files:
with open(filename, mode) as fd:
loader.dump(test_data_ind, fd)
test(filename)
with open(filename, mode) as fd:
loader.dump(test_data_xyz, fd)
test(filename)
shutil.rmtree(tmp_dir, ignore_errors=True)
test(template.landm)
test(template.landm_xyz)
m = Mesh(filename=template_fname, landmarks=[0, 1, 2])
self.assertEqual(m.landm, {'0': 0, '1': 1, '2': 2})
m = Mesh(filename=template_fname,
landmarks=[template.v[0], template.v[7]])
self.assertDictOfArraysAlmostEqual(m.landm_xyz, {
'0': template.v[0],
'1': template.v[7]
})
m = Mesh(filename=template_fname,
landmarks=[template.v[0].tolist(), template.v[7].tolist()])
self.assertDictOfArraysAlmostEqual(m.landm_xyz, {
'0': template.v[0],
'1': template.v[7]
})
def test(landmarks):
m = Mesh(filename=template_fname, landmarks=landmarks)
self.assertEqual(m.landm, template.landm)
self.assertDictOfArraysAlmostEqual(m.landm_xyz, template.landm_xyz)
def test_load_ply(self):
m = Mesh(filename=self.test_ply_path, ppfilename=self.test_pp_path)
self.assertTrue((m.v == self.box_v).all())
self.assertTrue((m.f == self.box_f).all())
self.assertTrue(m.landm == self.landm)
