def lambertian_spotlight(v, vn, pos, dir, spot_exponent, camcoord=False, camera_t=None, camera_rt=None):
"""
:param v: vertices
:param vn: vertex normals
:param light_pos: light position
:param light_dir: light direction
:param spot_exponent: spot exponent (a la opengl)
:param camcoord: if True, then pos and dir are wrt the camera
:param camera_t: 3-vector indicating translation of camera
:param camera_rt: 3-vector indicating direction of camera
:return: Vx1 array of brightness
"""
if camcoord: # Transform pos and dir from camera to world coordinate system
assert(camera_t is not None and camera_rt is not None)
from opendr.geometry import Rodrigues
rot = Rodrigues(rt=camera_rt)
pos = rot.T.dot(pos-camera_t)
dir = rot.T.dot(dir)
dir = dir / ch.sqrt(ch.sum(dir**2.))
v_minus_light = v - pos.reshape((1,3))
v_distances = ch.sqrt(ch.sum(v_minus_light**2, axis=1))
v_minus_light_normed = v_minus_light / v_distances.reshape((-1,1))
cosangle = v_minus_light_normed.dot(dir.reshape((3,1)))
light_dot_normal = ch.sum(vn*v_minus_light_normed, axis=1)
light_dot_normal.label = 'light_dot_normal'
cosangle.label = 'cosangle'
result = light_dot_normal.ravel() * cosangle.ravel()**spot_exponent
result = result / v_distances ** 2.
result = maximum(result, 0.0)
return result
def _set_up(self):
self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template
body_height = (self.v_shaped[2802, 1] + self.v_shaped[6262, 1]) - (
self.v_shaped[2237, 1] + self.v_shaped[6728, 1])
# print('111111111111111111111111111111111111111')
# print(np.array(body_height)[0])
# print(type(body_height))
# print('22222222222222222222222222222222222222222')
self.scale = 1.66 / np.array(body_height)[0]
self.v_shaped_personal = self.scale * self.v_shaped + self.v_personal
if sp.issparse(self.J_regressor):
self.J = self.scale * sp_dot(self.J_regressor, self.v_shaped)
else:
self.J = self.scale * ch.sum(self.J_regressor.T.reshape(-1, 1, 24)
* self.v_shaped.reshape(-1, 3, 1),
axis=0).T
self.v_posevariation = self.posedirs.dot(
posemap(self.bs_type)(self.pose))
self.v_poseshaped = self.v_shaped_personal + self.v_posevariation
self.A, A_global = self._global_rigid_transformation()
self.Jtr = ch.vstack([g[:3, 3] for g in A_global])
self.J_transformed = self.Jtr + self.trans.reshape((1, 3))
self.V = self.A.dot(self.weights.T)
rest_shape_h = ch.hstack(
(self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1))))
self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1),
axis=1)[:, :3]
self.v = self.v_posed + self.trans
def LightDotNormal(num_verts):
normalize_rows = lambda v : v / col(ch.sqrt(ch.sum(v.reshape((-1,3))**2, axis=1)))
sum_rows = lambda v : ch.sum(v.reshape((-1,3)), axis=1)
return Ch(lambda light_pos, v, vn :
sum_rows(normalize_rows(light_pos.reshape((1,3)) - v.reshape((-1,3))) * vn.reshape((-1,3))))
def _set_up(self):
self.v_shaped = self.shapedirs.dot(self.betas) + self.v_template
self.v_shaped_personal = self.v_shaped + self.v_personal
if sp.issparse(self.J_regressor):
self.J = sp_dot(self.J_regressor, self.v_shaped)
else:
self.J = ch.sum(self.J_regressor.T.reshape(-1, 1, 24) *
self.v_shaped.reshape(-1, 3, 1),
axis=0).T
self.v_posevariation = self.posedirs.dot(
posemap(self.bs_type)(self.pose))
self.v_poseshaped = self.v_shaped_personal + self.v_posevariation
self.A, A_global = self._global_rigid_transformation()
self.Jtr = ch.vstack([g[:3, 3] for g in A_global])
self.J_transformed = self.Jtr + self.trans.reshape((1, 3))
self.V = self.A.dot(self.weights.T)
rest_shape_h = ch.hstack(
(self.v_poseshaped, ch.ones((self.v_poseshaped.shape[0], 1))))
self.v_posed = ch.sum(self.V.T * rest_shape_h.reshape(-1, 4, 1),
axis=1)[:, :3]
self.v = self.v_posed + self.trans
def LightDotNormal(num_verts):
normalize_rows = lambda v : v / ch.sqrt(ch.sum(v.reshape((-1,3))**2, axis=1)).reshape((-1,1))
sum_rows = lambda v : ch.sum(v.reshape((-1,3)), axis=1)
return Ch(lambda light_pos, v, vn :
sum_rows(normalize_rows(light_pos.reshape((1,3)) - v.reshape((-1,3))) * vn.reshape((-1,3))))
def lambertian_spotlight(v, vn, pos, dir, spot_exponent, camcoord=False, camera_t=None, camera_rt=None):
"""
:param v: vertices
:param vn: vertex normals
:param light_pos: light position
:param light_dir: light direction
:param spot_exponent: spot exponent (a la opengl)
:param camcoord: if True, then pos and dir are wrt the camera
:param camera_t: 3-vector indicating translation of camera
:param camera_rt: 3-vector indicating direction of camera
:return: Vx1 array of brightness
"""
if camcoord: # Transform pos and dir from camera to world coordinate system
assert(camera_t is not None and camera_rt is not None)
from opendr.geometry import Rodrigues
rot = Rodrigues(rt=camera_rt)
pos = rot.T.dot(pos-camera_t)
dir = rot.T.dot(dir)
dir = dir / ch.sqrt(ch.sum(dir**2.))
v_minus_light = v - pos.reshape((1,3))
v_distances = ch.sqrt(ch.sum(v_minus_light**2, axis=1))
v_minus_light_normed = v_minus_light / v_distances.reshape((-1,1))
cosangle = v_minus_light_normed.dot(dir.reshape((3,1)))
light_dot_normal = ch.sum(vn*v_minus_light_normed, axis=1)
light_dot_normal.label = 'light_dot_normal'
cosangle.label = 'cosangle'
result = light_dot_normal.ravel() * cosangle.ravel()**spot_exponent
result = result / v_distances ** 2.
result = maximum(result, 0.0)
return result
def chZernikeProjection(image, numCoeffs=20, win=50):
croppedImage = image[image.shape[0] / 2 - win:image.shape[0] / 2 + win,
image.shape[1] / 2 - win:image.shape[1] / 2 + win, :]
zpolys = zernikePolynomials(imageSize=(croppedImage.shape[0],
croppedImage.shape[1]),
numCoeffs=numCoeffs)
imageProjections = croppedImage[:, :, :, None] * zpolys.reshape(
[zpolys.shape[0], zpolys.shape[1], 1, -1])
coeffs = ch.sum(ch.sum(imageProjections, axis=0), axis=0)
return coeffs, imageProjections
def transformObject(v, vn, chScale, chObjAz, chObjDisplacement, chObjRotation,
targetPosition):
scaleMat = geometry.Scale(x=chScale[0], y=chScale[1], z=chScale[2])[0:3,
0:3]
chRotAzMat = geometry.RotateZ(a=-chObjAz)[0:3, 0:3]
transformation = ch.dot(chRotAzMat, scaleMat)
invTranspModel = ch.transpose(ch.inv(transformation))
objDisplacementMat = computeHemisphereTransformation(
chObjRotation, 0, chObjDisplacement, np.array([0, 0, 0]))
newPos = objDisplacementMat[0:3, 3]
vtransf = []
vntransf = []
for mesh_i, mesh in enumerate(v):
vtransf = vtransf + [
ch.dot(v[mesh_i], transformation) + newPos + targetPosition
]
# ipdb.set_trace()
# vtransf = vtransf + [v[mesh_i] + chPosition]
vndot = ch.dot(vn[mesh_i], invTranspModel)
vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None]
vntransf = vntransf + [vndot]
return vtransf, vntransf, newPos
def run(self, beta=None, theta=None, garment_d=None, garment_class=None):
"""Outputs body and garment of specified garment class given theta, beta and displacements."""
if beta is not None:
self.smpl_base.betas[:beta.shape[0]] = beta
else:
self.smpl_base.betas[:] = 0
if theta is not None:
self.smpl_base.pose[:] = theta
else:
self.smpl_base.pose[:] = 0
self.smpl_base.v_personal[:] = 0
if garment_d is not None and garment_class is not None:
if 'skirt' not in garment_class:
vert_indices = self.class_info[garment_class]['vert_indices']
f = self.class_info[garment_class]['f']
self.smpl_base.v_personal[vert_indices] = garment_d
garment_m = Mesh(v=self.smpl_base.r[vert_indices], f=f)
else:
# vert_indices = self.class_info[garment_class]['vert_indices']
f = self.class_info[garment_class]['f']
A = self.smpl_base.A.reshape((16, 24)).T
skirt_V = self.skirt_skinning.dot(A).reshape((-1, 4, 4))
verts = self.skirt_weight.dot(self.smpl_base.v_poseshaped)
verts = verts + garment_d
verts_h = ch.hstack((verts, ch.ones((verts.shape[0], 1))))
verts = ch.sum(skirt_V * verts_h.reshape(-1, 1, 4),
axis=-1)[:, :3]
garment_m = Mesh(v=verts, f=f)
else:
garment_m = None
self.smpl_base.v_personal[:] = 0
body_m = Mesh(v=self.smpl_base.r, f=self.smpl_base.f)
return body_m, garment_m
def transformObjectFull(v, vn, chScale, chObjAz, chObjAx, chObjAz2,
chPosition):
if chScale.size == 1:
scaleMat = geometry.Scale(x=chScale[0], y=chScale[0],
z=chScale[0])[0:3, 0:3]
elif chScale.size == 2:
scaleMat = geometry.Scale(x=chScale[0], y=chScale[0],
z=chScale[1])[0:3, 0:3]
else:
scaleMat = geometry.Scale(x=chScale[0], y=chScale[1],
z=chScale[2])[0:3, 0:3]
chRotAzMat = geometry.RotateZ(a=chObjAz)[0:3, 0:3]
chRotAxMat = geometry.RotateX(a=-chObjAx)[0:3, 0:3]
chRotAzMat2 = geometry.RotateZ(a=chObjAz2)[0:3, 0:3]
transformation = ch.dot(
ch.dot(ch.dot(chRotAzMat, chRotAxMat), chRotAzMat2), scaleMat)
invTranspModel = ch.transpose(ch.inv(transformation))
vtransf = []
vntransf = []
for mesh_i, mesh in enumerate(v):
vtransf = vtransf + [ch.dot(v[mesh_i], transformation) + chPosition]
vndot = ch.dot(vn[mesh_i], invTranspModel)
vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None]
vntransf = vntransf + [vndot]
return vtransf, vntransf
def joints_coco(smpl):
J = smpl.J_transformed
nose = smpl[VERT_NOSE]
ear_l = smpl[VERT_EAR_L]
ear_r = smpl[VERT_EAR_R]
eye_l = smpl[VERT_EYE_L]
eye_r = smpl[VERT_EYE_R]
shoulders_m = ch.sum(J[[14, 13]], axis=0) / 2.
neck = J[12] - 0.55 * (J[12] - shoulders_m)
return ch.vstack((
nose,
neck,
2.1 * (J[14] - shoulders_m) + neck,
J[[19, 21]],
2.1 * (J[13] - shoulders_m) + neck,
J[[18, 20]],
J[2] + 0.38 * (J[2] - J[1]),
J[[5, 8]],
J[1] + 0.38 * (J[1] - J[2]),
J[[4, 7]],
eye_r,
eye_l,
ear_r,
ear_l,
))
def on_changed(self, which):
if not hasattr(self, 'normalized'):
self.normalized = True
if hasattr(self, 'v') and hasattr(self, 'f'):
if 'f' not in which and hasattr(self, 'faces_by_vertex') and self.faces_by_vertex.shape[0]/3 == self.v.shape[0]:
self.tns.v = self.v
else: # change in f or in size of v. shouldn't happen often.
f = self.f
IS = f.ravel()
JS = np.array([range(f.shape[0])]*3).T.ravel()
data = np.ones(len(JS))
IS = np.concatenate((IS*3, IS*3+1, IS*3+2))
JS = np.concatenate((JS*3, JS*3+1, JS*3+2))
data = np.concatenate((data, data, data))
sz = self.v.size
self.faces_by_vertex = sp.csc_matrix((data, (IS, JS)), shape=(sz, f.size))
self.tns = Ch(lambda v : CrossProduct(TriEdges(f,1,0,v), TriEdges(f,2,0, v)))
self.tns.v = self.v
if self.normalized:
tmp = MatVecMult(self.faces_by_vertex, self.tns)
self.normals = NormalizedNx3(tmp)
else:
test = self.faces_by_vertex.dot(np.ones(self.faces_by_vertex.shape[1]))
faces_by_vertex = sp.diags([1. / test], [0]).dot(self.faces_by_vertex).tocsc()
normals = MatVecMult(faces_by_vertex, self.tns).reshape((-1,3))
normals = normals / (ch.sum(normals**2, axis=1) ** .25).reshape((-1,1))
self.normals = normals
def setup_silhouette_obj(silhs, rends, f):
n_model = [ch.sum(rend[:, :, 0] > 0) for rend in rends]
dist_tsf = [cv2.distanceTransform(np.uint8(1 - silh), cv2.DIST_L2, cv2.DIST_MASK_PRECISE) for silh in silhs]
# Make sigma proportional to image area.
# This gives radius 400 for 960 x 1920 image.
sigma_ratio = 0.2727
sigma = [np.sqrt(sigma_ratio * (silh.shape[0] * silh.shape[1]) / np.pi) for silh in silhs]
# Model2silhouette ==> Consistency (contraction)
def obj_m2s(w, i):
return w * (GMOf(rends[i][:, :, 0] * dist_tsf[i], sigma[i]) / np.sqrt(n_model[i]))
# silhouette error term (scan-to-model) ==> Coverage (expansion)
coords = [np.fliplr(np.array(np.where(silh > 0)).T) + 0.5 for silh in silhs]# is this offset needed?
scan_flat_v = [np.hstack((coord, ch.zeros(len(coord)).reshape((-1, 1)))) for coord in coords]
scan_flat = [Mesh(v=sflat_v, f=[]) for sflat_v in scan_flat_v]
# 2d + all 0.
sv_flat = [ch.hstack((rend.camera, ch.zeros(len(rend.v)).reshape((-1, 1)))) for rend in rends]
for i in range(len(rends)):
sv_flat[i].f = f
def obj_s2m(w, i):
from sbody.mesh_distance import ScanToMesh
return w * ch.sqrt(GMOf(ScanToMesh(scan_flat[i], sv_flat[i], f), sigma[i]))
# For vis
for i in range(len(rends)):
scan_flat[i].vc = np.tile(np.array([0, 0, 1]), (len(scan_flat[i].v), 1))
return obj_m2s, obj_s2m, dist_tsf
def transformObject(v, vn, chScale, chObjAz, chPosition):
#print ('utils.py:16 transformObject')
#print ('v', type(v))
#import ipdb
#ipdb.set_trace()
if chScale.size == 1:
scaleMat = geometry.Scale(x=chScale[0], y=chScale[0],
z=chScale[0])[0:3, 0:3]
elif chScale.size == 2:
scaleMat = geometry.Scale(x=chScale[0], y=chScale[0],
z=chScale[1])[0:3, 0:3]
else:
scaleMat = geometry.Scale(x=chScale[0], y=chScale[1],
z=chScale[2])[0:3, 0:3]
chRotAzMat = geometry.RotateZ(a=chObjAz)[0:3, 0:3]
chRotAzMatX = geometry.RotateX(a=0)[0:3, 0:3]
# transformation = scaleMat
transformation = ch.dot(ch.dot(chRotAzMat, chRotAzMatX), scaleMat)
invTranspModel = ch.transpose(ch.inv(transformation))
vtransf = []
vntransf = []
for mesh_i, mesh in enumerate(v):
vtransf = vtransf + [ch.dot(v[mesh_i], transformation) + chPosition]
vndot = ch.dot(vn[mesh_i], invTranspModel)
vndot = vndot / ch.sqrt(ch.sum(vndot**2, 1))[:, None]
vntransf = vntransf + [vndot]
return vtransf, vntransf
def modelLogLikelihoodRobustRegionCh(image, template, testMask,
backgroundModel, layerPriors, variances):
likelihood = pixelLikelihoodRobustRegionCh(image, template, testMask,
backgroundModel, layerPriors,
variances)
liksum = ch.sum(ch.log(likelihood))
return liksum
def test_sum_mean_std_var(self):
for fn in [ch.sum, ch.mean, ch.var, ch.std]:
# Create fake input and differences in input space
data1 = ch.ones((3, 4, 7, 2))
data2 = ch.array(data1.r + .1 *
np.random.rand(data1.size).reshape(data1.shape))
diff = data2.r - data1.r
# Compute outputs
result1 = fn(data1, axis=2)
result2 = fn(data2, axis=2)
# Empirical and predicted derivatives
gt = result2.r - result1.r
pred = result1.dr_wrt(data1).dot(diff.ravel()).reshape(gt.shape)
#print(np.max(np.abs(gt - pred)))
if fn in [ch.std, ch.var]:
self.assertTrue(1e-2 > np.max(np.abs(gt - pred)))
else:
self.assertTrue(1e-14 > np.max(np.abs(gt - pred)))
# test caching
dr0 = result1.dr_wrt(data1)
data1[:] = np.random.randn(data1.size).reshape(data1.shape)
self.assertTrue(
result1.dr_wrt(data1) is
dr0) # changing values shouldn't force recompute
result1.axis = 1
self.assertTrue(result1.dr_wrt(data1) is not dr0)
self.assertEqual(
ch.mean(ch.eye(3), axis=1).ndim,
np.mean(np.eye(3), axis=1).ndim)
self.assertEqual(
ch.mean(ch.eye(3), axis=0).ndim,
np.mean(np.eye(3), axis=0).ndim)
self.assertEqual(
ch.sum(ch.eye(3), axis=1).ndim,
np.sum(np.eye(3), axis=1).ndim)
self.assertEqual(
ch.sum(ch.eye(3), axis=0).ndim,
np.sum(np.eye(3), axis=0).ndim)
def face_bases(v, f):
t1 = TriEdges(f, 1, 0, v).reshape((-1,3))
t2 = TriEdges(f, 2, 0, v).reshape((-1,3))
#t3 = NormalizedNx3(CrossProduct(t1, t2)).reshape((-1,3))
#t3 = CrossProduct(t1, t2).reshape((-1,3))
# Problem: cross-product is proportional in length to len(t1)*len(t2)
# Solution: divide by sqrt(sqrt(len(cross-product)))
t3 = CrossProduct(t1, t2).reshape((-1,3)); t3 = t3 / col(ch.sum(t3**2., axis=1)**.25)
result = ch.hstack((t1, t2, t3)).reshape((-1,3,3))
return result
def axis2quat(p):
angle = ch.sqrt(ch.clip(ch.sum(ch.square(p), 1), 1e-16, 1e16))
norm_p = p / angle[:, np.newaxis]
cos_angle = ch.cos(angle / 2)
sin_angle = ch.sin(angle / 2)
qx = norm_p[:, 0] * sin_angle
qy = norm_p[:, 1] * sin_angle
qz = norm_p[:, 2] * sin_angle
qw = cos_angle - 1
return ch.concatenate([
qx[:, np.newaxis], qy[:, np.newaxis], qz[:, np.newaxis], qw[:,
np.newaxis]
],
axis=1)
def LightDotNormal(num_verts):
def normalize_rows(v):
b = ch.sqrt(ch.sum(v.reshape((-1, 3))**2, axis=1)).reshape((-1, 1))
return v / b.compute_r()
sum_rows = lambda v: ch.sum(v.reshape((-1, 3)), axis=1)
def f(light_pos, v, vn):
light_pos = light_pos[0, ]
a = np.array([light_pos, light_pos, light_pos])
v = v[0, ]
b = np.array([v, v, v])
return sum_rows(normalize_rows(a - b) * vn.reshape((-1, 3)))
return Ch(f)
def volume(v, f):
# Construct a 3D matrix which is of size (nfaces x 3 x 3)
# Each row corresponds to a face; the third dimension indicates
# which triangle in that face is being referred to
vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :]))
v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0]
v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0]
v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1]
v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1]
v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2]
v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2]
volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1.0 / 6.0)
return ch.abs(ch.sum(volumes))
def volume(v, f):
# Construct a 3D matrix which is of size (nfaces x 3 x 3)
# Each row corresponds to a face; the third dimension indicates
# which triangle in that face is being referred to
vs = ch.dstack((v[f[:, 0], :], v[f[:, 1], :], v[f[:, 2], :]))
v321 = vs[:, 0, 2] * vs[:, 1, 1] * vs[:, 2, 0]
v231 = vs[:, 0, 1] * vs[:, 1, 2] * vs[:, 2, 0]
v312 = vs[:, 0, 2] * vs[:, 1, 0] * vs[:, 2, 1]
v132 = vs[:, 0, 0] * vs[:, 1, 2] * vs[:, 2, 1]
v213 = vs[:, 0, 1] * vs[:, 1, 0] * vs[:, 2, 2]
v123 = vs[:, 0, 0] * vs[:, 1, 1] * vs[:, 2, 2]
volumes = (-v321 + v231 + v312 - v132 - v213 + v123) * (1. / 6.)
return ch.abs(ch.sum(volumes))
def on_changed(self, which):
if not hasattr(self, "normalized"):
self.normalized = True
for w in which:
if w not in ("v", "f"):
raise Exception("VertNormals has only v and f now, and you specified %s." % (w))
if hasattr(self, "v") and hasattr(self, "f"):
if (
"f" not in which
and hasattr(self, "faces_by_vertex")
and self.faces_by_vertex.shape[0] / 3 == self.v.shape[0]
):
self.tns.v = self.v
else: # change in f or in size of v. shouldn't happen often.
f = self.f
IS = f.ravel()
JS = np.array([range(f.shape[0])] * 3).T.ravel()
data = np.ones(len(JS))
IS = np.concatenate((IS * 3, IS * 3 + 1, IS * 3 + 2))
JS = np.concatenate((JS * 3, JS * 3 + 1, JS * 3 + 2))
data = np.concatenate((data, data, data))
sz = self.v.size
self.faces_by_vertex = sp.csc_matrix((data, (IS, JS)), shape=(sz, f.size))
self.tns = Ch(lambda v: CrossProduct(TriEdges(f, 1, 0, v), TriEdges(f, 2, 0, v)))
self.tns.v = self.v
if self.normalized:
tmp = MatVecMult(self.faces_by_vertex, self.tns)
self.normals = NormalizedNx3(tmp)
else:
test = self.faces_by_vertex.dot(np.ones(self.faces_by_vertex.shape[1]))
faces_by_vertex = sp.diags([1.0 / test], [0]).dot(self.faces_by_vertex).tocsc()
normals = MatVecMult(faces_by_vertex, self.tns).reshape((-1, 3))
normals = normals / (ch.sum(normals ** 2, axis=1) ** 0.25).reshape((-1, 1))
self.normals = normals
def run(self, beta=None, theta=None, garment_d=None, garment_class=None):
"""Outputs body and garment of specified garment class given theta, beta and displacements."""
if beta is not None:
self.smpl_base.betas[:beta.shape[0]] = beta
else:
self.smpl_base.betas[:] = 0
if theta is not None:
self.smpl_base.pose[:] = theta
else:
self.smpl_base.pose[:] = 0
self.smpl_base.v_personal[:] = 0
if garment_d is not None and garment_class is not None:
if 'skirt' not in garment_class:
vert_indices = self.class_info[garment_class]['vert_indices']
f = self.class_info[garment_class]['f']
self.smpl_base.v_personal[vert_indices] = garment_d
garment_m = Mesh(v=self.smpl_base.r[vert_indices], f=f)
else:
vert_indices = self.class_info[garment_class]['vert_indices']
f = self.class_info[garment_class]['f']
verts = self.smpl_base.v_poseshaped[vert_indices] + garment_d
verts_h = ch.hstack((verts, ch.ones((verts.shape[0], 1))))
verts = ch.sum(self.smpl_base.V.T[vert_indices] *
verts_h.reshape(-1, 4, 1),
axis=1)[:, :3]
# if theta is not None:
# rotmat = self.smpl_base.A.r[:, :, 0]
# verts_homo = np.hstack(
# (verts, np.ones((verts.shape[0], 1))))
# verts = verts_homo.dot(rotmat.T)[:, :3]
garment_m = Mesh(v=verts, f=f)
else:
garment_m = None
self.smpl_base.v_personal[:] = 0
body_m = Mesh(v=self.smpl_base.r, f=self.smpl_base.f)
return body_m, garment_m
win = 40
colors = image[image.shape[0] / 2 - win:image.shape[0] / 2 + win,
image.shape[1] / 2 - win:image.shape[1] / 2 + win, :].reshape(
[4 * win * win, 3])
gmm.fit(colors)
imshape = [win * 2, win * 2, 3]
numPixels = win * 2 * win * 2
chInput = ch.Ch(colors)
numVars = chInput.size
recSoftmaxW = ch.Ch(np.random.uniform(0, 1, [nRecComps, numVars]) / numVars)
chRecLogistic = ch.exp(ch.dot(recSoftmaxW, chInput.reshape([numVars, 1])))
chRecSoftmax = chRecLogistic.ravel() / ch.sum(chRecLogistic)
chZRecComps = ch.zeros([numVars, nRecComps])
chZ = ch.zeros([numVars])
recMeans = ch.Ch(np.random.uniform(0, 1, [3, nRecComps]))
recCovars = 0.2
chRecLogLikelihoods = -0.5 * (chZ.reshape([numPixels, 3, 1]) - ch.tile(
recMeans, [numPixels, 1, 1]))**2 - ch.log(
(2 * recCovars) * (1 / (ch.sqrt(recCovars) * np.sqrt(2 * np.pi))))
genZCovars = 0.2
chGenComponentsProbs = ch.Ch(gmm.weights_)
chCompMeans = ch.zeros([nComps, 3])
def mano_inverse_kinematics(mano_hand,
desired_zimmerman_coords,
plot_progress=False):
joint_idxs_mano = np.asarray(range(16))
joint_idxs_zimmerman = MANO_TO_ZIMMERMAN_INDICES[joint_idxs_mano]
assert (joint_idxs_zimmerman.shape == joint_idxs_mano.shape)
joint_coords_mano = mano_hand.J_transformed[joint_idxs_mano]
joint_coords_zimmerman = desired_zimmerman_coords[joint_idxs_zimmerman]
mano_coords_centered = joint_coords_mano - joint_coords_mano[0]
zimmerman_coords_centered = joint_coords_zimmerman - joint_coords_zimmerman[
0]
zimmerman_coords_centered *= 0.05
if plot_progress:
plot_mano_skeleton(mano_coords_centered, zimmerman_coords_centered)
tip_idxs_mano = np.asarray([1, 4, 10, 7]) # knuckles (align palms)
scale = ch.var(1)
mano_coords_scaled = mano_coords_centered * scale
position_loss = (mano_coords_scaled[tip_idxs_mano] -
zimmerman_coords_centered[tip_idxs_mano])**2
loss = position_loss
inputs = [mano_hand.pose[:3], scale]
objective = {'loss': loss}
opt = {'maxiter': 10}
print("Rotating and scaling palms together...")
ch.minimize(objective, x0=inputs, method='dogleg', options=opt)
bone_idxs = []
for zb0, zb1 in BONES:
if zb0 in ZIMMERMAN_TO_MANO_DICT and zb1 in ZIMMERMAN_TO_MANO_DICT:
b0 = ZIMMERMAN_TO_MANO_DICT[zb0]
b1 = ZIMMERMAN_TO_MANO_DICT[zb1]
bone_idxs.append([b0, b1])
bone_idxs = np.asarray(bone_idxs)
zimmerman_displacements = zimmerman_coords_centered[
bone_idxs[:, 1]] - zimmerman_coords_centered[bone_idxs[:, 0]]
mano_displacements = mano_coords_scaled[
bone_idxs[:, 1]] - mano_coords_scaled[bone_idxs[:, 0]]
displacement_loss = ch.sum(
ch.multiply(zimmerman_displacements, mano_displacements))
regularization_loss = 1e-5 * ch.sum(mano_hand.pose[3:]**2)
# MAGIC HAPPENING RIGHT HERE:
# This huge number is important.
# Apparently, ch.minimize will attempt to push the loss to zero,
# not towards minus infinity.
# Here we're trying to maximize the dot product of all bones
# in both hands (thereby making them all point in the same
# direction). The dot product needs to be maximally positive,
# and should not be zero, hence the huge number below.
loss = (1000000 - displacement_loss) + regularization_loss
inputs = [mano_hand.pose[3:]]
opt = {'maxiter': 10}
objective = {'loss': loss}
print("Aligning all bones to be co-linear...")
ch.minimize(objective, x0=inputs, method='dogleg', options=opt)
if plot_progress:
plot_mano_skeleton(mano_coords_scaled, zimmerman_coords_centered)
def diffHog(image, drconv=None, numOrient=9, cwidth=8, cheight=8):
imagegray = 0.3 * image[:, :, 0] + 0.59 * image[:, :,
1] + 0.11 * image[:, :, 2]
sy, sx = imagegray.shape
# gx = ch.empty(imagegray.shape, dtype=np.double)
gx = imagegray[:, 2:] - imagegray[:, :-2]
gx = ch.hstack([np.zeros([sy, 1]), gx, np.zeros([sy, 1])])
gy = imagegray[2:, :] - imagegray[:-2, :]
# gy = imagegray[:, 2:] - imagegray[:, :-2]
gy = ch.vstack([np.zeros([1, sx]), gy, np.zeros([1, sx])])
gx += 1e-5
# gy = imagegray[:-2,1:-1] - imagegray[2:,1:-1] + 0.00001
# gx = imagegray[1:-1,:-2] - imagegray[1:-1, 2:] + 0.00001
distFilter = np.ones([2 * cheight, 2 * cwidth], dtype=np.uint8)
distFilter[np.int(2 * cheight / 2), np.int(2 * cwidth / 2)] = 0
distFilter = (cv2.distanceTransform(distFilter, cv2.DIST_L2, 3) - np.max(
cv2.distanceTransform(distFilter, cv2.DIST_L2, 3))) / (
-np.max(cv2.distanceTransform(distFilter, cv2.DIST_L2, 3)))
magn = ch.sqrt(gy**2 + gx**2) * 180 / np.sqrt(2)
angles = ch.arctan(gy / gx) * 180 / np.pi + 90
# meanOrient = np.linspace(0, 180, numOrient)
orientations_arr = np.arange(numOrient)
meanOrient = orientations_arr / numOrient * 180
fb_resttmp = 1 - ch.abs(
ch.expand_dims(angles[:, :], 2) -
meanOrient[1:].reshape([1, 1, numOrient - 1])) * numOrient / 180
zeros_rest = np.zeros([sy, sx, numOrient - 1, 1])
fb_rest = ch.max(ch.concatenate([fb_resttmp[:, :, :, None], zeros_rest],
axis=3),
axis=3)
chMinOrient0 = ch.min(ch.concatenate([
ch.abs(
ch.expand_dims(angles[:, :], 2) -
meanOrient[0].reshape([1, 1, 1]))[:, :, :, None],
ch.abs(180 - ch.expand_dims(angles[:, :], 2) -
meanOrient[0].reshape([1, 1, 1]))[:, :, :, None]
],
axis=3),
axis=3)
zeros_fb0 = np.zeros([sy, sx, 1])
fb0_tmp = ch.concatenate(
[1 - chMinOrient0[:, :] * numOrient / 180, zeros_fb0], axis=2)
fb_0 = ch.max(fb0_tmp, axis=2)
fb = ch.concatenate([fb_0[:, :, None], fb_rest], axis=2)
# fb[:,:,0] = ch.max(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180,0)
# fb = 1./(1. + ch.exp(1 - ch.abs(ch.expand_dims(angles,2) - meanOrient.reshape([1,1,numOrient]))*numOrient/180))
Fb = ch.expand_dims(magn, 2) * fb
if drconv is None:
drconv = dr_wrt_convolution(Fb[:, :, 0], distFilter)
Fs_list = [
convolve2D(x=Fb[:, :, Fbi], filter=distFilter,
convolve2DDr=drconv).reshape([Fb.shape[0], Fb.shape[1], 1])
for Fbi in range(numOrient)
]
# Fs_list = [scipy.signal.convolve2d(Fb[:,:,Fbi], distFilter).reshape([Fb.shape[0], Fb.shape[1],1]) for Fbi in range(numOrient)]
Fs = ch.concatenate(Fs_list, axis=2)
# cellCols = np.arange(start=cwidth/2, stop=Fs.shape[1]-cwidth/2 , step=cwidth)
# cellRows = np.arange(start=cheight/2, stop=Fs.shape[0]-cheight/2 , step=cheight)
Fcells = Fs[0:Fs.shape[0]:cheight, 0:Fs.shape[1]:cwidth, :]
epsilon = 1e-5
v = Fcells / ch.sqrt(ch.sum(Fcells**2) + epsilon)
# v = Fcells
# hog, hogim = skimage.feature.hog(imagegray, orientations=numOrient, pixels_per_cell=(cheight, cwidth), visualise=True)
hog_image = HogImage(image=image,
hog=Fcells,
numOrient=numOrient,
cwidth=cwidth,
cheight=cheight)
# plt.imshow(hog_image)
# plt.figure()
# plt.imshow(hogim)
# ipdb.set_trace()
return v, hog_image, drconv
def modelLogLikelihoodCh(image, template, testMask, backgroundModel,
variances):
logLikelihood = logPixelLikelihoodCh(image, template, testMask,
backgroundModel, variances)
return ch.sum(logLikelihood)
c_pre={};
if (W_Joints):
k=a['Joints_Target'];
j_to_consider = range(0,24)
J_distances = J_reposed[j_to_consider,:] - (k[j_to_consider,:]+trans);
c_pre['Joints']= J_distances*W_Joints;
if(W_FMP2P):
c_pre['FMP2P']= distances*W_FMP2P;
if(W_Norm_B):
c_pre['Norm_B']= ((result.betas)**2)*W_Norm_B;
if(W_Norm_T):
pose_res=result.pose.reshape(-1,3);
angles=ch.sum(ch.abs(pose_res)**2,axis=-1)**(1./2)
pesi = np.ones(24)*8/18
pesi[[0]] = np.ones(1)*[2]
pesi[[10, 11, 22, 23, 15]] = np.ones(5)*[2./18]
pesi[[6,3, 7,8]] = np.ones(4)*[5./18]
pesi[[12, 15]] = np.ones(2)*[1./36]
costo_T= (angles/(ch.pi*pesi))**12
c_pre['Norm_T']=costo_T*W_Norm_T;
if(W_Landmarks):
Tar_Land = Tar_shift[a['landmarks1'].reshape(5)-1];
SMPL_Land=result[a['landmarks2'].reshape(5)-1];
c_pre['Landmarks']= (SMPL_Land-Tar_Land)*W_Landmarks;
(r,b,t)=ch.minimize(c_pre, x0 = [result.pose, result.betas,trans],
method = 'dogleg', callback = None,
def normalize_rows(v):
b = ch.sqrt(ch.sum(v.reshape((-1, 3))**2, axis=1)).reshape((-1, 1))
return v / b.compute_r()
def lift2Dto3D(projPtsGT,
camMat,
filename,
img,
JVis=np.ones((21, ), dtype=np.float32),
trans=None,
beta=None,
wrist3D=None,
withPoseCoeff=True,
weights=1.0,
relDepGT=None,
rel3DCoordGT=None,
rel3DCoordNormGT=None,
img2DGT=None,
outDir=None,
poseCoeffInit=None,
transInit=None,
betaInit=None):
loss = {}
if withPoseCoeff:
numComp = 30
m, poseCoeffCh, betaCh, transCh, fullposeCh = getHandModelPoseCoeffs(
numComp)
if poseCoeffInit is not None:
poseCoeffCh[:] = poseCoeffInit
if transInit is not None:
transCh[:] = transInit
if betaInit is not None:
betaCh[:] = betaInit
freeVars = [poseCoeffCh]
if beta is None:
freeVars = freeVars + [betaCh]
loss['shape'] = 1e2 * betaCh
else:
betaCh[:] = beta
if trans is None:
freeVars = freeVars + [transCh]
else:
transCh[:] = trans
# loss['pose'] = 0.5e2 * poseCoeffCh[3:]/stdPCACoeff[:numComp]
thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints(
fullposeCh[3:])
loss['constMin'] = 5e2 * thetaConstMin
loss['constMax'] = 5e2 * thetaConstMax
loss['invalidTheta'] = 1e3 * fullposeCh[Constraints().invalidThetaIDs]
else:
m, rotCh, jointsCh, transCh, betaCh = getHandModel()
thetaConstMin, thetaConstMax = Constraints().getHandJointConstraints(
jointsCh)
loss['constMin'] = 5e3 * thetaConstMin
loss['constMax'] = 5e3 * thetaConstMax
validTheta = jointsCh[Constraints().validThetaIDs[3:] - 3]
freeVars = [validTheta, rotCh]
if beta is None:
freeVars = freeVars + [betaCh]
loss['shape'] = 0.5e2 * betaCh
else:
betaCh[:] = beta
if trans is None:
freeVars = freeVars + [transCh]
else:
transCh[:] = trans
if relDepGT is not None:
relDepPred = m.J_transformed[:, 2] - m.J_transformed[0, 2]
loss['relDep'] = (relDepPred - relDepGT) * weights[:, 0] * 5e1
if rel3DCoordGT is not None:
rel3DCoordPred = m.J_transformed - m.J_transformed[0:1, :]
loss['rel3DCoord'] = (rel3DCoordPred - rel3DCoordGT) * np.tile(
weights[:, 0:1], [1, 3]) * 5e1
if rel3DCoordNormGT is not None:
rel3DCoordPred = m.J_transformed[jointsMap][
1:, :] - m.J_transformed[jointsMap][0:1, :]
rel3DCoordPred = rel3DCoordPred / ch.expand_dims(
ch.sqrt(ch.sum(ch.square(rel3DCoordPred), axis=1)), axis=1)
loss['rel3DCoordNorm'] = (
1. - ch.sum(rel3DCoordPred * rel3DCoordNormGT, axis=1)) * 1e4
# loss['rel3DCoordNorm'] = \
# (rel3DCoordNormGT*ch.expand_dims(ch.sum(rel3DCoordPred*rel3DCoordNormGT, axis=1), axis=1) - rel3DCoordPred) * 1e2#5e2
projPts = utilsEval.chProjectPoints(m.J_transformed, camMat,
False)[jointsMap]
JVis = np.tile(np.expand_dims(JVis, 1), [1, 2])
loss['joints2D'] = (projPts - projPtsGT) * JVis * weights * 1e0
loss['joints2DClip'] = clipIden(projPts - projPtsGT) * JVis * weights * 1e1
if wrist3D is not None:
dep = wrist3D[2]
if dep < 0:
dep = -dep
loss['wristDep'] = (m.J_transformed[0, 2] - dep) * 1e2
# vis_mesh(m)
render = False
def cbPass(_):
pass
# print(loss['joints'].r)
print(filename)
warnings.simplefilter('ignore')
loss['joints2D'] = loss[
'joints2D'] * 1e1 / weights # dont want to use confidence now
if True:
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cbPass if render else cbPass,
method='dogleg',
options={'maxiter': 50})
else:
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
freeVars = [poseCoeffCh[:3], transCh]
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cbPass,
method='dogleg',
options={'maxiter': 20})
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
freeVars = [poseCoeffCh[3:]]
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cb if render else cbPass,
method='dogleg',
options={'maxiter': 20})
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
freeVars = [poseCoeffCh, transCh]
if beta is None:
freeVars = freeVars + [betaCh]
ch.minimize({k: loss[k]
for k in loss.keys() if k != 'joints2DClip'},
x0=freeVars,
callback=cb if render else cbPass,
method='dogleg',
options={'maxiter': 20})
if False:
open3dVisualize(m)
else:
manoVis.dump3DModel2DKpsHand(img,
m,
filename,
camMat,
est2DJoints=projPtsGT,
gt2DJoints=img2DGT,
outDir=outDir)
# vis_mesh(m)
joints3D = m.J_transformed.r[jointsMap]
# print(betaCh.r)
# print((relDepPred.r - relDepGT))
return joints3D, poseCoeffCh.r.copy(), betaCh.r.copy(), transCh.r.copy(
), loss['joints2D'].r.copy(), m.r.copy()
def objFun(vs):
vs = np.array(vs)
res = []
for vs_it, vs_i in enumerate(vs):
changevars(vs_i, free_variables)
import densecrf_model
vis_im = np.array(renderer.indices_image == 1).copy().astype(
np.bool)
bound_im = renderer.boundarybool_image.astype(np.bool)
segmentation, Q = densecrf_model.crfInference(
rendererGT.r, vis_im, bound_im, [0.75, 0.25, 0.01],
resultDir + 'imgs/crf/Q_' + str(test_i) + '_it' + str(vs_it))
vColor = color
if updateColor:
if np.sum(segmentation == 0) > 5:
segmentRegion = segmentation == 0
vColor = np.median(rendererGT.reshape(
[-1, 3])[segmentRegion.ravel()],
axis=0) * 1.4
vColor = vColor / max(np.max(vColor), 1.)
chVColors[:] = vColor
chSHLightCoeffs[:] = lightCoeffs
variances = stds**2
fgProb = ch.exp(-(renderer - rendererGT)**2 /
(2 * variances)) * (1. /
(stds * np.sqrt(2 * np.pi)))
h = renderer.r.shape[0]
w = renderer.r.shape[1]
occProb = np.ones([h, w])
bgProb = np.ones([h, w])
errorFun = -ch.sum(
ch.log(vis_im[:, :, None] *
((Q[0].reshape([h, w, 1]) * fgProb) +
(Q[1].reshape([h, w]) * occProb +
Q[2].reshape([h, w]) * bgProb)[:, :, None]) +
(1 - vis_im[:, :, None]))) / (h * w)
if minAppLight:
options = {'disp': False, 'maxiter': 10}
def cb(_):
print("Error: " + str(errorFun.r))
ch.minimize({'raw': errorFun},
bounds=None,
method=method,
x0=free_variables_app_light,
callback=cb,
options=options)
res = res + [errorFun.r.reshape([1, 1])]
return np.vstack(res)
def get_objective(self):
return ch.sum((ch.exp(-((ch.sum(
(self.sph_v[self.ids0] - self.sph_v[self.ids1])**2, axis=1)) /
(self.radiuss)) / 2.))**2)**.5
