def setupCamera(v, chAz, chEl, chDist, objCenter, width, height):
chCamModelWorld = computeHemisphereTransformation(chAz, chEl, chDist,
objCenter)
chMVMat = ch.dot(chCamModelWorld,
np.array(mathutils.Matrix.Rotation(radians(270), 4, 'X')))
chInvCam = ch.inv(chMVMat)
modelRotation = chInvCam[0:3, 0:3]
chRod = opendr.geometry.Rodrigues(rt=modelRotation).reshape(3)
chTranslation = chInvCam[0:3, 3]
translation, rotation = (chTranslation, chRod)
camera = ProjectPoints(v=v,
rt=rotation,
t=translation,
f=1.12 * ch.array([width, width]),
c=ch.array([width, height]) / 2.0,
k=ch.zeros(5))
camera.openglMat = np.array(mathutils.Matrix.Rotation(
radians(180), 4, 'X'))
return camera, modelRotation, chMVMat
def main(mesh_list, out_list, scale=1.0, move_scale=True):
assert len(mesh_list) == len(out_list)
for mesh_file, out_file in zip(mesh_list, out_list):
mesh = load_obj_data_binary(mesh_file)
if move_scale: # move to center and scale to unit bounding box
mesh['v'] = (mesh['v'] - np.array([128, -192, 128]) +
0.5) * voxel_size
if not ('vn' in mesh and mesh['vn'] is not None):
mesh['vn'] = np.array(VertNormals(f=mesh['f'], v=mesh['v']))
V = ch.array(mesh['v']) * scale
V -= trans
C = np.ones_like(mesh['v'])
C *= np.array([186, 212, 255], dtype=np.float32) / 255.0
# C *= np.array([158, 180, 216], dtype=np.float32) / 250.0
C = np.minimum(C, 1.0)
A = np.zeros_like(mesh['v'])
A += LambertianPointLight(f=mesh['f'],
v=V,
vn=-mesh['vn'],
num_verts=len(V),
light_pos=np.array([0, -50, -50]),
light_color=np.array([1.0, 1.0, 1.0]),
vc=C)
cam_t, cam_r = ch.array((0, 0, 0)), ch.array((3.14, 0, 0))
U = ProjectPoints(v=V,
f=[flength, flength],
c=[w / 2., h / 2.],
k=ch.zeros(5),
t=cam_t,
rt=cam_r)
rn = ColoredRenderer(camera=U,
v=V,
f=mesh['f'],
vc=A,
bgcolor=np.array([1.0, 0.0, 0.0]),
frustum={
'width': w,
'height': h,
'near': 0.1,
'far': 20
})
img = np.asarray(rn)[:, :, (2, 1, 0)]
msk = np.sum(np.abs(img - np.array([[[0, 0, 1.0]]], dtype=np.float32)),
axis=-1,
keepdims=True)
msk[msk > 0] = 1
img = cv.resize(img, (img.shape[1] // 2, img.shape[0] // 2))
msk = cv.resize(msk, (msk.shape[1] // 2, msk.shape[0] // 2),
interpolation=cv.INTER_AREA)
msk[msk < 1] = 0
msk = msk[:, :, np.newaxis]
img = np.concatenate([img, msk], axis=-1)
cv.imshow('render3', img)
cv.waitKey(3)
cv.imwrite(out_file, np.uint8(img * 255))
def load_results(pkl_path, model):
from smpl_webuser.verts import verts_decorated as SmplModel
with open(pkl_path, 'rb') as f:
res = pkl.load(f, encoding='latin1')
sv = SmplModel(trans=ch.array(res['trans']),
pose=ch.array(res['pose']),
v_template=model.v_template,
J=model.J,
weights=model.weights,
kintree_table=model.kintree_table,
bs_style='lbs',
f=model.f,
betas=ch.array(res['betas']),
shapedirs=model.shapedirs[:, :, :len(res['betas'])])
if 'E' in res.keys():
E = res['E']
else:
E = None
if 'params' in res.keys():
params = res['params']
else:
params = None
return sv, res['kp'][0], res['kp'][1], res['flength'], E, res[
'landmarks'], res['cam_t'], res['cam_rt'], res['cam_k'], params
def test_depth_image(self):
# Create renderer
import chumpy as ch
from renderer import DepthRenderer
rn = DepthRenderer()
# Assign attributes to renderer
from util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,0]), rotation=ch.zeros(3))
m.v = m.v * .01
m.v[:,2] += 4
w, h = (320, 240)
from camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc*0+1, bgcolor=ch.zeros(3))
# import time
# tm = time.time()
# rn.r
# print 'took %es' % (time.time() - tm)
# print np.min(rn.r.ravel())
# print np.max(rn.r.ravel())
self.assertLess(np.abs(np.min(rn.r.ravel()) - 3.98), 1e-5)
self.assertLess(np.abs(np.min(m.v[:,2]) - np.min(rn.r.ravel())), 1e-5)
self.assertLess(np.abs(rn.r[h/2,w/2] - 3.98), 1e-5)
def test_nandivide(self):
foo = ch.array(np.random.randn(16).reshape((4, 4)))
bar = ch.array(np.random.randn(16).reshape((4, 4)))
bar[2, 2] = 0
self.assertEqual(ch.NanDivide(foo, bar)[2, 2].r, 0.)
foo[2, 2] = 0
self.assertEqual(ch.NanDivide(foo, bar)[2, 2].r, 0.)
def getHandModelPoseCoeffsMultiFrame(numComp, numFrames, isOpenGLCoord):
chGlobalPoseCoeff = ch.zeros((numComp, ))
chGlobalBeta = ch.zeros((10, ))
chRotList = []
chTransList = []
mList = []
for i in range(numFrames):
chRot = ch.zeros((3, ))
if isOpenGLCoord:
chTrans = ch.array([0., 0., -0.5])
else:
chTrans = ch.array([0., 0., 0.5])
chRotList.append(chRot)
chTransList.append(chTrans)
m = load_model_withInputs_poseCoeffs(os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'../mano/models/MANO_RIGHT.pkl'),
chRot=chRot,
chTrans=chTrans,
chPoseCoeff=chGlobalPoseCoeff,
chBetas=chGlobalBeta,
ncomps=numComp)
mList.append(m)
return mList, chRotList, chGlobalPoseCoeff, chTransList, chGlobalBeta
def test_depth_image(self):
# Create renderer
import chumpy as ch
from .renderer import DepthRenderer
rn = DepthRenderer()
# Assign attributes to renderer
from .util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3))
m.v = m.v * .01
m.v[:, 2] += 4
w, h = (320, 240)
from .camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v,
rt=ch.zeros(3),
t=ch.zeros(3),
f=ch.array([w, w]) / 2.,
c=ch.array([w, h]) / 2.,
k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, vc=m.vc * 0 + 1, bgcolor=ch.zeros(3))
# import time
# tm = time.time()
# rn.r
# print 'took %es' % (time.time() - tm)
# print np.min(rn.r.ravel())
# print np.max(rn.r.ravel())
self.assertLess(np.abs(np.min(rn.r.ravel()) - 3.98), 1e-5)
self.assertLess(np.abs(np.min(m.v[:, 2]) - np.min(rn.r.ravel())), 1e-5)
self.assertLess(np.abs(rn.r[h / 2, w / 2] - 3.98), 1e-5)
def get_renderer():
import chumpy as ch
from opendr.everything import *
# Load mesh
m = load_mesh('/Users/matt/geist/OpenDR/test_dr/nasa_earth.obj')
m.v += ch.array([0, 0, 3.])
w, h = (320, 240)
trans = ch.array([[0, 0, 0]])
# Construct renderer
rn = TexturedRenderer()
rn.camera = ProjectPoints(v=m.v,
rt=ch.zeros(3),
t=ch.zeros(3),
f=ch.array([w, w]) / 2.,
c=ch.array([w, h]) / 2.,
k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=trans + m.v,
f=m.f,
texture_image=m.texture_image[:, :, ::-1],
ft=m.ft,
vt=m.vt,
bgcolor=ch.zeros(3))
rn.vc = SphericalHarmonics(vn=VertNormals(v=rn.v, f=rn.f),
components=ch.array([4., 0., 0., 0.]),
light_color=ch.ones(3))
return rn
def chumpy_get_H(p1, p2):
""" Compute differentiable homography from p1 to p2.
Parameters
----------
p1,p2 : array_like, shape (4,2)
Containing points to match.
"""
xmin = 0
ymin = 0
xmax = 1024
ymax = 576
p1 = 2 * p1 / ch.array([[xmax, ymax]])
p1 = p1 - 1.0
p2 = 2 * p2 / ch.array([[xmax, ymax]])
p2 = p2 - 1.0
N = p1.shape[0]
A1 = ch.vstack((ch.zeros((3, N)), -p1.T, -ch.ones(
(1, N)), p2[:, 1] * p1[:, 0], p2[:, 1] * p1[:, 1], p2[:, 1])).T
A2 = ch.vstack((p1.T, ch.ones((1, N)), ch.zeros(
(3, N)), -p2[:, 0] * p1[:, 0], -p2[:, 0] * p1[:, 1], -p2[:, 0])).T
A = ch.vstack((A1, A2))
U, S, V = ch.linalg.svd(A.T.dot(A))
H_new = V[-1, :].reshape((3, 3))
# Re-normalize
ML = ch.array([[xmax / 2.0, 0.0, xmax / 2.0], [0, ymax / 2.0, ymax / 2.0],
[0, 0, 1.0]])
MR = ch.array([[2.0 / xmax, 0.0, -1.0], [0, 2.0 / ymax, -1.0], [0, 0,
1.0]])
H_new = ML.dot(H_new).dot(MR)
return H_new / H_new[2, 2]
def fit_joints(both_joints, n_pose_params=15, shape_sigma=10.0,
save_filename=None):
"""
Fits the MANO model to hand joint 3D locations
both_jonts: tuple of length 2, 21 joints per hand, e.g. output of ContactPose.hand_joints()
n_pose_params: number of pose parameters (excluding 3 global rotation params)
shape_sigma: reciprocal of shape regularization strength
save_filename: file where the fitting output will be saved in JSON format
"""
mano_params = []
for hand_idx, joints in enumerate(both_joints):
if joints is None: # hand is not present
mano_params.append(mano_param_dict(n_pose_params)) # dummy
continue
cp_joints = openpose2mano(joints)
# MANO model
m = mutils.load_mano_model(MANOFitter._mano_dicts[hand_idx],
ncomps=n_pose_params, flat_hand_mean=False)
m.betas[:] = np.zeros(m.betas.size)
m.pose[:] = np.zeros(m.pose.size)
mano_joints = mano_joints_with_fingertips(m)
mano_joints_np = np.array([[float(mm) for mm in m] for m in mano_joints])
# align palm
cp_palm = get_palm_joints(np.asarray(cp_joints))
mano_palm = get_palm_joints(np.asarray(mano_joints_np))
mTc = register_pcs(cp_palm, mano_palm)
cp_joints = np.dot(mTc, np.vstack((cp_joints.T, np.ones(len(cp_joints)))))
cp_joints = cp_joints[:3].T
cp_joints = ch.array(cp_joints)
# set up objective
objective = [m-c for m,c in zip(mano_joints, cp_joints)]
mean_betas = ch.array(np.zeros(m.betas.size))
objective.append((m.betas - mean_betas) / shape_sigma)
# optimize
ch.minimize(objective, x0=(m.pose, m.betas, m.trans), method='dogleg')
p = mano_param_dict(n_pose_params)
p['pose'] = np.array(m.pose).tolist()
p['betas'] = np.array(m.betas).tolist()
p['valid'] = True
p['mTc']['translation'] = (mTc[:3, 3] - np.array(m.trans)).tolist()
p['mTc']['rotation'] = txq.mat2quat(mTc[:3, :3]).tolist()
mano_params.append(p)
# # to access hand mesh vertices and faces
# vertices = np.array(m.r)
# vertices = mutils.tform_points(np.linalg.inv(mTc), vertices)
# faces = np.array(m.f)
if save_filename is not None:
with open(save_filename, 'w') as f:
json.dump(mano_params, f, indent=4, separators=(',', ':'))
print('{:s} written'.format(save_filename))
return mano_params
def scalecam(cam, imres):
if imres == 1:
return cam
# Returns a camera which shares camera parameters by reference,
# and whose scale is resized according to imres
return ProjectPoints(rt=cam.rt,
t=cam.t,
f=array(cam.f.r * imres),
c=array(cam.c.r * imres),
k=cam.k)
def _fit_rot_trans(
model, # pylint: disable=too-many-arguments, too-many-locals
j2d,
center,
init_t,
init_pose,
conf,
flength):
"""Find a rotation and translation to minimize the projection error of the pose."""
opt_pose = _ch.array(init_pose) # pylint: disable=no-member
opt_trans = _ch.array(init_t) # pylint: disable=no-member
(_, A_global) = _global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) + opt_trans # pylint: disable=no-member
cam = _ProjectPoints(f=_np.array([flength, flength]),
rt=_np.zeros(3),
t=_np.zeros(3),
k=_np.zeros(5),
c=center)
cam_3d = _ProjectPoints3D(f=_np.array([flength, flength]),
rt=_np.zeros(3),
t=_np.zeros(3),
k=_np.zeros(5),
c=center)
cam.v = Jtr
cam_3d.v = Jtr
free_variables = [opt_pose[:3],
opt_trans] # Optimize global rotation and translation.
j2d_ids = range(12)
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
#_CID_TORSO = [2, 3, 8, 9]
#torso_smpl_ids = [2, 1, 17, 16]
_ch.minimize(
{
'j2d': (j2d.T[j2d_ids] - cam[smpl_ids]).T * conf[j2d_ids],
# 'dev^2': 1e2 * (opt_trans[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=None, #on_step_vis,
options={
'maxiter': 100,
'e_3': .0001,
'disp': 0
})
_LOGGER.debug("Global rotation: %s, global translation: %s.",
str(opt_pose[:3].r), str(opt_trans.r))
_LOGGER.debug("Points 3D: %s.", str(cam_3d.r[:10, :]))
_LOGGER.debug("Points 2D: %s.", str(cam.r[:10, :]))
return opt_pose[:3].r, opt_trans.r, cam_3d.r[:,
2].min(), cam_3d.r[:,
2].max()
def __init__(self, obj_path, model_path, w=224, h=224):
self.m = get_body_mesh(obj_path, trans=ch.array([0, 0, 4]), rotation=ch.array([np.pi / 2, 0, 0]))
# Load SMPL model (here we load the female model)
self.body = load_model(model_path)
self.w = w
self.h = h
self.img_size = min(self.w, self.h)
self.num_cam = 3
self.num_theta = 72
self.num_beta = 10
def test_derivatives2(self):
import chumpy as ch
import numpy as np
from .renderer import DepthRenderer
rn = DepthRenderer()
# Assign attributes to renderer
from .util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0, 0, 4]), rotation=ch.zeros(3))
w, h = (320, 240)
from .camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v,
rt=ch.zeros(3),
t=ch.zeros(3),
f=ch.array([w, w]) / 2.,
c=ch.array([w, h]) / 2.,
k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))
if visualize:
import matplotlib.pyplot as plt
plt.ion()
plt.figure()
for which in range(3):
r1 = rn.r
adder = np.random.rand(rn.v.r.size).reshape(rn.v.r.shape) * .01
change = rn.v.r * 0 + adder
dr_pred = rn.dr_wrt(rn.v).dot(change.ravel()).reshape(rn.shape)
rn.v = rn.v.r + change
r2 = rn.r
dr_emp = r2 - r1
#print np.mean(np.abs(dr_pred-dr_emp))
self.assertLess(np.mean(np.abs(dr_pred - dr_emp)), .024)
if visualize:
plt.subplot(2, 3, which + 1)
plt.imshow(dr_pred)
plt.clim(-.01, .01)
plt.title('emp')
plt.subplot(2, 3, which + 4)
plt.imshow(dr_emp)
plt.clim(-.01, .01)
plt.title('pred')
plt.draw()
plt.show()
def test_earth():
m = get_earthmesh(trans=ch.array([0, 0, 0]), rotation=ch.zeros(3))
# Create V, A, U, f: geometry, brightness, camera, renderer
V = ch.array(m.v)
A = SphericalHarmonics(vn=VertNormals(v=V, f=m.f),
components=[3., 2., 0., 0., 0., 0., 0., 0., 0.],
light_color=ch.ones(3))
# camera
U = ProjectPoints(v=V,
f=[w, w],
c=[w / 2., h / 2.],
k=ch.zeros(5),
t=ch.zeros(3),
rt=ch.zeros(3))
f = TexturedRenderer(vc=A,
camera=U,
f=m.f,
bgcolor=[0., 0., 0.],
texture_image=m.texture_image,
vt=m.vt,
ft=m.ft,
frustum={
'width': w,
'height': h,
'near': 1,
'far': 20
})
# Parameterize the vertices
translation, rotation = ch.array([0, 0, 8]), ch.zeros(3)
f.v = translation + V.dot(Rodrigues(rotation))
observed = f.r
np.random.seed(1)
# this is reactive
# in the sense that changes to values will affect function which depend on them.
translation[:] = translation.r + np.random.rand(3)
rotation[:] = rotation.r + np.random.rand(3) * .2
# Create the energy
E_raw = f - observed
E_pyr = gaussian_pyramid(E_raw, n_levels=6, normalization='size')
Image.fromarray((observed * 255).astype(np.uint8)).save(
os.path.join(save_dir, "reference.png"))
step = 0
Image.fromarray((f.r * 255).astype(np.uint8)).save(
os.path.join(save_dir, "step_{:05d}.png".format(step)))
print('OPTIMIZING TRANSLATION, ROTATION, AND LIGHT PARMS')
free_variables = [translation, rotation]
ch.minimize({'pyr': E_pyr}, x0=free_variables, callback=create_callback(f))
ch.minimize({'raw': E_raw}, x0=free_variables, callback=create_callback(f))
def get_cam_params(self):
v_raw = np.sin(np.arange(900)).reshape((-1,3))
v_raw[:, 2] -= 2
rt = ch.zeros(3)
t = ch.zeros(3)
f = ch.array([500,500])
c = ch.array([320,240])
k = ch.zeros(5)
cam_params = {'v': ch.Ch(v_raw), 'rt': rt, 't': t, 'f': f, 'c': c, 'k': k}
return cam_params
def get_cam_params(self):
v_raw = np.sin(np.arange(900)).reshape((-1,3))
v_raw[:, 2] -= 2
rt = ch.zeros(3)
t = ch.zeros(3)
f = ch.array([500,500])
c = ch.array([320,240])
k = ch.zeros(5)
cam_params = {'v': ch.Ch(v_raw), 'rt': rt, 't': t, 'f': f, 'c': c, 'k': k}
return cam_params
def prepare_network_input_semantic(img_dir, zip_intermedia_results=True):
log_str = ''
img_size = 384
# create semantic map
smpl = objio.load_obj_data_binary(img_dir[:-4] + '_smpl_2.obj')
smpl_volume = voxel_util.voxelize_2(img_dir[:-4] + '_smpl_2.obj',
dim_h, dim_w, VOXELIZER_PATH)
smpl_volume = voxel_util.binary_fill_from_corner_3D(smpl_volume)
smpl_v_volume = voxel_util.calc_vmap_volume(smpl_volume, smpl['v'],
dim_h, dim_w, voxel_size)
mesh_volume = np.zeros((dim_w, dim_h, dim_w), dtype=np.float32)
sio.savemat(img_dir[:-4] + '_volume.mat',
{'mesh_volume': mesh_volume, 'smpl_v_volume': smpl_v_volume},
do_compression=True)
# render semantic map
smpl_v_ = renderers.compress_along_z_axis_single(smpl['v'])
u = renderers.project_vertices(smpl_v_, img_size, img_size,
cam_r=ch.array((3.14, 0, 0)),
cam_t=ch.array((0, 0, 0)))
smpl_vc = dutil.get_smpl_semantic_code()
v_map = renderers.render_color_model_without_lighting(img_size, img_size,
smpl_v_, smpl_vc,
smpl['f'], u,
bg_img=None)
v_map = np.float32(np.copy(v_map))
cv.imwrite(img_dir[:-4] + '_vmap.png', np.uint8(v_map*255))
# test orthogonal projection from SMPL's volume
smpl_volume_proj = np.max(smpl_volume, axis=-1)
smpl_volume_proj = np.flipud(np.transpose(smpl_volume_proj))
smpl_volume_proj = cv.resize(np.uint8(smpl_volume_proj) * 255, (256, 384))
cv.imwrite(img_dir[:-4] + '_test_proj.png', smpl_volume_proj)
# save smpl semantic volume for visualization
voxel_util.save_v_volume(smpl_v_volume, img_dir[:-4] + '_v_volume.obj',
dim_h, dim_w, voxel_size)
suffixes = ['_final.txt', '_smpl.obj', '_smpl_2.obj', '_smpl_proj.png',
'_test_proj.png', '_v_volume.obj']
if zip_intermedia_results:
z = zipfile.ZipFile(img_dir[:-4] + '_intermediate_prepare.zip', 'w')
for suffix in suffixes:
z.write(img_dir[:-4] + suffix)
os.remove(img_dir[:-4] + suffix)
z.close()
print(log_str)
return log_str
def initialize_camera(model,
j2d,
img,
init_pose,
flength=flength):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on
both the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be
considered, 3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
rt = ch.zeros(3) # initial camera rotation
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t) # initial camera translation
opt_pose = ch.array(init_pose)
_, A_global = global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera and project SMPL joints
cam = ProjectPoints(f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
cam.v = Jtr # project SMPL joints
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables, method='dogleg', callback=None,
options={'maxiter': 100, 'e_3': .0001, 'disp': 0})
return cam, opt_pose[:3].r
def render(self, thetas, texture_bgr, rotate=np.array([0, 0, 0]), background_img=None):
"""
get the rendered image and rendered silhouette
:param thetas: model parameters, 3 * camera parameter + 72 * body pose + 10 * body shape
:param texture_bgr: texture image in bgr format
:return: the rendered image and deviation of rendered image to texture image
(rendered image, deviation of rendered image, silhouette)
"""
self.set_texture(texture_bgr)
thetas = thetas.reshape(-1)
cams = thetas[:self.num_cam]
theta = thetas[self.num_cam: (self.num_cam + self.num_theta)]
beta = thetas[(self.num_cam + self.num_theta):]
self.body.pose[:] = theta
self.body.betas[:] = beta
#
# size = cams[0] * min(self.w, self.h)
# position = cams[1:3] * min(self.w, self.h) / 2 + min(self.w, self.h) / 2
"""
####################################################################
ATTENTION!
I do not know why the flength is 500.
But it worked
####################################################################
"""
texture_rn = TexturedRenderer()
texture_rn.camera = ProjectPoints(v=self.body, rt=rotate, t=ch.array([0, 0, 2]),
f=np.ones(2) * self.img_size * 0.62,
c=np.array([self.w / 2, self.h / 2]),
k=ch.zeros(5))
texture_rn.frustum = {'near': 1., 'far': 10., 'width': self.w, 'height': self.h}
texture_rn.set(v=self.body, f=self.m.f, vc=self.m.vc, texture_image=self.m.texture_image, ft=self.m.ft,
vt=self.m.vt)
if background_img is not None:
texture_rn.background_image = background_img / 255. if background_img.max() > 1 else background_img
silhouette_rn = ColoredRenderer()
silhouette_rn.camera = ProjectPoints(v=self.body, rt=rotate, t=ch.array([0, 0, 2]),
f=np.ones(2) * self.img_size * 0.62,
c=np.array([self.w / 2, self.h / 2]),
k=ch.zeros(5))
silhouette_rn.frustum = {'near': 1., 'far': 10., 'width': self.w, 'height': self.h}
silhouette_rn.set(v=self.body, f=self.m.f, vc=np.ones_like(self.body), bgcolor=np.zeros(3))
return texture_rn.r, texture_dr_wrt(texture_rn, silhouette_rn.r), silhouette_rn.r
def ready_arguments(fname_or_dict):
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict))
else:
dd = fname_or_dict
backwards_compatibility_replacements(dd) #insert or change the name of some index
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1]*3 #shape[1] = 24, shape[0] = 2, why shape[0] = 2?
# *3 is to sca.e the kintree_table to QUATERNION
if 'trans' not in dd:
dd['trans'] = np.zeros(3) # why trans = (0,0,0) what trans?
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms) # dim of pose is 24*3 = 72
if 'shapedirs' in dd and 'betas' not in dd: # dim of shapedirs 6890 * 3 * 10
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1]) # dim of beta 10
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
return dd
def sample_view(self, novel_view):
if novel_view:
cam_t = ch.array((0, 0, np.random.uniform(1.5, 2.5)))
theta = np.random.uniform(0, np.pi)
phi = np.random.uniform(0, 2 * np.pi)
angle = np.random.uniform(0, 2 * np.pi)
else:
cam_t = ch.array((0, 0, 2.0))
theta = np.random.uniform(0, 0.3)
phi = np.random.uniform(0, 2 * np.pi)
angle = np.random.uniform(0, 2 * np.pi)
cam_r = np.array([np.sin(theta) * np.cos(phi), np.cos(theta), np.sin(theta) * np.sin(phi)])
cam_r = ch.array(cam_r * angle)
return cam_t, cam_r
def compute_imu_data(gender, betas, poses, frame_rate):
if gender == 'male':
Jdirs = Jdirs_male
model = model_male
else:
Jdirs = Jdirs_female
model = model_female
betas[:] = 0
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
A_global_list = []
print('Length of poses: %d' % (len(poses) / 1))
for idx, p in enumerate(poses):
(_, A_global) = global_rigid_transformation(p,
J_onbetas,
model.kintree_table,
xp=ch)
A_global_list.append(A_global)
vertex = []
for idx, p in enumerate(poses):
model.pose[:] = p
model.betas[:] = 0
model.betas[:10] = betas
tmp = model.r[VERTEX_IDS]
vertex.append(tmp) # 6*3
orientation, acceleration = get_ori_accel(A_global_list, vertex,
frame_rate)
return orientation, acceleration
def guess_init(model, focal_length, j2d, init_pose):
"""Initialize the camera translation via triangle similarity, by using the torso joints .
:param model: SMPL model
:param focal_length: camera focal length (kept fixed)
:param j2d: 14x2 array of CNN joints
:param init_pose: 72D vector of pose parameters used for initialization (kept fixed)
:returns: 3D vector corresponding to the estimated camera translation
"""
cids = np.arange(0, 12)
# map from LSP to SMPL joints
j2d_here = j2d[cids]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(
opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
Jtr = Jtr[smpl_ids].r
# 9 is L shoulder, 3 is L hip
# 8 is R shoulder, 2 is R hip
diff3d = np.array([Jtr[9] - Jtr[3], Jtr[8] - Jtr[2]])
mean_height3d = np.mean(np.sqrt(np.sum(diff3d**2, axis=1)))
diff2d = np.array([j2d_here[9] - j2d_here[3], j2d_here[8] - j2d_here[2]])
mean_height2d = np.mean(np.sqrt(np.sum(diff2d**2, axis=1)))
est_d = focal_length * (mean_height3d / mean_height2d)
# just set the z value
init_t = np.array([0., 0., est_d])
return init_t
def render_color_model_with_lighting(w, h, v, vn, vc, f, u,
sh_comps=None, light_c=ch.ones(3),
vlight_pos=None, vlight_color=None,
bg_img=None):
"""renders colored model with lighting effect"""
assert(sh_comps is not None or vlight_pos is not None)
V = ch.array(v)
A = np.zeros_like(v)
# SH lighting
if sh_comps is not None:
A += vc * SphericalHarmonics(vn=vn, components=sh_comps, light_color=light_c)
# single point lighting (grey light)
if vlight_color is not None and vlight_pos is not None \
and len(vlight_pos.shape) == 1:
A += LambertianPointLight(f=f, v=v, num_verts=len(v), light_pos=vlight_pos,
light_color=vlight_color, vc=vc)
# multiple point lighting (grey light)
if vlight_color is not None and vlight_pos is not None \
and len(vlight_pos.shape) == 2:
for vlp in vlight_pos:
A += LambertianPointLight(f=f, v=v, num_verts=len(v), light_pos=vlp,
light_color=vlight_color, vc=vc)
black_img = np.array(np.zeros((w, h, 3)), dtype=np.float32)
bg_img_ = bg_img if bg_img is not None else black_img
rn = ColoredRenderer(camera=u, v=V, f=f, vc=A, background_image=bg_img_,
frustum={'width': w, 'height': h, 'near': 0.1, 'far': 20})
return rn.r
def unproject_depth_image(self, depth_image, camera_space=False):
us = np.arange(depth_image.size) % depth_image.shape[1]
vs = np.arange(depth_image.size) // depth_image.shape[1]
ds = depth_image.ravel()
uvd = ch.array(np.vstack((us.ravel(), vs.ravel(), ds.ravel())).T)
xyz = self.unproject_points(uvd, camera_space=camera_space)
return xyz.reshape((depth_image.shape[0], depth_image.shape[1], -1))
def _global_rigid_transformation(self):
results = {}
pose = self.pose.reshape((-1, 3))
parent = {
i: self.kintree_table[0, i]
for i in range(1, self.kintree_table.shape[1])
}
with_zeros = lambda x: ch.vstack((x, ch.array([[0.0, 0.0, 0.0, 1.0]])))
pack = lambda x: ch.hstack([ch.zeros((4, 3)), x.reshape((4, 1))])
results[0] = with_zeros(
ch.hstack((Rodrigues(pose[0, :]), self.J[0, :].reshape((3, 1)))))
for i in range(1, self.kintree_table.shape[1]):
results[i] = results[parent[i]].dot(
with_zeros(
ch.hstack((
Rodrigues(pose[i, :]), # rotation around bone endpoint
(self.J[i, :] - self.J[parent[i], :]).reshape(
(3, 1)) # bone
))))
results = [results[i] for i in sorted(results.keys())]
results_global = results
# subtract rotated J position
results2 = [
results[i] -
(pack(results[i].dot(ch.concatenate((self.J[i, :], [0])))))
for i in range(len(results))
]
result = ch.dstack(results2)
return result, results_global
def render_mask(w, h, v, f, u):
"""renders silhouette"""
V = ch.array(v)
A = np.ones(v.shape)
rn = ColoredRenderer(camera=u, v=V, f=f, vc=A, bgcolor=ch.zeros(3),
frustum={'width': w, 'height': h, 'near': 0.1, 'far': 20})
return rn.r
def guess_init(model, focal_length, j2d, init_pose):
"""Initialize the camera translation via triangle similarity, by using the torso joints .
:param model: SMPL model
:param focal_length: camera focal length (kept fixed)
:param j2d: 14x2 array of CNN joints
:param init_pose: 72D vector of pose parameters used for initialization (kept fixed)
:returns: 3D vector corresponding to the estimated camera translation
"""
cids = np.arange(0, 12)
# map from LSP to SMPL joints
j2d_here = j2d[cids]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
Jtr = Jtr[smpl_ids].r
# 9 is L shoulder, 3 is L hip
# 8 is R shoulder, 2 is R hip
diff3d = np.array([Jtr[9] - Jtr[3], Jtr[8] - Jtr[2]])
mean_height3d = np.mean(np.sqrt(np.sum(diff3d**2, axis=1)))
diff2d = np.array([j2d_here[9] - j2d_here[3], j2d_here[8] - j2d_here[2]])
mean_height2d = np.mean(np.sqrt(np.sum(diff2d**2, axis=1)))
est_d = focal_length * (mean_height3d / mean_height2d)
# just set the z value
init_t = np.array([0., 0., est_d])
return init_t
def unproject_depth_image(self, depth_image, camera_space=False):
us = np.arange(depth_image.size) % depth_image.shape[1]
vs = np.arange(depth_image.size) // depth_image.shape[1]
ds = depth_image.ravel()
uvd = ch.array(np.vstack((us.ravel(), vs.ravel(), ds.ravel())).T)
xyz = self.unproject_points(uvd, camera_space=camera_space)
return xyz.reshape((depth_image.shape[0], depth_image.shape[1], -1))
def ready_arguments(fname_or_dict):
if not isinstance(fname_or_dict, dict):
dd = pickle.load(open(fname_or_dict, 'rb'), encoding="latin1")
else:
dd = fname_or_dict
backwards_compatibility_replacements(dd)
want_shapemodel = 'shapedirs' in dd
nposeparms = dd['kintree_table'].shape[1]*3
if 'trans' not in dd:
dd['trans'] = np.zeros(3)
if 'pose' not in dd:
dd['pose'] = np.zeros(nposeparms)
if 'shapedirs' in dd and 'betas' not in dd:
dd['betas'] = np.zeros(dd['shapedirs'].shape[-1])
for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
if want_shapemodel:
dd['v_shaped'] = dd['shapedirs'].dot(dd['betas'])+dd['v_template']
v_shaped = dd['v_shaped']
J_tmpx = MatVecMult(dd['J_regressor'], v_shaped[:,0])
J_tmpy = MatVecMult(dd['J_regressor'], v_shaped[:,1])
J_tmpz = MatVecMult(dd['J_regressor'], v_shaped[:,2])
dd['J'] = ch.vstack((J_tmpx, J_tmpy, J_tmpz)).T
dd['v_posed'] = v_shaped + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
else:
dd['v_posed'] = dd['v_template'] + dd['posedirs'].dot(posemap(dd['bs_type'])(dd['pose']))
return dd
def test_derivatives2(self):
import chumpy as ch
import numpy as np
from renderer import DepthRenderer
rn = DepthRenderer()
# Assign attributes to renderer
from util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
w, h = (320, 240)
from camera import ProjectPoints
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=m.v, f=m.f, bgcolor=ch.zeros(3))
if visualize:
import matplotlib.pyplot as plt
plt.ion()
plt.figure()
for which in range(3):
r1 = rn.r
adder = np.random.rand(rn.v.r.size).reshape(rn.v.r.shape)*.01
change = rn.v.r * 0 + adder
dr_pred = rn.dr_wrt(rn.v).dot(change.ravel()).reshape(rn.shape)
rn.v = rn.v.r + change
r2 = rn.r
dr_emp = r2 - r1
#print np.mean(np.abs(dr_pred-dr_emp))
self.assertLess(np.mean(np.abs(dr_pred-dr_emp)), .024)
if visualize:
plt.subplot(2,3,which+1)
plt.imshow(dr_pred)
plt.clim(-.01,.01)
plt.title('emp')
plt.subplot(2,3,which+4)
plt.imshow(dr_emp)
plt.clim(-.01,.01)
plt.title('pred')
plt.draw()
plt.show()
def guess_init(
model, # pylint: disable=too-many-locals, too-many-arguments
focal_length,
j2d,
weights,
init_pose,
use_gt_guess):
"""Initialize the camera depth using the torso points."""
cids = _np.arange(0, 12)
j2d_here = j2d[cids]
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# The initial pose is a t-pose with all body parts planar to the camera
# (i.e., they have their maximum length).
opt_pose = _ch.array(init_pose) # pylint: disable=no-member
(_, A_global) = _global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=_ch)
Jtr = _ch.vstack([g[:3, 3] for g in A_global]) # pylint: disable=no-member
Jtr = Jtr[smpl_ids].r
if use_gt_guess:
# More robust estimate:
# Since there is no fore'lengthening', only foreshortening, take the longest
# visible body part as a robust estimate in the case of noise-free data.
longest_part = None
longest_part_length = -1.
for conn in connections_lsp:
if (conn[0] < 12 and conn[1] < 12 and weights[conn[0]] > 0.
and weights[conn[1]] > 0.):
part_length_proj_gt = _np.sqrt(
_np.sum(_np.array(j2d_here[conn[0]] -
j2d_here[conn[1]])**2,
axis=0))
if part_length_proj_gt > longest_part_length:
longest_part_length = part_length_proj_gt
longest_part = conn
#length2d = _np.sqrt(_np.sum(
# _np.array(j2d_here[longest_part[0]] - j2d_here[longest_part[1]])**2, axis=0))
part_length_3D = _np.sqrt(
_np.sum(_np.array(Jtr[longest_part[0]] - Jtr[longest_part[1]])**2,
axis=0))
est_d = focal_length * (part_length_3D / longest_part_length)
else:
diff3d = _np.array([Jtr[9] - Jtr[3], Jtr[8] - Jtr[2]])
mean_height3d = _np.mean(_np.sqrt(_np.sum(diff3d**2, axis=1)))
diff2d = _np.array(
[j2d_here[9] - j2d_here[3], j2d_here[8] - j2d_here[2]])
mean_height2d = _np.mean(_np.sqrt(_np.sum(diff2d**2, axis=1)))
f = focal_length # pylint: disable=redefined-outer-name
if mean_height2d == 0.:
_LOGGER.warn(
"Depth can not be correctly estimated. Guessing wildly.")
est_d = 60.
else:
est_d = f * (mean_height3d / mean_height2d)
init_t = _np.array([0., 0., est_d])
return init_t
def on_changed(self, which):
if 'model' in which:
if not isinstance(self.model, dict):
dd = pkl.load(open(self.model, "rb"), encoding="latin1")
else:
dd = self.model
backwards_compatibility_replacements(dd)
# for s in ['v_template', 'weights', 'posedirs', 'pose', 'trans', 'shapedirs', 'betas', 'J']:
for s in ['posedirs', 'shapedirs']:
if (s in dd) and not hasattr(dd[s], 'dterms'):
dd[s] = ch.array(dd[s])
self.f = dd['f']
self.shapedirs = dd['shapedirs']
self.J_regressor = dd['J_regressor']
if 'J_regressor_prior' in dd:
self.J_regressor_prior = dd['J_regressor_prior']
self.bs_type = dd['bs_type']
self.bs_style = dd['bs_style']
self.weights = ch.array(dd['weights'])
if 'vert_sym_idxs' in dd:
self.vert_sym_idxs = dd['vert_sym_idxs']
if 'weights_prior' in dd:
self.weights_prior = dd['weights_prior']
self.kintree_table = dd['kintree_table']
self.posedirs = dd['posedirs']
if not hasattr(self, 'betas'):
self.betas = ch.zeros(self.shapedirs.shape[-1])
if not hasattr(self, 'trans'):
self.trans = ch.zeros(3)
if not hasattr(self, 'pose'):
self.pose = ch.zeros(72)
if not hasattr(self, 'v_template'):
self.v_template = ch.array(dd['v_template'])
if not hasattr(self, 'v_personal'):
self.v_personal = ch.zeros_like(self.v_template)
self._set_up()
def make_prdicted_mesh_neutral(predicted_params_path, flame_model_path):
params = np.load(predicted_params_path,
allow_pickle=True,
encoding='latin1')
#print(params)
params = params[()]
pose = np.zeros(15)
#expression = np.zeros(100)
shape = np.hstack(
(params['shape'], np.zeros(300 - params['shape'].shape[0])))
#pose = np.hstack((params['pose'], np.zeros(15-params['pose'].shape[0])))
expression = np.hstack(
(params['expression'], np.zeros(100 - params['expression'].shape[0])))
flame_genral_model = load_model(flame_model_path)
generated_neutral_mesh = verts_decorated(
#ch.array([0.0,0.0,0.0]),
ch.array(params['cam']),
ch.array(pose),
ch.array(flame_genral_model.r),
flame_genral_model.J_regressor,
ch.array(flame_genral_model.weights),
flame_genral_model.kintree_table,
flame_genral_model.bs_style,
flame_genral_model.f,
bs_type=flame_genral_model.bs_type,
posedirs=ch.array(flame_genral_model.posedirs),
betas=ch.array(
np.hstack((shape, expression))
), #betas=ch.array(np.concatenate((theta[0,75:85], np.zeros(390)))), #
shapedirs=ch.array(flame_genral_model.shapedirs),
want_Jtr=True)
# neutral_mesh = Mesh(v=generated_neutral_mesh.r, f=generated_neutral_mesh.f)
neutral_mesh = trimesh.Trimesh(vertices=generated_neutral_mesh.r,
faces=generated_neutral_mesh.f)
return neutral_mesh
def get_renderer():
import chumpy as ch
from opendr.everything import *
# Load mesh
m = load_mesh('/Users/matt/geist/OpenDR/test_dr/nasa_earth.obj')
m.v += ch.array([0,0,3.])
w, h = (320, 240)
trans = ch.array([[0,0,0]])
# Construct renderer
rn = TexturedRenderer()
rn.camera = ProjectPoints(v=m.v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
rn.set(v=trans+m.v, f=m.f, texture_image=m.texture_image[:,:,::-1], ft=m.ft, vt=m.vt, bgcolor=ch.zeros(3))
rn.vc = SphericalHarmonics(vn=VertNormals(v=rn.v, f=rn.f), components=ch.array([4.,0.,0.,0.]), light_color=ch.ones(3))
return rn
def optimize_on_joints(j2d,
model,
cam,
img,
prior,
try_both_orient,
body_orient,
n_betas=10,
regs=None,
conf=None,
viz=False):
"""Fit the model to the given set of joints, given the estimated camera
:param j2d: 14x2 array of CNN joints
:param model: SMPL model
:param cam: estimated camera
:param img: h x w x 3 image
:param prior: mixture of gaussians pose prior
:param try_both_orient: boolean, if True both body_orient and its flip are considered for the fit
:param body_orient: 3D vector, initialization for the body orientation
:param n_betas: number of shape coefficients considered during optimization
:param regs: regressors for capsules' axis and radius, if not None enables the interpenetration error term
:param conf: 14D vector storing the confidence values from the CNN
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the optimized model, its joints projected on image space, the camera translation
"""
t0 = time()
# define the mapping LSP joints -> SMPL joints
# cids are joints ids for LSP:
cids = range(12) + [13]
# joint ids for SMPL
# SMPL does not have a joint for head, instead we use a vertex for the head
# and append it later.
smpl_ids = [8, 5, 2, 1, 4, 7, 21, 19, 17, 16, 18, 20]
# the vertex id for the joint corresponding to the head
head_id = 411
# weights assigned to each joint during optimization;
# the definition of hips in SMPL and LSP is significantly different so set
# their weights to zero
base_weights = np.array(
[1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1], dtype=np.float64)
if try_both_orient:
flipped_orient = cv2.Rodrigues(body_orient)[0].dot(
cv2.Rodrigues(np.array([0., np.pi, 0]))[0])
flipped_orient = cv2.Rodrigues(flipped_orient)[0].ravel()
orientations = [body_orient, flipped_orient]
else:
orientations = [body_orient]
if try_both_orient:
# store here the final error for both orientations,
# and pick the orientation resulting in the lowest error
errors = []
svs = []
cams = []
for o_id, orient in enumerate(orientations):
# initialize the shape to the mean shape in the SMPL training set
betas = ch.zeros(n_betas)
# initialize the pose by using the optimized body orientation and the
# pose prior
init_pose = np.hstack((orient, prior.weights.dot(prior.means)))
# instantiate the model:
# verts_decorated allows us to define how many
# shape coefficients (directions) we want to consider (here, n_betas)
sv = verts_decorated(
trans=ch.zeros(3),
pose=ch.array(init_pose),
v_template=model.v_template,
J=model.J_regressor,
betas=betas,
shapedirs=model.shapedirs[:, :, :n_betas],
weights=model.weights,
kintree_table=model.kintree_table,
bs_style=model.bs_style,
f=model.f,
bs_type=model.bs_type,
posedirs=model.posedirs)
# make the SMPL joints depend on betas
Jdirs = np.dstack([model.J_regressor.dot(model.shapedirs[:, :, i])
for i in range(len(betas))])
J_onbetas = ch.array(Jdirs).dot(betas) + model.J_regressor.dot(
model.v_template.r)
# get joint positions as a function of model pose, betas and trans
(_, A_global) = global_rigid_transformation(
sv.pose, J_onbetas, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global]) + sv.trans
# add the head joint, corresponding to a vertex...
Jtr = ch.vstack((Jtr, sv[head_id]))
# ... and add the joint id to the list
if o_id == 0:
smpl_ids.append(len(Jtr) - 1)
# update the weights using confidence values
weights = base_weights * conf[
cids] if conf is not None else base_weights
# project SMPL joints on the image plane using the estimated camera
cam.v = Jtr
# data term: distance between observed and estimated joints in 2D
obj_j2d = lambda w, sigma: (
w * weights.reshape((-1, 1)) * GMOf((j2d[cids] - cam[smpl_ids]), sigma))
# mixture of gaussians pose prior
pprior = lambda w: w * prior(sv.pose)
# joint angles pose prior, defined over a subset of pose parameters:
# 55: left elbow, 90deg bend at -np.pi/2
# 58: right elbow, 90deg bend at np.pi/2
# 12: left knee, 90deg bend at np.pi/2
# 15: right knee, 90deg bend at np.pi/2
alpha = 10
my_exp = lambda x: alpha * ch.exp(x)
obj_angle = lambda w: w * ch.concatenate([my_exp(sv.pose[55]), my_exp(-sv.pose[
58]), my_exp(-sv.pose[12]), my_exp(-sv.pose[15])])
if viz:
import matplotlib.pyplot as plt
plt.ion()
def on_step(_):
"""Create visualization."""
plt.figure(1, figsize=(10, 10))
plt.subplot(1, 2, 1)
# show optimized joints in 2D
tmp_img = img.copy()
for coord, target_coord in zip(
np.around(cam.r[smpl_ids]).astype(int),
np.around(j2d[cids]).astype(int)):
if (coord[0] < tmp_img.shape[1] and coord[0] >= 0 and
coord[1] < tmp_img.shape[0] and coord[1] >= 0):
cv2.circle(tmp_img, tuple(coord), 3, [0, 0, 255])
if (target_coord[0] < tmp_img.shape[1] and
target_coord[0] >= 0 and
target_coord[1] < tmp_img.shape[0] and
target_coord[1] >= 0):
cv2.circle(tmp_img, tuple(target_coord), 3,
[0, 255, 0])
plt.imshow(tmp_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-2)
on_step(_)
else:
on_step = None
if regs is not None:
# interpenetration term
sp = SphereCollisions(
pose=sv.pose, betas=sv.betas, model=model, regs=regs)
sp.no_hands = True
# weight configuration used in the paper, with joints + confidence values from the CNN
# (all the weights used in the code were obtained via grid search, see the paper for more details)
# the first list contains the weights for the pose priors,
# the second list contains the weights for the shape prior
opt_weights = zip([4.04 * 1e2, 4.04 * 1e2, 57.4, 4.78],
[1e2, 5 * 1e1, 1e1, .5 * 1e1])
# run the optimization in 4 stages, progressively decreasing the
# weights for the priors
for stage, (w, wbetas) in enumerate(opt_weights):
_LOGGER.info('stage %01d', stage)
objs = {}
objs['j2d'] = obj_j2d(1., 100)
objs['pose'] = pprior(w)
objs['pose_exp'] = obj_angle(0.317 * w)
objs['betas'] = wbetas * betas
if regs is not None:
objs['sph_coll'] = 1e3 * sp
ch.minimize(
objs,
x0=[sv.betas, sv.pose],
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
'disp': 0})
t1 = time()
_LOGGER.info('elapsed %.05f', (t1 - t0))
if try_both_orient:
errors.append((objs['j2d'].r**2).sum())
svs.append(sv)
cams.append(cam)
if try_both_orient and errors[0] > errors[1]:
choose_id = 1
else:
choose_id = 0
if viz:
plt.ioff()
return (svs[choose_id], cams[choose_id].r, cams[choose_id].t.r)
def initialize_camera(model,
j2d,
img,
init_pose,
flength=5000.,
pix_thsh=25.,
viz=False):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
_LOGGER.info('initializing translation via similar triangles')
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(
opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(
f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target (CNN) joints
for coord in np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0 and
coord[1] < img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and
coord[1] < viz_img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-3)
else:
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables,
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
def test_occlusion(self):
if visualize:
import matplotlib.pyplot as plt
plt.ion()
# Create renderer
import chumpy as ch
import numpy as np
from opendr.renderer import TexturedRenderer, ColoredRenderer
#rn = TexturedRenderer()
rn = ColoredRenderer()
# Assign attributes to renderer
from util_tests import get_earthmesh
m = get_earthmesh(trans=ch.array([0,0,4]), rotation=ch.zeros(3))
rn.texture_image = m.texture_image
rn.ft = m.ft
rn.vt = m.vt
m.v[:,2] = np.mean(m.v[:,2])
# red is front and zero
# green is back and 1
t0 = ch.array([1,0,.1])
t1 = ch.array([-1,0,.1])
v0 = ch.array(m.v) + t0
if False:
v1 = ch.array(m.v*.4 + np.array([0,0,3.8])) + t1
else:
v1 = ch.array(m.v) + t1
vc0 = v0*0 + np.array([[.4,0,0]])
vc1 = v1*0 + np.array([[0,.4,0]])
vc = ch.vstack((vc0, vc1))
v = ch.vstack((v0, v1))
f = np.vstack((m.f, m.f+len(v0)))
w, h = (320, 240)
rn.camera = ProjectPoints(v=v, rt=ch.zeros(3), t=ch.zeros(3), f=ch.array([w,w])/2., c=ch.array([w,h])/2., k=ch.zeros(5))
rn.camera.t = ch.array([0,0,-2.5])
rn.frustum = {'near': 1., 'far': 10., 'width': w, 'height': h}
m.vc = v.r*0 + np.array([[1,0,0]])
rn.set(v=v, f=f, vc=vc)
t0[:] = np.array([1.4, 0, .1-.02])
t1[:] = np.array([-0.6, 0, .1+.02])
target = rn.r
if visualize:
plt.figure()
plt.imshow(target)
plt.title('target')
plt.figure()
plt.show()
im_orig = rn.r.copy()
from cvwrap import cv2
tr = t0
eps_emp = .02
eps_pred = .02
#blur = lambda x : cv2.blur(x, ksize=(5,5))
blur = lambda x : x
for tr in [t0, t1]:
if tr is t0:
sum_limits = np.array([2.1e+2, 6.9e+1, 1.6e+2])
else:
sum_limits = [1., 5., 4.]
if visualize:
plt.figure()
for i in range(3):
dr_pred = np.array(rn.dr_wrt(tr[i]).todense()).reshape(rn.shape) * eps_pred
dr_pred = blur(dr_pred)
# central differences
tr[i] = tr[i].r + eps_emp/2.
rn_greater = rn.r.copy()
tr[i] = tr[i].r - eps_emp/1.
rn_lesser = rn.r.copy()
tr[i] = tr[i].r + eps_emp/2.
dr_emp = blur((rn_greater - rn_lesser) * eps_pred / eps_emp)
dr_pred_shown = np.clip(dr_pred, -.5, .5) + .5
dr_emp_shown = np.clip(dr_emp, -.5, .5) + .5
if visualize:
plt.subplot(3,3,i+1)
plt.imshow(dr_pred_shown)
plt.title('pred')
plt.axis('off')
plt.subplot(3,3,3+i+1)
plt.imshow(dr_emp_shown)
plt.title('empirical')
plt.axis('off')
plt.subplot(3,3,6+i+1)
diff = np.abs(dr_emp - dr_pred)
if visualize:
plt.imshow(diff)
diff = diff.ravel()
if visualize:
plt.title('diff (sum: %.2e)' % (np.sum(diff)))
plt.axis('off')
# print 'dr pred sum: %.2e' % (np.sum(np.abs(dr_pred.ravel())),)
# print 'dr emp sum: %.2e' % (np.sum(np.abs(dr_emp.ravel())),)
#import pdb; pdb.set_trace()
self.assertTrue(np.sum(diff) < sum_limits[i])
