def compute_texture_map(source_img, verts, faces, cam, texture_data):
'''
Given an image and a mesh aligned with the image (under scale-orthographic projection), project the image onto the
mesh and return a texture map.
'''
x_coords = texture_data.get('x_coords')
y_coords = texture_data.get('y_coords')
valid_pixel_ids = texture_data.get('valid_pixel_ids')
valid_pixel_3d_faces = texture_data.get('valid_pixel_3d_faces')
valid_pixel_b_coords = texture_data.get('valid_pixel_b_coords')
img_size = texture_data.get('img_size')
pixel_3d_points = verts[valid_pixel_3d_faces[:, 0], :] * valid_pixel_b_coords[:, 0][:, np.newaxis] + \
verts[valid_pixel_3d_faces[:, 1], :] * valid_pixel_b_coords[:, 1][:, np.newaxis] + \
verts[valid_pixel_3d_faces[:, 2], :] * valid_pixel_b_coords[:, 2][:, np.newaxis]
vertex_normals = Mesh(verts, faces).estimate_vertex_normals()
pixel_3d_normals = vertex_normals[valid_pixel_3d_faces[:, 0], :] * valid_pixel_b_coords[:, 0][:, np.newaxis] + \
vertex_normals[valid_pixel_3d_faces[:, 1], :] * valid_pixel_b_coords[:, 1][:, np.newaxis] + \
vertex_normals[valid_pixel_3d_faces[:, 2], :] * valid_pixel_b_coords[:, 2][:, np.newaxis]
n_dot_view = pixel_3d_normals[:,2]
proj_2d_points = ProjectPoints(f=cam[0] * np.ones(2), rt=np.zeros(3), t=np.zeros(3), k=np.zeros(5), c=cam[1:3])
proj_2d_points.v = pixel_3d_points
proj_2d_points = np.round(proj_2d_points.r).astype(int)
texture = np.zeros((img_size, img_size, 3))
for i, (x, y) in enumerate(proj_2d_points):
if n_dot_view[i] > 0.0:
continue
if x > 0 and x < source_img.shape[1] and y > 0 and y < source_img.shape[0]:
texture[y_coords[valid_pixel_ids[i]].astype(int), x_coords[valid_pixel_ids[i]].astype(int), :3] = source_img[y, x]
return texture
def initialize_camera(model,
j2d,
img,
init_pose,
flength=flength):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on
both the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be
considered, 3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
rt = ch.zeros(3) # initial camera rotation
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t) # initial camera translation
opt_pose = ch.array(init_pose)
_, A_global = global_rigid_transformation(opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera and project SMPL joints
cam = ProjectPoints(f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
cam.v = Jtr # project SMPL joints
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables, method='dogleg', callback=None,
options={'maxiter': 100, 'e_3': .0001, 'disp': 0})
return cam, opt_pose[:3].r
def main(mesh, sv_file, side, size, fmap_location, bmap_location, cam_file,
gar_type):
from opendr.camera import ProjectPoints
from psbody.mesh import Mesh
print(fmap_location)
cam_data = pkl.load(open(cam_file, 'r'))
cam_z, cam_y = cam_data[gar_type]['cam_z'], cam_data[gar_type]['cam_y']
mesh = Mesh(filename=mesh)
fmap = np.load(fmap_location)
bmap = np.load(bmap_location)
cam = ProjectPoints(v=mesh.v,
t=np.array([0, cam_y, cam_z]),
rt=np.zeros(3),
f=[1000, 1000],
c=[1000 / 2., 1000 / 2.],
k=np.zeros(5))
points = uv_to_xyz_and_normals(mesh, fmap, bmap)
cam.v = points
projection = cam.r.astype(np.int32)
projection[projection > 999] = 999
projection = np.fliplr(np.around(projection.squeeze())).astype(np.int32)
pixels_to_set = np.array(np.where(fmap != -1)).T
x_to_set = pixels_to_set[:, 0]
y_to_set = pixels_to_set[:, 1]
cords_ret = -999 * np.ones((fmap.shape[0], fmap.shape[1], 2))
cords_ret[x_to_set, y_to_set, :] = projection
cords_ret = cords_ret.astype('float64')
cords_ret = 2 * ((cords_ret) / 999) - 1
cords_ret = np.flip(cords_ret, 2)
if side == 'front':
cords_ret = cords_ret[500:1500, 0:1000]
else:
cords_ret = cords_ret[500:1500, 1000:2000]
cords_ret = cv2.resize(cords_ret, (size, size))
np.save(sv_file, cords_ret)
def proj_smpl_onto_img(img, smpl, pose, shape, cam_f, cam_t):
if isinstance(cam_f, float):
cam_f = np.array([cam_f, cam_f])
smpl.pose[:] = pose
smpl.betas[:] = shape
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
cam = ProjectPoints(
f=cam_f, rt=ch.zeros(3), t=cam_t, k=np.zeros(5), c=center)
cam.v = smpl.r
for v in cam.r:
r = int(round(v[1]))
c = int(round(v[0]))
if 0 <= r < img.shape[0] and 0 <= c < img.shape[1]:
img[r, c, :] = np.asarray([255, 255, 255])
return img
for i, (joints, pose, beta,
cam_t) in enumerate(zip(est.T, poses, betas, cam_ts)):
joints = joints[lsp_ids, :]
# Pose the model:
model.pose[:] = pose
# Shape the model: the model requires 10 dimensional betas,
# pad it with 0 if length of beta is less than 10
model.betas[:] = np.hstack((beta, np.zeros(10 - len(beta))))
# Set camera location.
cam.t = cam_t
# make it project SMPL joints in 3D.
cam.v = model.J_transformed[smpl_ids]
# projected SMPL joints in image coordinate.
smpl_joints = cam.r
# Render this.
res_im = (render_model(model.r, model.f, w, h, cam) *
255.).astype('uint8')
plt.show()
plt.subplot(121)
plt.imshow(np.ones((h, w, 3)))
plt.scatter(smpl_joints[:, 0], smpl_joints[:, 1], c='w')
plt.scatter(joints[:, 0], joints[:, 1], c=joints[:, 2])
plt.axis('off')
plt.subplot(122)
plt.cla()
def initialize_camera(model,
j2d,
img,
init_pose,
flength=5000.,
pix_thsh=25.,
viz=False):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
_LOGGER.info('initializing translation via similar triangles')
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(f=np.array([flength, flength]),
rt=rt,
t=t,
k=np.zeros(5),
c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target (CNN) joints
for coord in np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0
and coord[1] < img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0
and coord[1] < viz_img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-3)
else:
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{
'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=on_step,
options={
'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0
})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
for i, (joints, pose, beta,
cam_t) in enumerate(zip(est.T, poses, betas, cam_ts)):
joints = joints[lsp_ids, :]
# Pose the model:
model.pose[:] = pose
# Shape the model: the model requires 10 dimensional betas,
# pad it with 0 if length of beta is less than 10
model.betas[:] = np.hstack((beta, np.zeros(10 - len(beta))))
# Set camera location.
cam.t = cam_t
# make it project SMPL joints in 3D.
cam.v = model.J_transformed[smpl_ids]
# projected SMPL joints in image coordinate.
smpl_joints = cam.r
# Render this.
res_im = (render_model(model.r, model.f, w, h, cam) *
255.).astype('uint8')
plt.show()
plt.subplot(121)
plt.imshow(np.ones((h, w, 3)))
plt.scatter(smpl_joints[:, 0], smpl_joints[:, 1], c='w')
plt.scatter(joints[:, 0], joints[:, 1], c=joints[:, 2])
plt.axis('off')
plt.subplot(122)
plt.cla()
def initialize_camera(model, j2d, img, init_pose, flength=5000., pix_thsh=25.):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
print 'initializing translation via similar triangles'
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(opt_pose,
model.J,
model.kintree_table,
xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(f=np.array([flength, flength]),
rt=rt,
t=t,
k=np.zeros(5),
c=center)
# we are going to project the SMPL joints
cam.v = Jtr
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{
'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])
},
x0=free_variables,
method='dogleg',
callback=on_step,
# maxiter: 100, e_3: .0001
options={
'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0
})
return (cam, try_both_orient, opt_pose[:3].r)
def initialize_camera(model,
j2d,
img,
init_pose,
flength=5000.,
pix_thsh=25.,
viz=False):
"""Initialize camera translation and body orientation
:param model: SMPL model
:param j2d: 14x2 array of CNN joints
:param img: h x w x 3 image
:param init_pose: 72D vector of pose parameters used for initialization
:param flength: camera focal length (kept fixed)
:param pix_thsh: threshold (in pixel), if the distance between shoulder joints in 2D
is lower than pix_thsh, the body orientation as ambiguous (so a fit is run on both
the estimated one and its flip)
:param viz: boolean, if True enables visualization during optimization
:returns: a tuple containing the estimated camera,
a boolean deciding if both the optimized body orientation and its flip should be considered,
3D vector for the body orientation
"""
# optimize camera translation and body orientation based on torso joints
# LSP torso ids:
# 2=right hip, 3=left hip, 8=right shoulder, 9=left shoulder
torso_cids = [2, 3, 8, 9]
# corresponding SMPL torso ids
torso_smpl_ids = [2, 1, 17, 16]
center = np.array([img.shape[1] / 2, img.shape[0] / 2])
# initialize camera rotation
rt = ch.zeros(3)
# initialize camera translation
_LOGGER.info('initializing translation via similar triangles')
init_t = guess_init(model, flength, j2d, init_pose)
t = ch.array(init_t)
# check how close the shoulder joints are
try_both_orient = np.linalg.norm(j2d[8] - j2d[9]) < pix_thsh
opt_pose = ch.array(init_pose)
(_, A_global) = global_rigid_transformation(
opt_pose, model.J, model.kintree_table, xp=ch)
Jtr = ch.vstack([g[:3, 3] for g in A_global])
# initialize the camera
cam = ProjectPoints(
f=np.array([flength, flength]), rt=rt, t=t, k=np.zeros(5), c=center)
# we are going to project the SMPL joints
cam.v = Jtr
if viz:
viz_img = img.copy()
# draw the target (CNN) joints
for coord in np.around(j2d).astype(int):
if (coord[0] < img.shape[1] and coord[0] >= 0 and
coord[1] < img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 255, 0])
import matplotlib.pyplot as plt
plt.ion()
# draw optimized joints at each iteration
def on_step(_):
"""Draw a visualization."""
plt.figure(1, figsize=(5, 5))
plt.subplot(1, 1, 1)
viz_img = img.copy()
for coord in np.around(cam.r[torso_smpl_ids]).astype(int):
if (coord[0] < viz_img.shape[1] and coord[0] >= 0 and
coord[1] < viz_img.shape[0] and coord[1] >= 0):
cv2.circle(viz_img, tuple(coord), 3, [0, 0, 255])
plt.imshow(viz_img[:, :, ::-1])
plt.draw()
plt.show()
plt.pause(1e-3)
else:
on_step = None
# optimize for camera translation and body orientation
free_variables = [cam.t, opt_pose[:3]]
ch.minimize(
# data term defined over torso joints...
{'cam': j2d[torso_cids] - cam[torso_smpl_ids],
# ...plus a regularizer for the camera translation
'cam_t': 1e2 * (cam.t[2] - init_t[2])},
x0=free_variables,
method='dogleg',
callback=on_step,
options={'maxiter': 100,
'e_3': .0001,
# disp set to 1 enables verbose output from the optimizer
'disp': 0})
if viz:
plt.ioff()
return (cam, try_both_orient, opt_pose[:3].r)
for i, (joints, pose, beta,
cam_t) in enumerate(zip(est.T, poses, betas, cam_ts)):
joints = joints[lsp_ids, :]
# Pose the model:
# Model requires 300 dimensional betas
# we only used the first 10 principal components so our beta is 10-D.
model.pose[:] = pose
model.betas[:] = np.hstack((beta, np.zeros(300 - len(beta))))
# Set camera location.
cam.t = cam_t
# make it project SMPL joints in 3D.
cam.v = model.Jtr[smpl_ids]
# projected SMPL joints in image coordinate.
smpl_joints = cam.r
# Render this.
res_im = (render_model(model.r, model.f, w, h, cam) *
255.).astype('uint8')
plt.show()
plt.subplot(121)
plt.imshow(np.ones((h, w, 3)))
plt.scatter(smpl_joints[:, 0], smpl_joints[:, 1], c='w')
plt.scatter(joints[:, 0], joints[:, 1], c=joints[:, 2])
plt.axis('off')
plt.subplot(122)
plt.cla()
plt.imshow(res_im)