def main():
fdir = dirname(__file__)
mesh = Mesh(filename='%s/teapot.obj' % fdir)
mv = MeshViewer()
mv.dynamic_meshes = [mesh]
print "~~\nThis should display a MeshViewer window with a teapot"
print "You can click and drag on the window to rotate around the mesh."
print "Press enter to continue. This will save the mesh as an obj file to your /tmp folder.\n~~\n"
raw_input()
outname = '/tmp/teapot.obj'
mesh.write_obj(outname)
print 'Saved file: ', outname
def lc(loss, res):
return
m = Mesh(v=res)
m2 = Mesh(v=j3d_gt)
m.show()
m2.show()
import time
time.sleep(3)
def wh(img_path):
print img_path
imgs, j2ds, segs, cams = util.load_data(img_path, util.NUM_VIEW)
#j2ds = tf.constant(j2ds, dtype=tf.float32)
initial_param, pose_mean, pose_covariance = util.load_initial_param()
pose_mean = tf.constant(pose_mean, dtype=tf.float32)
pose_covariance = tf.constant(pose_covariance, dtype=tf.float32)
param_shape = tf.Variable(initial_param[:10].reshape([1, -1]),
dtype=tf.float32)
param_rot = tf.Variable(initial_param[10:13].reshape([1, -1]),
dtype=tf.float32)
param_pose = tf.Variable(initial_param[13:82].reshape([1, -1]),
dtype=tf.float32)
param_trans = tf.Variable(initial_param[-3:].reshape([1, -1]),
dtype=tf.float32)
param = tf.concat([param_shape, param_rot, param_pose, param_trans],
axis=1)
smpl_model = SMPL(util.SMPL_PATH)
j3ds, v = smpl_model.get_3d_joints(param, util.SMPL_JOINT_IDS)
j3ds = tf.reshape(j3ds, [-1, 3])
j2ds_est = []
for idx in range(0, util.NUM_VIEW):
tmp = cams[idx].project(tf.squeeze(j3ds))
j2ds_est.append(tmp)
j2ds_est = tf.convert_to_tensor(j2ds_est)
#j2ds_est = tf.concat(j2ds_est, axis=0)
def lc(j2d_est):
_, ax = plt.subplots(1, 3)
for idx in range(0, util.NUM_VIEW):
import copy
tmp = copy.copy(imgs[idx])
for j2d in j2ds[idx]:
x = int(j2d[1])
y = int(j2d[0])
if x > imgs[0].shape[0] or x > imgs[0].shape[1]:
continue
tmp[x:x + 5, y:y + 5, :] = np.array([0, 0, 255])
for j2d in j2d_est[idx]:
x = int(j2d[1])
y = int(j2d[0])
if x > imgs[0].shape[0] or x > imgs[0].shape[1]:
continue
tmp[x:x + 5, y:y + 5, :] = np.array([255, 0, 0])
ax[idx].imshow(tmp)
plt.show()
if util.VIS_OR_NOT:
func_lc = lc
else:
func_lc = None
objs = {}
for idx in range(0, util.NUM_VIEW):
for j, jdx in enumerate(util.TORSO_IDS):
objs['J2D_%d_%d' % (idx, j)] = tf.reduce_sum(
tf.square(j2ds_est[idx][jdx] - j2ds[idx][jdx]))
loss = tf.reduce_mean(objs.values())
sess = tf.Session()
sess.run(tf.global_variables_initializer())
optimizer = scipy_pt(loss=loss,
var_list=[param_rot, param_trans],
options={
'ftol': 0.001,
'maxiter': 500,
'disp': True
},
method='L-BFGS-B')
optimizer.minimize(sess, fetches=[j2ds_est], loss_callback=func_lc)
objs = {}
pose_diff = tf.reshape(param_pose - pose_mean, [1, -1])
objs['J2D_Loss'] = tf.reduce_sum(tf.square(j2ds_est - j2ds))
objs['Prior_Loss'] = 5 * tf.squeeze(
tf.matmul(tf.matmul(pose_diff, pose_covariance),
tf.transpose(pose_diff)))
objs['Prior_Shape'] = 5 * tf.squeeze(tf.reduce_sum(tf.square(param_shape)))
loss = tf.reduce_mean(objs.values())
optimizer = scipy_pt(
loss=loss,
var_list=[param_rot, param_trans, param_pose, param_shape],
options={
'ftol': 0.001,
'maxiter': 500,
'disp': True
},
method='L-BFGS-B')
optimizer.minimize(sess, fetches=[j2ds_est], loss_callback=func_lc)
v_final = sess.run(v)
model_f = sess.run(smpl_model.f)
model_f = model_f.astype(int).tolist()
pose_final, betas_final, trans_final = sess.run(
[tf.concat([param_rot, param_pose], axis=1), param_shape, param_trans])
from psbody.meshlite import Mesh
m = Mesh(v=np.squeeze(v_final), f=model_f)
out_ply_path = img_path.replace('Image', 'Res_1')
extension = os.path.splitext(out_ply_path)[1]
out_ply_path = out_ply_path.replace(extension, '.ply')
m.write_ply(out_ply_path)
res = {'pose': pose_final, 'betas': betas_final, 'trans': trans_final}
out_pkl_path = out_ply_path.replace('.ply', '.pkl')
with open(out_pkl_path, 'wb') as fout:
pkl.dump(res, fout)
def project(self, points):
points = self.transform(points)
points = points + 1e-8
xs = tf.divide(points[:, 0], points[:, 2])
ys = tf.divide(points[:, 1], points[:, 2])
us = self.fl_x * xs + self.cx
vs = self.fl_y * ys + self.cy
#vs = 480 - vs
res = tf.stack([us, vs], axis=1)
return res
if __name__ == '__main__':
m = Mesh()
m.load_from_ply('~/Data/HEVA_Validate/S1_Box_1_C1/Res_1/frame0010.ply')
import cv2
img = cv2.imread('~/HEVA_Validate/S1_Box_1_C1/Image/frame0010.png')
import scipy.io as sio
cam_data = sio.loadmat('~/Data/HEVA_Validate/S1_Box_1_C1/GT/camera.mat',
squeeze_me=True,
struct_as_record=False)
cam_data = cam_data['camera']
cam = Perspective_Camera(cam_data.focal_length[0],
cam_data.focal_length[1],
cam_data.principal_pt[0],
cam_data.principal_pt[1], cam_data.t / 1000.0,
cam_data.R_angles)
v = tf.constant(m.v, dtype=tf.float32)
j2ds = cam.project(v)
def main(img_files):
imgs, j2ds, cams, poses, mean_betas, trans = util.load_data_temporal(
img_files)
j2ds = np.array(j2ds).reshape([-1, 2])
dct_mtx = util.load_dct_base()
dct_mtx = tf.constant(dct_mtx.T, dtype=tf.float32)
# For SMPL parameters
params_tem = []
params_pose_tem = []
param_shape = tf.constant(mean_betas, dtype=tf.float32)
for idx in range(0, util.BATCH_FRAME_NUM):
param_pose = tf.Variable(poses[idx],
dtype=tf.float32,
name='Pose_%d' % idx)
param_trans = tf.constant(trans[idx], dtype=tf.float32)
param = tf.concat([param_shape, param_pose, param_trans], axis=1)
params_tem.append(param)
params_pose_tem.append(param_pose)
params_tem = tf.concat(params_tem, axis=0)
# For DCT prior params
c_dct = tf.Variable(np.zeros(
[len(util.TEM_SMPL_JOINT_IDS), 3, util.DCT_NUM]),
dtype=tf.float32,
name='C_DCT')
smpl_model = SMPL(util.SMPL_PATH)
j3ds, vs = smpl_model.get_3d_joints(params_tem,
util.TEM_SMPL_JOINT_IDS) # N x M x 3
j3ds = j3ds[:, :-1]
j3ds_flatten = tf.reshape(j3ds, [-1, 3])
j2ds_est = []
for idx in range(0, util.NUM_VIEW):
tmp = cams[idx].project(j3ds_flatten)
j2ds_est.append(tmp)
j2ds_est = tf.concat(j2ds_est, axis=0)
j2ds_est = tf.reshape(
j2ds_est,
[util.NUM_VIEW, util.BATCH_FRAME_NUM,
len(util.TEM_SMPL_JOINT_IDS), 2])
j2ds_est = tf.transpose(j2ds_est, [1, 0, 2, 3])
j2ds_est = tf.reshape(j2ds_est, [-1, 2])
_, pose_mean, pose_covariance = util.load_initial_param()
pose_mean = tf.constant(pose_mean, dtype=tf.float32)
pose_covariance = tf.constant(pose_covariance, dtype=tf.float32)
objs = {}
objs['J2D_Loss'] = tf.reduce_sum(tf.square(j2ds_est - j2ds))
for i in range(0, util.BATCH_FRAME_NUM):
pose_diff = params_pose_tem[i][:, -69:] - pose_mean
objs['Prior_Loss_%d' % i] = 5 * tf.squeeze(
tf.matmul(tf.matmul(pose_diff, pose_covariance),
tf.transpose(pose_diff)))
for i, jid in enumerate(util.TEM_SMPL_JOINT_IDS):
for j, aid in enumerate([0, 1, 2]):
#for j, aid in enumerate([0, 2]):
trajectory = j3ds[:, i, aid]
'''
c_dct_initial = tf.matmul(tf.expand_dims(trajectory, axis=0), dct_mtx)
c_dct_initial = tf.squeeze(c_dct_initial)
'''
#import ipdb; ipdb.set_trace()
#with tf.control_dependencies( [tf.assign(c_dct[i, j], c_dct_initial)] ):
trajectory_dct = tf.matmul(dct_mtx,
tf.expand_dims(c_dct[i, j], axis=-1))
trajectory_dct = tf.squeeze(trajectory_dct)
objs['DCT_%d_%d' % (i, j)] = tf.reduce_sum(
tf.square(trajectory - trajectory_dct))
loss = tf.reduce_mean(objs.values())
if util.VIS_OR_NOT:
func_callback = on_step
else:
func_callback = None
sess = tf.Session()
sess.run(tf.global_variables_initializer())
def lc(j2d_est):
_, ax = plt.subplots(1, 3)
for idx in range(0, util.NUM_VIEW):
import copy
tmp = copy.copy(imgs[idx])
for j2d in j2ds[idx]:
x = int(j2d[1])
y = int(j2d[0])
if x > imgs[0].shape[0] or x > imgs[0].shape[1]:
continue
tmp[x:x + 5, y:y + 5, :] = np.array([0, 0, 255])
for j2d in j2d_est[idx]:
x = int(j2d[1])
y = int(j2d[0])
if x > imgs[0].shape[0] or x > imgs[0].shape[1]:
continue
tmp[x:x + 5, y:y + 5, :] = np.array([255, 0, 0])
ax[idx].imshow(tmp)
plt.show()
if util.VIS_OR_NOT:
func_lc = None
else:
func_lc = None
optimizer = scipy_pt(loss=loss,
var_list=params_pose_tem + [c_dct],
options={
'ftol': 0.001,
'maxiter': 500,
'disp': True
},
method='L-BFGS-B')
#optimizer.minimize(sess, fetches = [objs], loss_callback=func_lc)
optimizer.minimize(sess, loss_callback=func_lc)
print sess.run(c_dct)
vs_final = sess.run(vs)
pose_final = sess.run(params_pose_tem)
betas = sess.run(param_shape)
model_f = sess.run(smpl_model.f)
model_f = model_f.astype(int).tolist()
for fid in range(0, util.BATCH_FRAME_NUM / 2):
from psbody.meshlite import Mesh
m = Mesh(v=vs_final[fid], f=model_f)
out_ply_path = img_files[fid].replace('Image', 'Res_2')
extension = os.path.splitext(out_ply_path)[1]
out_ply_path = out_ply_path.replace(extension, '.ply')
m.write_ply(out_ply_path)
res = {'pose': pose_final[fid], 'betas': betas, 'trans': trans[fid]}
out_pkl_path = out_ply_path.replace('.ply', '.pkl')
print out_pkl_path
with open(out_pkl_path, 'wb') as fout:
pkl.dump(res, fout)
def project(self, points):
points = self.transform(points)
points = points + 1e-8
xs = tf.divide(points[:, 0], points[:, 2])
ys = tf.divide(points[:, 1], points[:, 2])
us = self.fl_x * xs + self.cx
vs = self.fl_y * ys + self.cy
#vs = 480 - vs
res = tf.stack([us, vs], axis=1)
return res
if __name__ == '__main__':
m = Mesh()
m.load_from_ply('/ps/scratch/yhuang/ESMPLify_4_12_Public/Data/HEVA_Validate/S1_Box_1_C1/Res_1/frame0010.ply')
import cv2
img = cv2.imread('/ps/scratch/yhuang/ESMPLify_4_12_Public/Data/HEVA_Validate/S1_Box_1_C1/Image/frame0010.png')
import scipy.io as sio
cam_data = sio.loadmat('/ps/scratch/yhuang/ESMPLify_4_12_Public/Data/HEVA_Validate/S1_Box_1_C1/GT/camera.mat', squeeze_me=True, struct_as_record=False)
cam_data = cam_data['camera']
cam = Perspective_Camera(cam_data.focal_length[0], cam_data.focal_length[1], cam_data.principal_pt[0], cam_data.principal_pt[1],
cam_data.t / 1000.0, cam_data.R_angles)
v = tf.constant(m.v, dtype=tf.float32)
j2ds = cam.project(v)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
j2ds = sess.run(j2ds)