def eval(self):
self.coarse_net.eval()
self.refine_net.eval()
ds_name = self.cfg.dataset_dir.split('/')[-1]
total_error_cnet = {}
total_error_rnet = {}
for split, ds in [('val', self.ds_val), ('test', self.ds_test),
('train', self.ds_train)]:
mean_error_cnet = []
mean_error_rnet = []
with torch.no_grad():
for dorig in ds:
dorig = {k: dorig[k].to(self.device) for k in dorig.keys()}
MESH_SCALER = 1000
drec_cnet = self.coarse_net(**dorig)
verts_hand_cnet = self.rhm_train(**drec_cnet).vertices
mean_error_cnet.append(
torch.mean(
torch.abs(dorig['verts_rhand'] - verts_hand_cnet) *
MESH_SCALER))
########## refine net
params_rnet = self.params_rnet(dorig)
dorig.update(params_rnet)
drec_rnet = self.refine_net(**dorig)
verts_hand_mano = self.rhm_train(**drec_rnet).vertices
mean_error_rnet.append(
torch.mean(
torch.abs(dorig['verts_rhand'] - verts_hand_mano) *
MESH_SCALER))
total_error_cnet[split] = {
'v2v_mae': float(to_cpu(torch.stack(mean_error_cnet).mean()))
}
total_error_rnet[split] = {
'v2v_mae': float(to_cpu(torch.stack(mean_error_rnet).mean()))
}
outpath = makepath(os.path.join(
self.cfg.work_dir, 'evaluations', 'ds_%s' % ds_name,
os.path.basename(self.cfg.best_cnet).replace(
'.pt', '_CoarseNet.json')),
isfile=True)
with open(outpath, 'w') as f:
json.dump(total_error_cnet, f)
with open(outpath.replace('.json', '_RefineNet.json'), 'w') as f:
json.dump(total_error_rnet, f)
return total_error_cnet, total_error_rnet
def vis_results(dorig, coarse_net, refine_net, rh_model , save=False, save_dir = None):
with torch.no_grad():
imw, imh = 1920, 780
cols = len(dorig['bps_object'])
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
mvs = MeshViewers(window_width=imw, window_height=imh, shape=[1, cols], keepalive=True)
drec_cnet = coarse_net.sample_poses(dorig['bps_object'])
verts_rh_gen_cnet = rh_model(**drec_cnet).vertices
_, h2o, _ = point2point_signed(verts_rh_gen_cnet, dorig['verts_object'].to(device))
drec_cnet['trans_rhand_f'] = drec_cnet['transl']
drec_cnet['global_orient_rhand_rotmat_f'] = aa2rotmat(drec_cnet['global_orient']).view(-1, 3, 3)
drec_cnet['fpose_rhand_rotmat_f'] = aa2rotmat(drec_cnet['hand_pose']).view(-1, 15, 3, 3)
drec_cnet['verts_object'] = dorig['verts_object'].to(device)
drec_cnet['h2o_dist']= h2o.abs()
drec_rnet = refine_net(**drec_cnet)
verts_rh_gen_rnet = rh_model(**drec_rnet).vertices
for cId in range(0, len(dorig['bps_object'])):
try:
from copy import deepcopy
meshes = deepcopy(dorig['mesh_object'])
obj_mesh = meshes[cId]
except:
obj_mesh = points_to_spheres(to_cpu(dorig['verts_object'][cId]), radius=0.002, vc=name_to_rgb['green'])
hand_mesh_gen_cnet = Mesh(v=to_cpu(verts_rh_gen_cnet[cId]), f=rh_model.faces, vc=name_to_rgb['pink'])
hand_mesh_gen_rnet = Mesh(v=to_cpu(verts_rh_gen_rnet[cId]), f=rh_model.faces, vc=name_to_rgb['gray'])
if 'rotmat' in dorig:
rotmat = dorig['rotmat'][cId].T
obj_mesh = obj_mesh.rotate_vertices(rotmat)
hand_mesh_gen_cnet.rotate_vertices(rotmat)
hand_mesh_gen_rnet.rotate_vertices(rotmat)
hand_mesh_gen_cnet.reset_face_normals()
hand_mesh_gen_rnet.reset_face_normals()
# mvs[0][cId].set_static_meshes([hand_mesh_gen_cnet] + obj_mesh, blocking=True)
mvs[0][cId].set_static_meshes([hand_mesh_gen_rnet,obj_mesh], blocking=True)
if save:
save_path = os.path.join(save_dir, str(cId))
makepath(save_path)
hand_mesh_gen_rnet.write_ply(filename=save_path + '/rh_mesh_gen_%d.ply' % cId)
obj_mesh[0].write_ply(filename=save_path + '/obj_mesh_%d.ply' % cId)
def get_meshes(dorig, coarse_net, refine_net, rh_model, save=False, save_dir=None):
with torch.no_grad():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
drec_cnet = coarse_net.sample_poses(dorig['bps_object'])
verts_rh_gen_cnet = rh_model(**drec_cnet).vertices
_, h2o, _ = point2point_signed(verts_rh_gen_cnet, dorig['verts_object'].to(device))
drec_cnet['trans_rhand_f'] = drec_cnet['transl']
drec_cnet['global_orient_rhand_rotmat_f'] = aa2rotmat(drec_cnet['global_orient']).view(-1, 3, 3)
drec_cnet['fpose_rhand_rotmat_f'] = aa2rotmat(drec_cnet['hand_pose']).view(-1, 15, 3, 3)
drec_cnet['verts_object'] = dorig['verts_object'].to(device)
drec_cnet['h2o_dist'] = h2o.abs()
drec_rnet = refine_net(**drec_cnet)
verts_rh_gen_rnet = rh_model(**drec_rnet).vertices
gen_meshes = []
for cId in range(0, len(dorig['bps_object'])):
try:
obj_mesh = dorig['mesh_object'][cId]
except:
obj_mesh = points2sphere(points=to_cpu(dorig['verts_object'][cId]), radius=0.002, vc=name_to_rgb['yellow'])
hand_mesh_gen_rnet = Mesh(vertices=to_cpu(verts_rh_gen_rnet[cId]), faces=rh_model.faces, vc=[245, 191, 177])
if 'rotmat' in dorig:
rotmat = dorig['rotmat'][cId].T
obj_mesh = obj_mesh.rotate_vertices(rotmat)
hand_mesh_gen_rnet.rotate_vertices(rotmat)
gen_meshes.append([obj_mesh, hand_mesh_gen_rnet])
if save:
save_path = os.path.join(save_dir, str(cId))
makepath(save_path)
hand_mesh_gen_rnet.export(filename=save_path + '/rh_mesh_gen_%d.ply' % cId)
obj_mesh.export(filename=save_path + '/obj_mesh_%d.ply' % cId)
return gen_meshes
def vis_results(ho,
dorig,
coarse_net,
refine_net,
rh_model,
save=False,
save_dir=None,
rh_model_pkl=None,
vis=True):
# with torch.no_grad():
imw, imh = 1920, 780
cols = len(dorig['bps_object'])
# device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
device = torch.device('cpu')
if vis:
mvs = MeshViewers(window_width=imw,
window_height=imh,
shape=[1, cols],
keepalive=True)
# drec_cnet = coarse_net.sample_poses(dorig['bps_object'])
#
# for k in drec_cnet.keys():
# print('drec cnet', k, drec_cnet[k].shape)
# verts_rh_gen_cnet = rh_model(**drec_cnet).vertices
drec_cnet = {}
hand_pose_in = torch.Tensor(ho.hand_pose[3:]).unsqueeze(0)
mano_out_1 = rh_model_pkl(hand_pose=hand_pose_in)
hand_pose_in = mano_out_1.hand_pose
mTc = torch.Tensor(ho.hand_mTc)
approx_global_orient = rotmat2aa(mTc[:3, :3].unsqueeze(0))
if torch.isnan(approx_global_orient).any(): # Using honnotate?
approx_global_orient = torch.Tensor(ho.hand_pose[:3]).unsqueeze(0)
approx_global_orient = approx_global_orient.squeeze(1).squeeze(1)
approx_trans = mTc[:3, 3].unsqueeze(0)
target_verts = torch.Tensor(ho.hand_verts).unsqueeze(0)
pose, trans, rot = util.opt_hand(rh_model, target_verts, hand_pose_in,
approx_trans, approx_global_orient)
# drec_cnet['hand_pose'] = torch.einsum('bi,ij->bj', [hand_pose_in, rh_model_pkl.hand_components])
drec_cnet['transl'] = trans
drec_cnet['global_orient'] = rot
drec_cnet['hand_pose'] = pose
verts_rh_gen_cnet = rh_model(**drec_cnet).vertices
_, h2o, _ = point2point_signed(verts_rh_gen_cnet,
dorig['verts_object'].to(device))
drec_cnet['trans_rhand_f'] = drec_cnet['transl']
drec_cnet['global_orient_rhand_rotmat_f'] = aa2rotmat(
drec_cnet['global_orient']).view(-1, 3, 3)
drec_cnet['fpose_rhand_rotmat_f'] = aa2rotmat(drec_cnet['hand_pose']).view(
-1, 15, 3, 3)
drec_cnet['verts_object'] = dorig['verts_object'].to(device)
drec_cnet['h2o_dist'] = h2o.abs()
print(
'Hand fitting err',
np.linalg.norm(
verts_rh_gen_cnet.squeeze().detach().numpy() - ho.hand_verts, 2,
1).mean())
orig_obj = dorig['mesh_object'][0].v
# print(orig_obj.shape, orig_obj)
# print('Obj fitting err', np.linalg.norm(orig_obj - ho.obj_verts, 2, 1).mean())
drec_rnet = refine_net(**drec_cnet)
mano_out = rh_model(**drec_rnet)
verts_rh_gen_rnet = mano_out.vertices
joints_out = mano_out.joints
if vis:
for cId in range(0, len(dorig['bps_object'])):
try:
from copy import deepcopy
meshes = deepcopy(dorig['mesh_object'])
obj_mesh = meshes[cId]
except:
obj_mesh = points_to_spheres(to_cpu(
dorig['verts_object'][cId]),
radius=0.002,
vc=name_to_rgb['green'])
hand_mesh_gen_cnet = Mesh(v=to_cpu(verts_rh_gen_cnet[cId]),
f=rh_model.faces,
vc=name_to_rgb['pink'])
hand_mesh_gen_rnet = Mesh(v=to_cpu(verts_rh_gen_rnet[cId]),
f=rh_model.faces,
vc=name_to_rgb['gray'])
if 'rotmat' in dorig:
rotmat = dorig['rotmat'][cId].T
obj_mesh = obj_mesh.rotate_vertices(rotmat)
hand_mesh_gen_cnet.rotate_vertices(rotmat)
hand_mesh_gen_rnet.rotate_vertices(rotmat)
# print('rotmat', rotmat)
hand_mesh_gen_cnet.reset_face_normals()
hand_mesh_gen_rnet.reset_face_normals()
# mvs[0][cId].set_static_meshes([hand_mesh_gen_cnet] + obj_mesh, blocking=True)
# mvs[0][cId].set_static_meshes([hand_mesh_gen_rnet,obj_mesh], blocking=True)
mvs[0][cId].set_static_meshes(
[hand_mesh_gen_rnet, hand_mesh_gen_cnet, obj_mesh],
blocking=True)
if save:
save_path = os.path.join(save_dir, str(cId))
makepath(save_path)
hand_mesh_gen_rnet.write_ply(filename=save_path +
'/rh_mesh_gen_%d.ply' % cId)
obj_mesh[0].write_ply(filename=save_path +
'/obj_mesh_%d.ply' % cId)
return verts_rh_gen_rnet, joints_out
def get_meshes(dorig,
coarse_net,
refine_net,
rh_model,
save=False,
save_dir=None):
with torch.no_grad():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
drec_cnet = coarse_net.sample_poses(dorig['bps_object'])
output = rh_model(**drec_cnet)
verts_rh_gen_cnet = output.vertices
_, h2o, _ = point2point_signed(verts_rh_gen_cnet,
dorig['verts_object'].to(device))
drec_cnet['trans_rhand_f'] = drec_cnet['transl']
drec_cnet['global_orient_rhand_rotmat_f'] = aa2rotmat(
drec_cnet['global_orient']).view(-1, 3, 3)
drec_cnet['fpose_rhand_rotmat_f'] = aa2rotmat(
drec_cnet['hand_pose']).view(-1, 15, 3, 3)
drec_cnet['verts_object'] = dorig['verts_object'].to(device)
drec_cnet['h2o_dist'] = h2o.abs()
drec_rnet = refine_net(**drec_cnet)
output = rh_model(**drec_rnet)
print("hand shape {} should be idtenty".format(output.betas))
verts_rh_gen_rnet = output.vertices
# Reorder joints to match visualization utilities (joint_mapper) (TODO)
joints_rh_gen_rnet = output.joints # [:, [0, 13, 14, 15, 16, 1, 2, 3, 17, 4, 5, 6, 18, 10, 11, 12, 19, 7, 8, 9, 20]]
transforms_rh_gen_rnet = output.transforms # [:, [0, 13, 14, 15, 1, 2, 3, 4, 5, 6, 10, 11, 12, 7, 8, 9]]
joints_rh_gen_rnet = to_cpu(joints_rh_gen_rnet)
transforms_rh_gen_rnet = to_cpu(transforms_rh_gen_rnet)
gen_meshes = []
for cId in range(0, len(dorig['bps_object'])):
try:
obj_mesh = dorig['mesh_object'][cId]
except:
obj_mesh = points2sphere(points=to_cpu(
dorig['verts_object'][cId]),
radius=0.002,
vc=[145, 191, 219])
hand_mesh_gen_rnet = Mesh(vertices=to_cpu(verts_rh_gen_rnet[cId]),
faces=rh_model.faces,
vc=[145, 191, 219])
hand_joint_gen_rnet = joints_rh_gen_rnet[cId]
hand_transform_gen_rnet = transforms_rh_gen_rnet[cId]
if 'rotmat' in dorig:
rotmat = dorig['rotmat'][cId].T
obj_mesh = obj_mesh.rotate_vertices(rotmat)
hand_mesh_gen_rnet.rotate_vertices(rotmat)
hand_joint_gen_rnet = hand_joint_gen_rnet @ rotmat.T
hand_transform_gen_rnet[:, :, :3, :3] = np.matmul(
rotmat[None, ...], hand_transform_gen_rnet[:, :, :3, :3])
gen_meshes.append([obj_mesh, hand_mesh_gen_rnet])
if save:
makepath(save_dir)
print("saving dir {}".format(save_dir))
np.save(save_dir + '/joints_%d.npy' % cId, hand_joint_gen_rnet)
np.save(save_dir + '/trans_%d.npy' % cId,
hand_transform_gen_rnet)
return gen_meshes
