def get_mano_closed_faces():
"""https://github.com/hassony2/handobjectconsist/blob/master/meshreg/models/manoutils.py"""
mano_layer = manolayer.ManoLayer(
joint_rot_mode="axisang", use_pca=False, mano_root='.', center_idx=None, flat_hand_mean=True
)
close_faces = torch.Tensor(
[
[92, 38, 122],
[234, 92, 122],
[239, 234, 122],
[279, 239, 122],
[215, 279, 122],
[215, 122, 118],
[215, 118, 117],
[215, 117, 119],
[215, 119, 120],
[215, 120, 108],
[215, 108, 79],
[215, 79, 78],
[215, 78, 121],
[214, 215, 121],
]
)
closed_faces = torch.cat([mano_layer.th_faces, close_faces.long()])
# Indices of faces added during closing --> should be ignored as they match the wrist
# part of the hand, which is not an external surface of the human
# Valid because added closed faces are at the end
hand_ignore_faces = [1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, 1550, 1551]
return closed_faces.detach().cpu().numpy() #, hand_ignore_faces
def get_closed_faces():
mano_layer = manolayer.ManoLayer(joint_rot_mode="axisang",
use_pca=False,
mano_root="assets/mano",
center_idx=None,
flat_hand_mean=True)
close_faces = torch.Tensor([
[92, 38, 122],
[234, 92, 122],
[239, 234, 122],
[279, 239, 122],
[215, 279, 122],
[215, 122, 118],
[215, 118, 117],
[215, 117, 119],
[215, 119, 120],
[215, 120, 108],
[215, 108, 79],
[215, 79, 78],
[215, 78, 121],
[214, 215, 121],
])
closed_faces = torch.cat([mano_layer.th_faces, close_faces.long()])
# Indices of faces added during closing --> should be ignored as they match the wrist
# part of the hand, which is not an external surface of the human
# Valid because added closed faces are at the end
hand_ignore_faces = [
1538, 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549,
1550, 1551
]
return closed_faces, hand_ignore_faces
def recon_eval(op_shapes, pre_j3ds, gt_j3ds, visual, key):
pose0 = torch.eye(3).repeat(1, 16, 1, 1)
mano = manolayer.ManoLayer(flat_hand_mean=True,
side="right",
mano_root='mano/models',
use_pca=False,
root_rot_mode='rotmat',
joint_rot_mode='rotmat')
j3d_recons = []
evaluator = EvalUtil()
for i in tqdm(range(pre_j3ds.shape[0])):
j3d_pre = pre_j3ds[i]
op_shape = torch.tensor(op_shapes[i]).float().unsqueeze(0)
_, j3d_p0_ops = mano(pose0, op_shape)
template = j3d_p0_ops.cpu().numpy().squeeze() / 1000.0 # template, m
ratio = np.linalg.norm(template[9] -
template[0]) / np.linalg.norm(j3d_pre[9] -
j3d_pre[0])
j3d_pre_process = j3d_pre * ratio # template, m
j3d_pre_process = j3d_pre_process - j3d_pre_process[0] + template[0]
pose_R = AIK.adaptive_IK(template, j3d_pre_process)
pose_R = torch.from_numpy(pose_R).float()
# reconstruction
hand_verts, j3d_recon = mano(pose_R, op_shape.float())
# visualization
if visual:
demo.display_hand(
{
'verts': hand_verts.cpu(),
'joints': j3d_recon.cpu()
},
mano_faces=mano.th_faces)
j3d_recon = j3d_recon.cpu().numpy().squeeze() / 1000.
j3d_recons.append(j3d_recon)
# visualization
if visual:
vis.multi_plot3d([j3d_recon, j3d_pre_process],
title=["recon", "pre"])
j3d_recons = np.array(j3d_recons)
gt_joint, j3d_recon_align_gt = align.global_align(gt_j3ds,
j3d_recons,
key=key)
for targj, predj_a in zip(gt_joint, j3d_recon_align_gt):
evaluator.feed(targj * 1000.0, predj_a * 1000.0)
(_1, _2, _3, auc_all, pck_curve_all,
thresholds) = evaluator.get_measures(20, 50, 15)
print("Reconstruction AUC all of {}_test_set is : {}".format(key, auc_all))
def __init__(
self,
image_size,
model,
fill_back=True,
use_backward=True,
lambda_data=1,
lambda_consist=1,
criterion="l1",
consist_scale=1,
first_only=True,
gt_refs=True,
progressive_consist=True,
progressive_steps=1000,
):
super().__init__()
self.fill_back = fill_back
self.use_backward = use_backward
max_size = max(image_size)
self.image_size = image_size
self.lambda_data = lambda_data
self.lambda_consist = lambda_consist
self.consist_scale = consist_scale
self.criterion = pyramidloss.PyramidCriterion(criterion)
self.first_only = first_only
self.progressive_consist = progressive_consist
self.progressive_steps = progressive_steps
self.gt_refs = gt_refs
self.step_count = 0
neurenderer = renderer.Renderer(
image_size=max_size,
R=torch.eye(3).unsqueeze(0).cuda(),
t=torch.zeros(1, 3).cuda(),
K=torch.ones(1, 3, 3).cuda(),
orig_size=max_size,
anti_aliasing=False,
fill_back=fill_back,
near=0.1,
no_light=True,
light_intensity_ambient=0.8,
)
self.renderer = neurenderer
self.model = model
self.mano_layer = manolayer.ManoLayer(
joint_rot_mode="axisang",
use_pca=False,
mano_root="assets/mano",
center_idx=None,
flat_hand_mean=True,
)
closed_faces, hand_ignore_faces = manoutils.get_closed_faces()
self.hand_ignore_faces = hand_ignore_faces
self.mano_layer.register_buffer("th_faces", closed_faces)
self.fill_back = fill_back
def __init__(self, device=torch.device('cpu'), _mano_root='mano/models'):
args = {
'flat_hand_mean': True,
'root_rot_mode': 'axisang',
'ncomps': 45,
'mano_root': _mano_root,
'no_pca': True,
'joint_rot_mode': 'axisang',
'side': 'right'
}
self.mano = manolayer.ManoLayer(
flat_hand_mean=args['flat_hand_mean'],
side=args['side'],
mano_root=args['mano_root'],
ncomps=args['ncomps'],
use_pca=not args['no_pca'],
root_rot_mode=args['root_rot_mode'],
joint_rot_mode=args['joint_rot_mode']).to(device)
self.device = device
module.load_state_dict(model_state)
print('load model finished')
shape_model = shape_net.ShapeNet()
shape_net.load_checkpoint(
shape_model, os.path.join('checkpoints', 'ckp_siknet_synth_41.pth.tar'))
for params in shape_model.parameters():
params.requires_grad = False
pose, shape = func.initiate("zero")
pre_useful_bone_len = np.zeros((1, 15))
pose0 = torch.eye(3).repeat(1, 16, 1, 1)
mano = manolayer.ManoLayer(flat_hand_mean=True,
side="right",
mano_root=_mano_root,
use_pca=False,
root_rot_mode='rotmat',
joint_rot_mode='rotmat')
print('start opencv')
point_fliter = smoother.OneEuroFilter(4.0, 0.0)
mesh_fliter = smoother.OneEuroFilter(4.0, 0.0)
shape_fliter = smoother.OneEuroFilter(4.0, 0.0)
cap = cv2.VideoCapture(0)
print('opencv finished')
flag = 1
plt.ion()
f = plt.figure()
fliter_ax = f.add_subplot(111, projection='3d')
plt.show()
view_mat = np.array([[1.0, 0.0, 0.0], [0.0, -1.0, 0], [0.0, 0, -1.0]])
def __init__(
self,
split,
split_mode="objects",
root="data",
fraction=1,
joint_nb=21,
mini_factor=0,
full_image=True,
mode="full",
full_sequences=False,
use_cache=True,
like_v1=True,
): # Put back one to compare with arxiv citation !!
super().__init__()
self.split_mode = split_mode
if split_mode == "paper" and mode == "weak":
raise ValueError(
f"split mode {split_mode} incompatible will mode {mode} != full"
)
self.name = "ho3dv2"
self.full_sequences = full_sequences
self.fraction = fraction
self.has_dist2strong = True
self.image_size = [640, 480]
cache_folder = os.path.join("data", "cache")
os.makedirs(cache_folder, exist_ok=True)
if like_v1:
prefix = "likev1"
else:
prefix = ""
cache_path = os.path.join(
cache_folder,
f"{self.name}_{prefix}_{split_mode}_{split}_{mode}_{fraction}.pkl")
self.root = os.path.join(root, self.name)
self.joint_nb = joint_nb
self.reorder_idxs = np.array([
0, 13, 14, 15, 16, 1, 2, 3, 17, 4, 5, 6, 18, 10, 11, 12, 19, 7, 8,
9, 20
])
self.mini_factor = mini_factor
self.full_image = full_image
if fraction != 1:
assert mini_factor in [
1, 0, None
], f"fraction and minifactor not simulatneously supported"
self.all_queries = [
BaseQueries.IMAGE,
BaseQueries.JOINTS2D,
BaseQueries.JOINTS3D,
BaseQueries.OBJVERTS2D,
BaseQueries.OBJVIS2D,
BaseQueries.OBJVERTS3D,
BaseQueries.OBJCORNERS2D,
BaseQueries.OBJCORNERS3D,
BaseQueries.OBJCANCORNERS,
# BaseQueries.OBJCANROTCORNERS,
# BaseQueries.OBJCANROTVERTS,
BaseQueries.OBJFACES,
BaseQueries.HANDVERTS2D,
BaseQueries.HANDVIS2D,
BaseQueries.HANDVERTS3D,
BaseQueries.OBJCANVERTS,
BaseQueries.SIDE,
BaseQueries.CAMINTR,
BaseQueries.JOINTVIS,
]
trans_queries = get_trans_queries(self.all_queries)
self.all_queries.extend(trans_queries)
# Fix dataset split
valid_splits = ["train", "trainval", "val", "test"]
assert split in valid_splits, "{} not in {}".format(
split, valid_splits)
self.split = split
self.cam_intr = np.array([[617.343, 0.0,
312.42], [0.0, 617.343, 241.42],
[0.0, 0.0, 1.0]]).astype(np.float32)
self.cam_extr = np.array([[1, 0, 0, 0], [0, -1, 0, 0], [0, 0, -1, 0],
[0, 0, 0, 1]])
self.layer = manolayer.ManoLayer(
joint_rot_mode="axisang",
use_pca=False,
mano_root="assets/mano",
center_idx=None,
flat_hand_mean=True,
)
if self.split_mode == "objects":
if self.split == "train":
if like_v1:
seqs = {
"SM5", "MC6", "MC4", "SM3", "SM4", "SS3", "SS2", "SM2",
"SS1", "MC5", "MC1"
}
else:
seqs = {
"ABF11",
"ABF12",
"ABF13",
"ABF14",
"BB10",
"BB12",
"BB13",
"BB14",
"GPMF10",
"GPMF11",
"GPMF13",
"GPMF14",
"GSF10",
"GSF11",
"GSF12",
"GSF14",
"MC2",
"MC4",
"MC5",
"MC6",
"MDF10",
"MDF11",
"MDF12",
"MDF13",
"SB10",
"SB12",
"ShSu12",
"ShSu13",
"ShSu14",
"SiBF10",
"SiBF12",
"SiBF13",
"SiBF14",
"SM2",
"SM4",
"SM5",
"SMu40",
"SMu41",
"SS1",
"SS3",
"SMu42",
}
subfolder = "train"
elif self.split == "trainval":
seqs = {
"ABF12",
"ABF13",
"ABF14",
"BB10",
"BB13",
"BB14",
"GPMF10",
"GPMF11",
"GPMF14",
"GSF10",
"GSF11",
"GSF12",
"MC2",
"MC4",
"MC5",
"MC6",
"MDF10",
"MDF11",
"MDF12",
"MDF13",
"SB10",
"SB12",
"ShSu12",
"ShSu13",
"ShSu14",
"SiBF10",
"SiBF12",
"SiBF13",
"SiBF14",
"SM2",
"SM4",
"SM5",
"SMu40",
"SMu41",
"SS1",
"SS3",
"SMu42",
}
subfolder = "train"
elif self.split == "val":
seqs = {"ABF11", "BB12", "GPMF13", "GSF14"}
subfolder = "train"
elif self.split == "test":
if like_v1:
seqs = {"MC2"}
else:
seqs = {
"ABF10",
"MC1",
"MDF14",
"BB11",
"GPMF12",
"GSF13",
"SB14",
"ShSu10",
"SM3",
"SMu1",
"SiBF11",
"SS2",
}
subfolder = "train"
warnings.warn(f"Using seqs {seqs} for evaluation")
print(f"Using seqs {seqs} for evaluation")
self.mode = mode
if os.path.exists(cache_path) and use_cache:
with open(cache_path, "rb") as p_f:
annotations = pickle.load(p_f)
else:
if self.split_mode == "objects":
all_seqs, seq_map, closeseq_map, strongs, weaks, idxs = ho3dv2utils.get_objectsplit_infos(
seqs, self.root, subfolder, fraction=fraction)
elif self.split_mode == "paper":
all_seqs, seq_map, closeseq_map, strongs, weaks, idxs = ho3dv2utils.get_papersplit_infos(
split, self.root)
annotations = {
"all_sequences": all_seqs,
"closeseq_map": closeseq_map,
"strongs": strongs,
"weaks": weaks,
"idxs": idxs,
"seq_map": seq_map,
}
with open(cache_path, "wb") as p_f:
pickle.dump(annotations, p_f)
self.all_sequences = annotations["all_sequences"]
self.closeseq_map = annotations["closeseq_map"]
self.strongs = annotations["strongs"]
self.weaks = annotations["weaks"]
self.idxs = annotations["idxs"]
self.seq_map = annotations["seq_map"]
self.fulls = self.strongs + self.weaks
assert len(self.fulls) == len(self.idxs)
assert not len((set(self.weaks) | set(self.strongs)) - set(self.fulls))
if len(self.weaks):
weak_distances = [
abs(key - self.closeseq_map[key]["closest"])
for key in self.weaks
]
assert min(weak_distances) == 1
if len(self.strongs):
strong_distances = [
abs(key - self.closeseq_map[key]["closest"])
for key in self.strongs
]
assert min(strong_distances) == 0
assert max(strong_distances) == 0
self.obj_meshes = ho3dfullutils.load_objects(
os.path.join(self.root, "modelsprocess"))
# Get paired links as neighboured joints
self.links = [
(0, 1, 2, 3, 4),
(0, 5, 6, 7, 8),
(0, 9, 10, 11, 12),
(0, 13, 14, 15, 16),
(0, 17, 18, 19, 20),
]
def __init__(
self,
root="data",
split="train",
split_type="actions",
joint_nb=21,
use_cache=True,
mini_factor=None,
use_objects=True,
filter_object=None,
center_hand=False,
fraction=1,
mode="strong",
):
"""
Args:
center_hand: to center on hand, else full image
fraction: fraction of data with full annotation
mode: [strong|full|weak]
split_type: [actions|subjects]
"""
super().__init__()
self.center_hand = center_hand
self.use_objects = use_objects
self.fraction = fraction
self.mode = mode
self.has_dist2strong = True
if fraction != 1:
assert mini_factor in [1, 0, None], f"fraction and minifactor not simulatneously supported"
# Set cache path
self.use_cache = use_cache
self.cache_folder = os.path.join("data", "cache", "fhb")
os.makedirs(self.cache_folder, exist_ok=True)
# Get queries
self.all_queries = [
BaseQueries.IMAGE,
BaseQueries.JOINTS2D,
BaseQueries.JOINTS3D,
BaseQueries.OBJVERTS2D,
# BaseQueries.OBJVIS2D,
BaseQueries.OBJVERTS3D,
BaseQueries.HANDVERTS3D,
BaseQueries.HANDVERTS2D,
# BaseQueries.OBJCANROTVERTS,
BaseQueries.OBJFACES,
BaseQueries.OBJCANVERTS,
BaseQueries.SIDE,
BaseQueries.CAMINTR,
BaseQueries.JOINTVIS,
]
trans_queries = get_trans_queries(self.all_queries)
self.all_queries.extend(trans_queries)
# Get camera info
self.cam_extr = np.array(
[
[0.999988496304, -0.00468848412856, 0.000982563360594, 25.7],
[0.00469115935266, 0.999985218048, -0.00273845880292, 1.22],
[-0.000969709653873, 0.00274303671904, 0.99999576807, 3.902],
[0, 0, 0, 1],
]
)
self.cam_intr = np.array([[1395.749023, 0, 935.732544], [0, 1395.749268, 540.681030], [0, 0, 1]])
self.reorder_idx = np.array(
[0, 1, 6, 7, 8, 2, 9, 10, 11, 3, 12, 13, 14, 4, 15, 16, 17, 5, 18, 19, 20]
)
self.name = "fhb"
self.joint_nb = joint_nb
self.mini_factor = mini_factor
split_opts = ["actions", "objects", "subjects"]
self.subjects = ["Subject_1", "Subject_2", "Subject_3", "Subject_4", "Subject_5", "Subject_6"]
if split_type not in split_opts:
raise ValueError(
"Split for dataset {} should be in {}, got {}".format(self.name, split_opts, split_type)
)
self.split_type = split_type
self.root = os.path.join(root, "fhbhands")
self.info_root = os.path.join(self.root, "Subjects_info")
self.info_split = os.path.join(self.root, "data_split_action_recognition.txt")
self.layer = manolayer.ManoLayer(
joint_rot_mode="axisang",
use_pca=False,
mano_root="assets/mano",
center_idx=None,
flat_hand_mean=True,
)
self.reduce_res = True
small_rgb = os.path.join(self.root, "Video_files_480")
if os.path.exists(small_rgb) and self.reduce_res:
print("Using reduced images for faster computations!")
self.rgb_root = small_rgb
self.reduce_factor = 1 / 4
else:
self.rgb_root = os.path.join(self.root, "Video_files")
self.reduce_factor = 1
self.skeleton_root = os.path.join(self.root, "Hand_pose_annotation_v1")
self.filter_object = filter_object
# Get file prefixes for images and annotations
self.split = split
self.rgb_template = "color_{:04d}.jpeg"
# Joints are numbered from tip to base, we want opposite
self.idxs = [0, 4, 3, 2, 1, 8, 7, 6, 5, 12, 11, 10, 9, 16, 15, 14, 13, 20, 19]
self.load_dataset()
# Infor for rendering
self.cam_intr[:2] = self.cam_intr[:2] * self.reduce_factor
self.image_size = [int(1920 * self.reduce_factor), int(1080 * self.reduce_factor)]
print("Got {} samples for split {}".format(len(self.image_names), self.split))
# get paired links as neighboured joints
self.links = [
(0, 1, 2, 3, 4),
(0, 5, 6, 7, 8),
(0, 9, 10, 11, 12),
(0, 13, 14, 15, 16),
(0, 17, 18, 19, 20),
]
def __init__(
self,
root="data/real_colibri_v1",
split="train",
joint_nb=21,
use_cache=True
):
"""
Args:
# TODO
"""
super().__init__()
self.name = "real_colibri_v1"
self.root = root
self.split = split
self.joint_nb = joint_nb
self.has_dist2strong = False
self.use_cache = use_cache
self.rgb_root = os.path.join(self.root, "rgb")
self.segm_root = os.path.join(self.root, "segm")
self.meta_root = os.path.join(self.root, "meta")
self.cache_folder = os.path.join("data", "cache", "real_colibri_v1")
os.makedirs(self.cache_folder, exist_ok=True)
# Get queries
self.all_queries = [
BaseQueries.IMAGE,
BaseQueries.SIDE,
BaseQueries.CAMINTR,
BaseQueries.JOINTS2D,
BaseQueries.JOINTS3D,
BaseQueries.JOINTVIS,
BaseQueries.HANDVERTS2D,
BaseQueries.HANDVERTS3D,
BaseQueries.OBJCORNERS3D,
BaseQueries.OBJFPS2D,
BaseQueries.OBJFPS3D,
BaseQueries.OBJFPSVECFIELD,
BaseQueries.OBJVERTS2D,
BaseQueries.OBJVERTS3D,
BaseQueries.OBJCANVERTS,
BaseQueries.OBJMASK,
BaseQueries.OBJFACES,
BaseQueries.OBJPOSE,
]
trans_queries = get_trans_queries(self.all_queries)
self.all_queries.extend(trans_queries)
self.layer = manolayer.ManoLayer(
joint_rot_mode="axisang",
use_pca=False,
mano_root="assets/mano",
center_idx=None,
flat_hand_mean=True,
)
self.obj_meshes = {}
self.obj_fps_3d = {}
self.load_dataset()
# Infor for rendering
self.image_size = np.array([256, 256])
print("Got {} samples for split {}".format(len(self.samples), self.split))
# get paired links as neighboured joints
self.links = [
(0, 1, 2, 3, 4),
(0, 5, 6, 7, 8),
(0, 9, 10, 11, 12),
(0, 13, 14, 15, 16),
(0, 17, 18, 19, 20),
]