def __init_platform__(self, module_path, window_size, visible=False):
view_mat = axangle2mat([1, 0, 0],
1 * np.pi) # align different coordinate systems
hand_mesh = HandMesh(module_path)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
#transfering meter to mm
mano_mesh = np.matmul(view_mat, hand_mesh.verts.T).T * 1000
mesh.vertices = \
o3d.utility.Vector3dVector(mano_mesh)
mesh.compute_vertex_normals()
viewer = o3d.visualization.Visualizer()
viewer.create_window(width=window_size + 1,
height=window_size + 1,
window_name='Minimal Hand - output',
visible=visible)
viewer.add_geometry(mesh)
self.hand_mesh = hand_mesh
self.mesh = mesh
self.view_mat = view_mat
return viewer, hand_mesh, mesh, view_mat
def live_application(capture, output_dirpath):
"""
Launch an application that reads from a webcam and estimates hand pose at
real-time.
The captured hand must be the right hand, but will be flipped internally
and rendered.
Parameters
----------
capture : object
An object from `capture.py` to read capture stream from.
"""
############ output visualization ############
view_mat = axangle2mat([1, 0, 0],
np.pi) # align different coordinate systems
window_size = 1080
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
viewer = o3d.visualization.Visualizer()
viewer.create_window(width=window_size + 1,
height=window_size + 1,
window_name='Minimal Hand - output')
viewer.add_geometry(mesh)
view_control = viewer.get_view_control()
cam_params = view_control.convert_to_pinhole_camera_parameters()
extrinsic = cam_params.extrinsic.copy()
extrinsic[0:3, 3] = 0
cam_params.extrinsic = extrinsic
cam_params.intrinsic.set_intrinsics(window_size + 1, window_size + 1,
config.CAM_FX, config.CAM_FY,
window_size // 2, window_size // 2)
view_control.convert_from_pinhole_camera_parameters(cam_params)
view_control.set_constant_z_far(1000)
render_option = viewer.get_render_option()
render_option.load_from_json('./render_option.json')
viewer.update_renderer()
############ input visualization ############
pygame.init()
display = pygame.display.set_mode((window_size, window_size))
pygame.display.set_caption('Minimal Hand - input')
############ misc ############
mesh_smoother = OneEuroFilter(4.0, 0.0)
clock = pygame.time.Clock()
model = ModelPipeline()
frame_index = 0
mano_params = []
while True:
frame_large = capture.read()
if frame_large is None:
print(f'none frame {frame_index}')
# if frame_index == 0:
# continue
break
# if frame_large.shape[0] > frame_large.shape[1]:
# margin = int((frame_large.shape[0] - frame_large.shape[1]) / 2)
# frame_large = frame_large[margin:-margin]
# else:
# margin = int((frame_large.shape[1] - frame_large.shape[0]) / 2)
# frame_large = frame_large[:, margin:-margin]
frame = imresize(frame_large, (128, 128))
_, theta_mpii = model.process(frame)
theta_mano = mpii_to_mano(theta_mpii)
mano_params.append(deepcopy(theta_mano.tolist()))
osp.join(output_dirpath, "%06d.jpg" % frame_index)
v = hand_mesh.set_abs_quat(theta_mano)
v *= 2 # for better visualization
v = v * 1000 + np.array([0, 0, 400])
v = mesh_smoother.process(v)
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = o3d.utility.Vector3dVector(np.matmul(view_mat, v.T).T)
mesh.paint_uniform_color(config.HAND_COLOR)
mesh.compute_triangle_normals()
mesh.compute_vertex_normals()
viewer.update_geometry()
viewer.poll_events()
viewer.capture_screen_image(
osp.join(output_dirpath, "%06d.jpg" % frame_index))
display.blit(
pygame.surfarray.make_surface(
np.transpose(imresize(frame_large, (window_size, window_size)),
(1, 0, 2))), (0, 0))
pygame.display.update()
# if keyboard.is_pressed("esc"):
# break
clock.tick(30)
frame_index += 1
with open(osp.join(output_dirpath, f'{capture.side}.pickle'), 'w') as f:
json.dump(mano_params, f)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--pid', required=True)
args = parser.parse_args()
print(f'{"=" * 20} processing {args.pid} {"=" * 20}')
kinect_unpacked_dp = "/home/renat/Desktop/offline_processor2/align"
kinect_unpacked_dp_bt = "/home/renat/Desktop/offline_processor2/align"
output_dirpath = "/home/renat/Desktop/minimal-hand"
sn = '000589692912'
apud = AzurePeopleUnpackedDataset(osp.join(kinect_unpacked_dp, args.pid),
serial_numbers=[sn])
apud.dataset_dirpath = osp.join(kinect_unpacked_dp_bt, args.pid)
apud.read_bt()
apud.dataset_dirpath = osp.join(kinect_unpacked_dp, args.pid)
apud.parse_cam_params()
color_dirpath = osp.join(apud.dataset_dirpath, sn, 'color_undistorted')
fns_dict = dict()
for fn in os.listdir(color_dirpath):
fn_no_ext = osp.splitext(fn)[0]
try:
frame_index = int(fn_no_ext)
fns_dict[frame_index] = fn
except:
pass
processor = Processor()
result = defaultdict(dict)
done = 0
view_mat = axangle2mat([1, 0, 0],
np.pi) # align different coordinate systems
window_size = 1080
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
viewer = o3d.visualization.Visualizer()
viewer.create_window(width=window_size + 1,
height=window_size + 1,
window_name='Minimal Hand - output')
viewer.add_geometry(mesh)
view_control = viewer.get_view_control()
cam_params = view_control.convert_to_pinhole_camera_parameters()
extrinsic = cam_params.extrinsic.copy()
extrinsic[0:3, 3] = 0
cam_params.extrinsic = extrinsic
cam_params.intrinsic.set_intrinsics(window_size + 1, window_size + 1,
config.CAM_FX, config.CAM_FY,
window_size // 2, window_size // 2)
view_control.convert_from_pinhole_camera_parameters(cam_params)
view_control.set_constant_z_far(1000)
render_option = viewer.get_render_option()
render_option.load_from_json('./render_option.json')
viewer.update_renderer()
############ input visualization ############
pygame.init()
display = pygame.display.set_mode((window_size, window_size))
pygame.display.set_caption('Minimal Hand - input')
############ misc ############
mesh_smoother = OneEuroFilter(4.0, 0.0)
clock = pygame.time.Clock()
start = time.time()
for frame_index in sorted(fns_dict.keys())[1000:]:
img_path = osp.join(color_dirpath, fns_dict[frame_index])
color_undistorted = io.imread(img_path)[:, :, :3]
try:
kinect_joints = apud.get_bt_joints(frame_index, sn)
except:
print(f'no joints for frame {frame_index}')
continue
kinect_joints = np.array(kinect_joints) / 1000
for hand_index, side in [
[15, 'right'],
# [8, 'left'],
]:
p = kinect_joints[hand_index]
p_cube = get_cube(p)
p_cube_proj = depth3d_to_color2d(p_cube, apud.cam_params[sn])
bbox = get_bbox(p_cube_proj)
print(bbox)
# crop_img = simple_crop(color_undistorted, bbox)
crop_img = pil_crop(color_undistorted, bbox)
print(crop_img.shape)
theta_mano, frame_large = processor.process(crop_img, side)
print(frame_large.shape)
v = hand_mesh.set_abs_quat(theta_mano)
v *= 2 # for better visualization
v = v * 1000 + np.array([0, 0, 400])
v = mesh_smoother.process(v)
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = o3d.utility.Vector3dVector(
np.matmul(view_mat, v.T).T)
mesh.paint_uniform_color(config.HAND_COLOR)
mesh.compute_triangle_normals()
mesh.compute_vertex_normals()
viewer.update_geometry()
viewer.poll_events()
# viewer.capture_screen_image(osp.join(output_dirpath, "%06d.jpg" % i))
display.blit(
pygame.surfarray.make_surface(
np.transpose(
imresize(crop_img, (window_size, window_size)),
(1, 0, 2))), (0, 0))
pygame.display.update()
# if keyboard.is_pressed("esc"):
# break
clock.tick(30)
result[frame_index][side] = theta_mano
done += 1
if done % 100 == 0:
print(f'done {done}, {time.time() - start}')
with open(osp.join(output_dirpath, f'{args.pid}.pickle'), 'wb') as f:
pickle.dump(result, f)
def live_application(capture):
"""
Launch an application that reads from a webcam and estimates hand pose at
real-time.
The captured hand must be the right hand, but will be flipped internally
and rendered.
Parameters
----------
capture : object
An object from `capture.py` to read capture stream from.
"""
############ output visualization ############
view_mat = axangle2mat([1, 0, 0],
np.pi) # align different coordinate systems
window_size = 1080
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
hand_mri = HandMRI()
tnerf = TinyNerf()
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
bone_pcl = o3d.geometry.PointCloud()
mri_pcl = o3d.geometry.PointCloud()
# viewer = o3d.visualization.Visualizer()
# viewer.create_window(
# width=window_size + 1, height=window_size + 1,
# window_name='Minimal Hand - output'
# )
# viewer.add_geometry(mesh)
#
# view_control = viewer.get_view_control()
# cam_params = view_control.convert_to_pinhole_camera_parameters()
# extrinsic = cam_params.extrinsic.copy()
# extrinsic[0:3, 3] = 0
# cam_params.extrinsic = extrinsic
# cam_params.intrinsic.set_intrinsics(
# window_size + 1, window_size + 1, config.CAM_FX, config.CAM_FY,
# window_size // 2, window_size // 2
# )
# view_control.convert_from_pinhole_camera_parameters(cam_params)
# view_control.set_constant_z_far(1000)
# render_option = viewer.get_render_option()
# render_option.load_from_json('./render_option.json')
# viewer.update_renderer()
############ input visualization ############
# pygame.init()
# display = pygame.display.set_mode((window_size, window_size))
# pygame.display.set_caption('Minimal Hand - input')
############ misc ############
mesh_smoother = OneEuroFilter(4.0, 0.0)
clock = pygame.time.Clock()
# model = ModelPipeline()
frameid = 0
while True:
frame_large = capture.read()
if frame_large is None:
break
frameid += 1
if frameid % 2 != 0:
continue
print(frameid)
if os.path.exists(
"../video/ev_20200708_151823/{:06d}_mri_vol.nii.gz".format(
frameid)):
continue
# if frameid != 68:
# continue
if frame_large.shape[0] > frame_large.shape[1]:
margin = int((frame_large.shape[0] - frame_large.shape[1]) / 2)
frame_large = frame_large[margin:-margin]
else:
margin = int((frame_large.shape[1] - frame_large.shape[0]) / 2)
frame_large = frame_large[:, margin:-margin]
frame_large = np.flip(frame_large, axis=1).copy()
frame = imresize(frame_large, (128, 128))
# _, theta_mpii = model.process(frame)
# theta_mano = mpii_to_mano(theta_mpii)
# np.save("../video/ev_20200708_151823/{:06d}_abs_quat.npy".format(frameid), theta_mano)
theta_mano = np.load(
"../video/ev_20200708_151823/{:06d}_abs_quat.npy".format(frameid))
v, j = hand_mesh.set_abs_quat(theta_mano)
# generate mri
print("=======generate mri")
pts_vol = hand_mri.mano_to_vol(v, spacing=mri_spacing)
vol_size = pts_vol.shape[:3]
# np.save("../video/ev_20200708_151823/{:06d}_mri_vol_q_size.npy".format(frameid), vol_size)
# transfer weights and back to mano canonical
print("=======transfer weights")
pts_vol_flatten = pts_vol.reshape(-1, 3)
pts_weights = hand_mri.transfer_weights(v, hand_mesh.weights,
pts_vol_flatten)
print("========back to mano canonical")
pts_vol_mano_c, j_mano_c = hand_mesh.reverse_abs_quat(
pts_vol_flatten, j, pts_weights, theta_mano)
# v_c, v_j_c = hand_mesh.reverse_abs_quat(v, j, hand_mesh.weights, theta_mano)
# np.savetxt("../video/ev_20200708_151823/{:06d}_mri_vol_manoc.xyz".format(frameid), v_c)
# continue
# warp to mri canonical
print("=======warp to mri canonical")
mri_v = hand_mri.warp_pts(pts_vol_mano_c, pts_weights)
# np.save("../video/ev_20200708_151823/{:06d}_mri_vol_q.npy".format(frameid), mri_v)
# mri_v_hand = hand_mri.warp_pts(v_c, hand_mesh.weights)
# np.savetxt("../video/ev_20200708_151823/{:06d}_mri_vol_mano.xyz".format(frameid), mri_v_hand)
# continue
# run tiny nerf
print("=======run nerf")
inside_mask = np.ones(mri_v.shape[0]).astype(np.int)
inside_mask = inside_mask > 0
mri, label, mri_nii, label_nii = tnerf.render_pts(mri_v,
inside_mask,
vol_size,
spacing=mri_spacing)
sitk.WriteImage(
mri_nii,
"../video/ev_20200708_151823/{:06d}_mri_vol.nii.gz".format(
frameid))
sitk.WriteImage(
mri_nii,
"/data/new_disk/liyuwei/nnUNet/Hand_MRI_segdata/tracking_test/{:06d}_mri_vol.nii.gz"
.format(frameid))
sitk.WriteImage(
label_nii,
"../video/ev_20200708_151823/{:06d}_mri_vol_label.nii.gz".format(
frameid))
mri_color = np.tile(mri.reshape(-1, 1), 3)
label_color = np.zeros_like(mri_color)
label = label.reshape(-1)
label_color[label == 1] = [1, 0, 0]
label_color[label == 2] = [0, 1, 0]
label_mask = label > 0
label_meshes = hand_mri.labelvol2mesh(pts_vol, label.reshape(vol_size),
mri_spacing)
v *= 2 # for better visualization
v = v * 1000 + np.array([0, 0, 400])
v = mesh_smoother.process(v)
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = o3d.utility.Vector3dVector(np.matmul(view_mat, v.T).T)
mesh.paint_uniform_color(config.HAND_COLOR)
mesh.compute_triangle_normals()
mesh.compute_vertex_normals()
pts_vol_flatten *= 2
pts_vol_flatten = pts_vol_flatten * 1000 + np.array([0, 0, 400])
pts_vol_flatten = np.matmul(view_mat, pts_vol_flatten.T).T
mri_pcl.points = o3d.utility.Vector3dVector(pts_vol_flatten)
mri_pcl.colors = o3d.utility.Vector3dVector(mri_color)
bone_pcl.points = o3d.utility.Vector3dVector(
pts_vol_flatten[label_mask])
bone_pcl.colors = o3d.utility.Vector3dVector(label_color[label_mask])
# marching cube
for m in range(len(label_meshes)):
label_meshes[m].vertices *= 2
label_meshes[
m].vertices = label_meshes[m].vertices * 1000 + np.array(
[0, 0, 400])
label_meshes[m].vertices = np.matmul(view_mat,
label_meshes[m].vertices.T).T
label_meshes[m].export(
"../video/ev_20200708_151823/{:06d}_mri_label{:d}.ply".format(
frameid, m + 1))
# save mesh to file
# np.save("../video/ev_20200708_151823/{:06d}_mri_vol.npy".format(frameid), pts_vol_flatten)
o3d.io.write_triangle_mesh(
"../video/ev_20200708_151823/{:06d}.ply".format(frameid), mesh)
o3d.io.write_point_cloud(
"../video/ev_20200708_151823/{:06d}_mri.ply".format(frameid),
mri_pcl)
o3d.io.write_point_cloud(
"../video/ev_20200708_151823/{:06d}_bone.ply".format(frameid),
bone_pcl)
cv2.imwrite("../video/ev_20200708_151823/{:06d}.png".format(frameid),
frame_large)
# for some version of open3d you may need `viewer.update_geometry(mesh)`
# viewer.update_geometry()
# viewer.poll_events()
#
# display.blit(
# pygame.surfarray.make_surface(
# np.transpose(
# imresize(frame_large, (window_size, window_size)
# ), (1, 0, 2))
# ),
# (0, 0)
# )
# pygame.display.update()
# if keyboard.is_pressed("esc"):
# break
clock.tick(30)
num_hands_detect = 2
smoother1 = OneEuroFilter(4.0, 0.0)
smoother2 = OneEuroFilter(4.0, 0.0)
# cv2.namedWindow('Single-Threaded Detection', cv2.WINDOW_NORMAL)
imgi = 0
dery = [False, False]
lboxes = None
view_mat = axangle2mat([1, 0, 0], np.pi)
window_size_w = 640
window_size_h = 480
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
# viewer1 = o3d.visualization.Visualizer()
# viewer1.create_window(
# width=window_size_w + 1, height=window_size_h + 1,
# window_name='Minimal Hand - output'
# )
# viewer1.add_geometry(mesh)
# viewer2 = o3d.visualization.Visualizer()
# viewer2.create_window(
def live_application(capture):
"""
Launch an application that reads from a webcam and estimates hand pose at
real-time.
The captured hand must be the right hand, but will be flipped internally
and rendered.
Parameters
----------
capture : object
An object from `capture.py` to read capture stream from.
"""
############ output visualization ############
view_mat = axangle2mat([1, 0, 0],
np.pi) # align different coordinate systems
window_size = 1080
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
viewer = o3d.visualization.Visualizer()
viewer.create_window(width=window_size + 1,
height=window_size + 1,
window_name='Minimal Hand - output')
viewer.add_geometry(mesh)
view_control = viewer.get_view_control()
cam_params = view_control.convert_to_pinhole_camera_parameters()
extrinsic = cam_params.extrinsic.copy()
extrinsic[0:3, 3] = 0
cam_params.extrinsic = extrinsic
cam_params.intrinsic.set_intrinsics(window_size + 1, window_size + 1,
config.CAM_FX, config.CAM_FY,
window_size // 2, window_size // 2)
view_control.convert_from_pinhole_camera_parameters(cam_params)
view_control.set_constant_z_far(1000)
render_option = viewer.get_render_option()
render_option.load_from_json('./render_option.json')
viewer.update_renderer()
############ input visualization ############
pygame.init()
display = pygame.display.set_mode((window_size, window_size))
pygame.display.set_caption('Minimal Hand - input')
############ misc ############
mesh_smoother = OneEuroFilter(4.0, 0.0)
clock = pygame.time.Clock()
model = ModelPipeline()
# image_paths = glob.glob("samples_benet/casa/*")
input_video_path = "../How2Sign/utterance_level/train/rgb_front/features/hand_video/CVmyQR31Dr4_5-3-rgb_front.mp4"
output_video_path = "../How2Sign/utterance_level/train/rgb_front/features/hand_pose_video/CVmyQR31Dr4_5-3-rgb_front.mp4"
output_frames_folder = "../How2Sign/utterance_level/train/rgb_front/features/hand_pose_frames/CVmyQR31Dr4_5-3-rgb_front"
output_pose_file = "../How2Sign/utterance_level/train/rgb_front/features/hand_pose/CVmyQR31Dr4_5-3-rgb_front.pickle"
os.system("mkdir " + output_frames_folder)
cap = cv2.VideoCapture(input_video_path)
length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = cap.get(cv2.CAP_PROP_FPS)
# for inputpath in image_paths:
# for i in range(length):
writer = cv2.VideoWriter(output_video_path,
cv2.VideoWriter_fourcc(*'PIM1'), fps, (1081, 731))
pose_data = []
for i in range(length):
# frame_large = capture.read()
#inputpath = "samples_benet/hand_far.jpg"
# frame_large = cv2.imread(inputpath)
ret, frame_large = cap.read()
frame_large = cv2.cvtColor(frame_large, cv2.COLOR_BGR2RGB)
if frame_large is None:
continue
if frame_large.shape[0] > frame_large.shape[1]:
margin = int((frame_large.shape[0] - frame_large.shape[1]) / 2)
frame_large = frame_large[margin:-margin]
elif frame_large.shape[0] < frame_large.shape[1]:
print()
margin = int((frame_large.shape[1] - frame_large.shape[0]) / 2)
frame_large = frame_large[:, margin:-margin]
frame_large = np.flip(frame_large, axis=1).copy()
frame = imresize(frame_large, (128, 128))
# plt.imshow(frame)
# plt.show()
_, theta_mpii = model.process(frame)
theta_mano = mpii_to_mano(theta_mpii)
pose_data.append(theta_mano)
v = hand_mesh.set_abs_quat(theta_mano)
v *= 2 # for better visualization
v = v * 1000 + np.array([0, 0, 400])
v = mesh_smoother.process(v)
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = o3d.utility.Vector3dVector(np.matmul(view_mat, v.T).T)
mesh.paint_uniform_color(config.HAND_COLOR)
mesh.compute_triangle_normals()
mesh.compute_vertex_normals()
# for some version of open3d you may need `viewer.update_geometry(mesh)`
viewer.update_geometry()
viewer.poll_events()
write_rendered_frame(writer, viewer, output_frames_folder, i)
display.blit(
pygame.surfarray.make_surface(
np.transpose(imresize(frame_large, (window_size, window_size)),
(1, 0, 2))), (0, 0))
pygame.display.update()
# pygame.image.save(display, "kk.jpeg")
# if keyboard.is_pressed("esc"):
# break
# clock.tick(30)
with open(output_pose_file, "wb") as output:
pickle.dump(pose_data, output)
label_nii = nda2vol(label.reshape(vol_size).transpose(2, 1, 0), spacing, self.nerf_dict['target_origin'],
self.nerf_dict['target_direction'])
return mri, label, mri_nii, label_nii
mri_spacing = [0.5, 0.5, 0.5]
# global_trans = [137.6553, 232.4672, 69.1232]
if __name__ == "__main__":
# q_lists = "/data/new_disk/liyuwei/mano/tiny_nerf/test/"
q_lists = "/data/new_disk/liyuwei/video/ev_20200708_151823/"
nerf_pkl = "../mano/tiny_nerf/nerf_vol4_rest_fg_n2_muscle/"
gpu = "2"
tnerf = TinyNerf(nerf_pkl, gpu="2")
hand_mri = HandMRI()
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
for i in range(500):
fname_mriq = q_lists + "{:06d}_mri_vol_q.npy".format(i + 1)
fname_mri = q_lists + "{:06d}_mri_vol.npy".format(i + 1)
if not os.path.exists(fname_mriq):
continue
fname_size = q_lists + "{:06d}_mri_vol_q_size.npy".format(i + 1)
pts_vol_size = np.load(fname_size).astype(np.int).reshape(-1)
theta_mano = np.load(q_lists + "{:06d}_abs_quat.npy".format(i + 1))
v, j = hand_mesh.set_abs_quat(theta_mano)
pts_vol = hand_mri.mano_to_vol(v, spacing=mri_spacing)
assert np.sum(pts_vol.shape[:3] - pts_vol_size) == 0
def live_application(capture):
"""
Launch an application that reads from a webcam and estimates hand pose at
real-time.
The captured hand must be the right hand, but will be flipped internally
and rendered.
Parameters
----------
capture : object
An object from `capture.py` to read capture stream from.
"""
view_mat = axangle2mat([1, 0, 0], np.pi)
window_size_w = 640
window_size_h = 480
hand_mesh = HandMesh(config.HAND_MESH_MODEL_PATH)
mesh = o3d.geometry.TriangleMesh()
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = \
o3d.utility.Vector3dVector(np.matmul(view_mat, hand_mesh.verts.T).T * 1000)
mesh.compute_vertex_normals()
viewer = o3d.visualization.Visualizer()
viewer.create_window(
width=window_size_w + 1, height=window_size_h + 1,
window_name='Minimal Hand - output'
)
viewer.add_geometry(mesh)
view_control = viewer.get_view_control()
cam_params = view_control.convert_to_pinhole_camera_parameters()
extrinsic = cam_params.extrinsic.copy()
extrinsic[0:3, 3] = 0
cam_params.extrinsic = extrinsic
cam_params.intrinsic.set_intrinsics(
window_size_w + 1, window_size_h + 1, config.CAM_FX, config.CAM_FY,
window_size_w // 2, window_size_h // 2
)
view_control.convert_from_pinhole_camera_parameters(cam_params)
view_control.set_constant_z_far(1000)
render_option = viewer.get_render_option()
render_option.load_from_json('./render_option.json')
viewer.update_renderer()
mesh_smoother = OneEuroFilter(4.0, 0.0)
cords_smoother = OneEuroFilter(4.0, 0.0)
model = ModelPipeline()
while True:
frame_large = capture.read()
if frame_large is None:
continue
if frame_large.shape[0] > frame_large.shape[1]:
margin = int((frame_large.shape[0] - frame_large.shape[1]) / 2)
frame_large = frame_large[margin:-margin]
else:
margin = int((frame_large.shape[1] - frame_large.shape[0]) / 2)
frame_large = frame_large[:, margin:-margin]
if config.left is False:
frame_large = np.flip(frame_large, axis=1).copy()
frame = imresize(frame_large, (128, 128))
cords, theta_mpii = model.process(frame)
theta_mano = mpii_to_mano(theta_mpii)
v = hand_mesh.set_abs_quat(theta_mano)
# v *= 2 # for better visualization
v = v * 1000 + np.array([0, 0, 400])
# v = np.array([x+cords[0] for x in v])
v = mesh_smoother.process(v)
mesh.triangles = o3d.utility.Vector3iVector(hand_mesh.faces)
mesh.vertices = o3d.utility.Vector3dVector(np.matmul(view_mat, v.T).T)
mesh.paint_uniform_color(config.HAND_COLOR)
mesh.compute_triangle_normals()
mesh.compute_vertex_normals()
# for some version of open3d you may need `viewer.update_geometry(mesh)`
viewer.update_geometry()
viewer.poll_events()
cords = cords_smoother.process(cords)
cords = cords * 150 + 200
cords = np.delete(cords, 2, 1)
# v = v * 1.5 + 200
# v = np.delete(v, 2, 1)
frame_large = cv2.cvtColor(frame_large, cv2.COLOR_RGB2BGR)
# cv2.polylines(frame_large, v, False, (0, 0, 0))
for x in cords:
cv2.drawMarker(frame_large, (int(x[0]), int(x[1])), (0, 0, 0))
# cv2.line(frame_large, (int(v[x][0]), int(v[x][1])), (int(v[x+1][0]), int(v[x+1][1])), (0, 0, 0))
# meshindices = np.array(mesh.triangles)
# meshvertices = np.array(mesh.vertices) - 80
# pts2d = cv2.projectPoints(meshvertices, (0, 0, 0), (0, 0, 0), np.array([[620.744, 0., 0.], [0., 621.151, 0.], [0., 0., 1.]]), None)[0].astype(int)
# for face in meshindices:
# cv2.fillConvexPoly(frame_large, pts2d[face], (64, 64, 192))
# cv2.polylines(frame_large, pts2d[meshindices], True, (255, 255, 255))
# cv2.polylines(frame_large, v, False, (0, 0, 0))
cv2.imshow("Hand AI", frame_large)
if keyboard.is_pressed("q"):
cv2.destroyAllWindows()
break
cv2.waitKey(1)
time.sleep(0.0015)