def viz(inpath, outpath='sample/', save_fig=False):
"""
viz_sample is a function that visualize key points annotation of a single hand from this dataset
Args:
:param inpath: path to this dataset
:param outpath: output path of the visualized image, if save_fig is True
:param save_fig: whether to save the visualized image. Default to False
:return: None
"""
for sub_dir in os.listdir(inpath):
dir = os.path.join(inpath, sub_dir)
out_dir = os.path.join(outpath, sub_dir)
frame_list = os.listdir(dir)
frame_list = [frame for frame in frame_list if frame.endswith('.png')]
for frame in frame_list:
imgpath = os.path.join(dir, frame)
kp_path = dir + frame[0:4] + '_joint2D.txt'
kp = read_kp(kp_path)
img = cv2.imread(imgpath)
plothand(img, kp)
if save_fig:
if not os.path.exists(out_dir):
os.makedirs(out_dir, exist_ok=True)
cv2.imwrite(os.path.join(outpath, frame[0:4]+'.jpg'), img)
else:
cv2.imshow(frame, img)
def viz(savefig=False):
"""
viz is a function that crop and visualize keypoint annotation on image
Args:
:param savefig : whether to save the visualized image. default to False
Returns:
:return None
"""
outpath = './sample/'
# Input data paths
folder_path = 'D:\Hand-data\CMU\hand143_panopticdb/' # Put your local path here
json_path = folder_path + 'hands_v143_14817.json'
with open(json_path, 'r') as fid:
dat_all = json.load(fid)
dat_all = dat_all['root']
# dat = dat_all[0] # Choose one element as an example
for dat in dat_all[0:3]:
pts = np.array(dat['joint_self'], dtype=float)
invalid = pts[:, 2] != 1
imgpath = dat['img_paths']
imgpath = folder_path + imgpath
print(imgpath)
img = cv2.imread(imgpath)
# find bounding point for each img (hand)
x_min = min(pts[:, 0])
x_max = max(pts[:, 0])
y_min = min(pts[:, 1])
y_max = max(pts[:, 1])
x_min = x_min - (x_max - x_min) / 2
x_max = x_max + (x_max - x_min) / 2
y_min = y_min - (y_max - y_min) / 2
y_max = y_max + (y_max - y_min) / 2
x_min, x_max = max(0, x_min), min(img.shape[1], x_max)
y_min, y_max = max(0, y_min), min(img.shape[0], y_max)
hand_bbox = [x_min, x_max, y_min, y_max]
# Plot annotations
img_crop = img[int(y_min):int(y_max), int(x_min):int(x_max)]
pts[:, 0] = pts[:, 0] - x_min
pts[:, 1] = pts[:, 1] - y_min
plothand(img_crop, pts)
if savefig:
if not os.path.isdir(outpath):
os.makedirs(outpath)
cv2.imwrite(outpath + dat['img_paths'][5:-4] + '.jpg', img_crop)
else:
cv2.imshow(dat['img_paths'][5:-4], img_crop)
cv2.waitKey(0)
cv2.destroyAllWindows()
def viz(outpath, savefig=False):
"""
viz is a function that crop and visualize keypoint annotation on image
Args:
:param outpath: the output path of image if savefile is True
:param savefig : whether to save the visualized image. default to False
Returns:
:return None
"""
with open(os.path.join(folder_path + 'annotation.pkl'), 'rb') as f:
data = pickle.load(f)
with open(os.path.join(folder_path + 'camera_data.pkl'), 'rb') as f:
cam = pickle.load(f)
mode_data = data['training_data']
# mode_data = data['testing_data']
for i, sample in enumerate(mode_data):
if i > 15:
break
seq_name = sample['seqName']
frame_str = sample['frame_str']
frame_path = os.path.join(folder_path, 'hdImgs/{}/{}'.format(seq_name, frame_str))
outdir_parent = os.path.join(outpath, seq_name, frame_str)
if 'left_hand' in sample:
# this means left hands exists, then we project 3d to 31 different camera
outdir = os.path.join(outdir_parent, 'left')
if savefig:
if not os.path.exists(outdir):
os.makedirs(outdir)
for c in range(31):
img_name = os.path.join(frame_path, '00_{:02d}_{}.jpg'.format(c, frame_str))
# since some camera image is missing, we need to judge whether image exists
if os.path.exists(img_name) and sum(sample['left_hand']['2D'][c]['occluded']) <= 5:
calib_data = cam[seq_name][c]
# kp process
left_hand_landmark = np.array(sample['left_hand']['landmarks']).reshape(-1, 3)
left_kp_2d, camera_3d = project(left_hand_landmark, calib_data)
angle = hand_angle(camera_3d)
# image process
img = cv2.imread(img_name)
x_min = min(left_kp_2d[:, 0])
x_max = max(left_kp_2d[:, 0])
y_min = min(left_kp_2d[:, 1])
y_max = max(left_kp_2d[:, 1])
crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2)))
crop_x_max = min(img.shape[1] - 1, int(x_max + ((x_max - x_min) / 2)))
crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2)))
crop_y_max = min(img.shape[0] - 1, int(y_max + ((y_max - y_min) / 2)))
img = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max]
left_kp_2d[:, 0] = left_kp_2d[:, 0] - crop_x_min
left_kp_2d[:, 1] = left_kp_2d[:, 1] - crop_y_min
plothand(img, left_kp_2d)
if savefig:
outpath_img = os.path.join(outdir, '00_{:02d}_{}.jpg'.format(c, frame_str))
cv2.imwrite(outpath_img, img)
else:
cv2.imshow('00_{:02d}_{}.jpg'.format(c, frame_str), img)
cv2.waitKey(0)
cv2.destroyAllWindows()
if 'right_hand' in sample:
# this means left hands exists, then we project 3d to 31 different camera
outdir = os.path.join(outdir_parent, 'right')
if savefig:
if not os.path.exists(outdir):
os.makedirs(outdir)
for c in range(31):
img_name = os.path.join(frame_path, '00_{:02d}_{}.jpg'.format(c, frame_str))
# since some camera image is missing, we need to judge whether image exists
if os.path.exists(img_name) and sum(sample['right_hand']['2D'][c]['occluded']) <= 5:
calib_data = cam[seq_name][c]
# kp process
right_hand_landmark = np.array(sample['right_hand']['landmarks']).reshape(-1, 3)
right_kp_2d, camera_3d = project(right_hand_landmark, calib_data)
angle = hand_angle(camera_3d)
# image process
img = cv2.imread(img_name)
x_min = min(right_kp_2d[:, 0])
x_max = max(right_kp_2d[:, 0])
y_min = min(right_kp_2d[:, 1])
y_max = max(right_kp_2d[:, 1])
crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2)))
crop_x_max = min(img.shape[1] - 1, int(x_max + ((x_max - x_min) / 2)))
crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2)))
crop_y_max = min(img.shape[0] - 1, int(y_max + ((y_max - y_min) / 2)))
img = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max]
right_kp_2d[:, 0] = right_kp_2d[:, 0] - crop_x_min
right_kp_2d[:, 1] = right_kp_2d[:, 1] - crop_y_min
plothand(img, right_kp_2d)
if savefig:
outpath_img = os.path.join(outdir, '00_{:02d}_{}.jpg'.format(c, frame_str))
cv2.imwrite(outpath_img, img)
else:
cv2.imshow('00_{:02d}_{}.jpg'.format(c, frame_str), img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def viz_sample(camera_list,
inpath,
outpath,
save_fig=False,
put_angle=False,
is_left=True):
"""
viz_sample is a function that visualize key points annotation of a single hand from this dataset
Args:
:param camera_list: camera calibration matrix
:param inpath: input path of a particular frame, such as 'HUMBI/subject_1/hand/00000001'
:param outpath: output path of the visualized image, if save_fig is True
:param save_fig: whether to save the visualized image. default to False
:param put_angle: whether to present hand angle info
:param is_left: whether the hand is left hand. (1-left hand, 0-right hand, -1-unknown)
Returns:
:return: None
"""
if is_left:
indir = inpath + '/image_cropped/left/'
kp_path = inpath + '/reconstruction/keypoints_l.txt'
else:
indir = inpath + '/image_cropped/right/'
kp_path = inpath + '/reconstruction/keypoints_r.txt'
kp_3d = read_kp_3d(kp_path)
with open(indir + 'list.txt') as file:
file_content = file.read()
crop_info = file_content.split('\n')
for info in crop_info:
if info != '':
id = int(info.split(' ')[0])
img_name = 'image%07d.png' % id
bbox = [float(num) for num in info.split(' ')[1:5]]
scale_x = (bbox[1] - bbox[0] + 1) / 250
scale_y = (bbox[3] - bbox[2] + 1) / 250
# find corresponding camera matrix
for cam in (item for item in camera_list if item['id'] == id):
camera_param = cam
M = camera_param['project']
C = camera_param['C']
R = camera_param['R']
K = camera_param['intrinsic']
T = -np.matmul(R, C)
ex = np.hstack((R, T)) # 3*4
M2 = np.matmul(K, ex)
kp_2d = project2d(kp_3d, M2) # 21*2
kp_camera = project_camera(kp_3d, ex) # 21*3
angle = hand_angle(kp_camera)
kp_2d[:, 0] = (kp_2d[:, 0] - bbox[0]) / scale_x
kp_2d[:, 1] = (kp_2d[:, 1] - bbox[2]) / scale_y
img = cv2.imread(indir + img_name)
plothand(img, kp_2d)
if put_angle:
cv2.putText(img, 'angle: %.2f' % angle, (15, 40),
cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 1)
if save_fig:
outpath_img = outpath + 'image%07d.jpg' % id
cv2.imwrite(outpath_img, img)
else:
cv2.imshow(str(id), img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def viz(data,
ann_index,
regressor,
save_fig=False,
db_root='D:\Hand-data\YouTube-3D-Hands/data/',
outpath='./data_train/'):
"""
viz_sample is a function that visualize key points annotation of a single hand from this dataset
Args:
:param data: dataset loaded from json annotation files
:param ann_index: annotation index
:param regressor: regressor of mesh points
:param save_fig: whether to save the visualized image. default to False
:param db_root: path to the database
:param outpath: output path of the visualized image, if save_fig is True
:return: None
"""
import numpy as np
from os.path import join
ann, img = retrieve_sample(data, ann_index)
video_id = img['name'].split('/')[1]
if video_id not in unavailable_list:
inpath_img = join(db_root, img['name'])
if os.path.exists(inpath_img):
img = cv2.imread(inpath_img)
vertices = np.array(ann['vertices'])
x_min = min(vertices[:, 0])
x_max = max(vertices[:, 0])
y_min = min(vertices[:, 1])
y_max = max(vertices[:, 1])
crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2)))
crop_x_max = min(img.shape[1] - 1,
int(x_max + ((x_max - x_min) / 2)))
crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2)))
crop_y_max = min(img.shape[0] - 1,
int(y_max + ((y_max - y_min) / 2)))
img_crop = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max]
# compute 2d keypoints info
temp = vertices + np.array([0.0, 0.0, 0.3])
Jtr_x = np.matmul(regressor, temp[:, 0])
Jtr_y = np.matmul(regressor, temp[:, 1])
Jtr_z = np.matmul(regressor, temp[:, 2])
kp = np.vstack([Jtr_x, Jtr_y, Jtr_z]).T # 16*3
vertices[:, 0] = vertices[:, 0] - crop_x_min
vertices[:, 1] = vertices[:, 1] - crop_y_min
kp[:, 0] = kp[:, 0] - crop_x_min
kp[:, 1] = kp[:, 1] - crop_y_min
pts = get_keypoints_from_mesh_ch(vertices, kp) # 21*3
plothand(img_crop, pts)
if save_fig:
outpath_img = outpath + str(ann['id']) + '.jpg'
cv2.imwrite(outpath_img, img_crop)
else:
cv2.imshow(ann['id'], img_crop)
cv2.waitKey(0)
cv2.destroyAllWindows()
else:
print('missing: ' + inpath_img)
def viz(savfig=False):
"""
viz is a function that crop and visualize keypoint annotation on image
Args:
:param savfig : whether to save the visualized image. default to False
Returns:
:return None
"""
# output data paths. Replace your own output path here
outpath = './sample/'
# Input data paths. Replace your own input path here
# paths = ['synth1/', 'synth2/', 'synth3', 'synth4']
paths = ['manual_test/', 'manual_train/']
inpath = paths[0]
files = sorted([f for f in os.listdir(inpath) if f.endswith('.json')])
for f in files[-2:-1]:
with open(inpath + f, 'r') as fid:
dat = json.load(fid)
# Each file contains 1 hand annotation, with 21 points in
# 'hand_pts' of size 21x3, following this scheme:
# https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md#hand-output-format
# The 3rd column is 1 if valid:
pts = np.array(dat['hand_pts'])
invalid = pts[:, 2] != 1
im = cv2.imread(inpath + f[0:-5] + '.jpg')
# find bounding box for each img (hand)
x_max = 0
y_max = 0
x_min = 10000
y_min = 10000
for p in range(pts.shape[0]):
if pts[p, 2] != 0:
if pts[p, 0] < x_min:
x_min = pts[p, 0]
if pts[p, 0] > x_max:
x_max = pts[p, 0]
if pts[p, 1] < y_min:
y_min = pts[p, 1]
if pts[p, 1] > y_max:
y_max = pts[p, 1]
crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2)))
crop_x_max = min(im.shape[1] - 1, int(x_max + ((x_max - x_min) / 2)))
crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2)))
crop_y_max = min(im.shape[0] - 1, int(y_max + ((y_max - y_min) / 2)))
im_crop = im[crop_y_min:crop_y_max, crop_x_min:crop_x_max]
pts[:, 0] = pts[:, 0] - crop_x_min
pts[:, 1] = pts[:, 1] - crop_y_min
# Plot annotations
plothand(im_crop, pts)
if savfig:
if not os.path.isdir(outpath):
os.makedirs(outpath)
cv2.imwrite(outpath + f[0:-5] + '.jpg', im_crop)
else:
cv2.imshow(f[0:-5], im_crop)
cv2.waitKey(0)
cv2.destroyAllWindows()
def viz(inpath, save_fig=False):
"""
normdat is a function that convert this dataset to standard ezxr format output
Args:
:param inpath: local path to the STB dataset
:param save_fig: whether to save the visualized image. default to False
Returns:
:return: None
"""
root = inpath
out_dir = out_path
dirs = os.listdir(root)
dirs = [
i for i in dirs
if i.startswith('B6') and os.path.isdir(os.path.join(root, i))
]
labels_dir = os.path.join(root, 'labels')
# exclude_dir = ['B1Counting', 'B1Random', 'B2Counting', 'B2Random']
# dirs = [x for x in dirs if x not in exclude_dir]
for img_dir in dirs:
imgs_name = os.listdir(os.path.join(root, img_dir))
imgs_name = [i for i in imgs_name if i.endswith('.png')]
left_imgs = [i for i in imgs_name if 'BB_left' in i]
right_imgs = [i for i in imgs_name if 'BB_right' in i]
sk_color_imgs = [i for i in imgs_name if 'SK_color' in i]
outpath = os.path.join(out_dir, img_dir)
lr_mat_data = sio.loadmat(os.path.join(
labels_dir, img_dir + '_BB.mat'))['handPara'] # 3*21*1500
sk_mat_data = sio.loadmat(os.path.join(labels_dir, img_dir +
'_SK.mat'))['handPara']
left_sample = left_imgs
right_sample = right_imgs
sk_sample = sk_color_imgs
for i in range(len(left_imgs)):
if i > 1:
break
# left
left_img_path = os.path.join(root, img_dir, left_sample[i])
ind_left = int((left_sample[i].strip('.png')).split('_')[-1])
img = cv2.imread(left_img_path)
label = lr_mat_data[..., ind_left]
pts = trans_uvd(label)
plothand(img, pts)
if save_fig:
if not os.path.exists(outpath):
os.mkdir(outpath)
outpath_img = outpath + '/' + left_sample[i][:-4] + '.jpg'
cv2.imwrite(outpath_img, img)
else:
cv2.imshow(left_sample[i][:-4], img)
cv2.waitKey(0)
cv2.destroyAllWindows()
# right
right_img_path = os.path.join(root, img_dir, right_sample[i])
ind_right = int((right_sample[i].strip('.png')).split('_')[-1])
img = cv2.imread(right_img_path)
right_label = lr_mat_data[..., ind_right]
label = right_label - np.mat(lr_trans).reshape([3, 1])
pts = trans_uvd(label)
plothand(img, pts)
if save_fig:
if not os.path.exists(outpath):
os.mkdir(outpath)
outpath_img = outpath + '/' + right_sample[i][:-4] + '.jpg'
cv2.imwrite(outpath_img, img)
else:
cv2.imshow(right_sample[i][:-4], img)
cv2.waitKey(0)
cv2.destroyAllWindows()
#sk
sk_img_path = os.path.join(root, img_dir, sk_sample[i])
ind_sk = int((sk_sample[i].strip('.png')).split('_')[-1])
img = cv2.imread(sk_img_path)
# depth = cv2.imread(sk_img_path.replace('color', 'depth'))
sk_label = sk_mat_data[..., ind_sk]
label = np.mat(sk_rot).transpose() * (
sk_label - np.mat(sk_trans).reshape([3, 1]))
pts = trans_uvd(label)
plothand(img, pts)
if save_fig:
if not os.path.exists(outpath):
os.mkdir(outpath)
outpath_img = outpath + '/' + sk_sample[i][:-4] + '.jpg'
cv2.imwrite(outpath_img, img)
else:
cv2.imshow(sk_sample[i][:-4], img)
cv2.waitKey(0)
cv2.destroyAllWindows()