def viz(inpath, outpath='sample/', save_fig=False): """ viz_sample is a function that visualize key points annotation of a single hand from this dataset Args: :param inpath: path to this dataset :param outpath: output path of the visualized image, if save_fig is True :param save_fig: whether to save the visualized image. Default to False :return: None """ for sub_dir in os.listdir(inpath): dir = os.path.join(inpath, sub_dir) out_dir = os.path.join(outpath, sub_dir) frame_list = os.listdir(dir) frame_list = [frame for frame in frame_list if frame.endswith('.png')] for frame in frame_list: imgpath = os.path.join(dir, frame) kp_path = dir + frame[0:4] + '_joint2D.txt' kp = read_kp(kp_path) img = cv2.imread(imgpath) plothand(img, kp) if save_fig: if not os.path.exists(out_dir): os.makedirs(out_dir, exist_ok=True) cv2.imwrite(os.path.join(outpath, frame[0:4]+'.jpg'), img) else: cv2.imshow(frame, img)
def viz(savefig=False): """ viz is a function that crop and visualize keypoint annotation on image Args: :param savefig : whether to save the visualized image. default to False Returns: :return None """ outpath = './sample/' # Input data paths folder_path = 'D:\Hand-data\CMU\hand143_panopticdb/' # Put your local path here json_path = folder_path + 'hands_v143_14817.json' with open(json_path, 'r') as fid: dat_all = json.load(fid) dat_all = dat_all['root'] # dat = dat_all[0] # Choose one element as an example for dat in dat_all[0:3]: pts = np.array(dat['joint_self'], dtype=float) invalid = pts[:, 2] != 1 imgpath = dat['img_paths'] imgpath = folder_path + imgpath print(imgpath) img = cv2.imread(imgpath) # find bounding point for each img (hand) x_min = min(pts[:, 0]) x_max = max(pts[:, 0]) y_min = min(pts[:, 1]) y_max = max(pts[:, 1]) x_min = x_min - (x_max - x_min) / 2 x_max = x_max + (x_max - x_min) / 2 y_min = y_min - (y_max - y_min) / 2 y_max = y_max + (y_max - y_min) / 2 x_min, x_max = max(0, x_min), min(img.shape[1], x_max) y_min, y_max = max(0, y_min), min(img.shape[0], y_max) hand_bbox = [x_min, x_max, y_min, y_max] # Plot annotations img_crop = img[int(y_min):int(y_max), int(x_min):int(x_max)] pts[:, 0] = pts[:, 0] - x_min pts[:, 1] = pts[:, 1] - y_min plothand(img_crop, pts) if savefig: if not os.path.isdir(outpath): os.makedirs(outpath) cv2.imwrite(outpath + dat['img_paths'][5:-4] + '.jpg', img_crop) else: cv2.imshow(dat['img_paths'][5:-4], img_crop) cv2.waitKey(0) cv2.destroyAllWindows()
def viz(outpath, savefig=False): """ viz is a function that crop and visualize keypoint annotation on image Args: :param outpath: the output path of image if savefile is True :param savefig : whether to save the visualized image. default to False Returns: :return None """ with open(os.path.join(folder_path + 'annotation.pkl'), 'rb') as f: data = pickle.load(f) with open(os.path.join(folder_path + 'camera_data.pkl'), 'rb') as f: cam = pickle.load(f) mode_data = data['training_data'] # mode_data = data['testing_data'] for i, sample in enumerate(mode_data): if i > 15: break seq_name = sample['seqName'] frame_str = sample['frame_str'] frame_path = os.path.join(folder_path, 'hdImgs/{}/{}'.format(seq_name, frame_str)) outdir_parent = os.path.join(outpath, seq_name, frame_str) if 'left_hand' in sample: # this means left hands exists, then we project 3d to 31 different camera outdir = os.path.join(outdir_parent, 'left') if savefig: if not os.path.exists(outdir): os.makedirs(outdir) for c in range(31): img_name = os.path.join(frame_path, '00_{:02d}_{}.jpg'.format(c, frame_str)) # since some camera image is missing, we need to judge whether image exists if os.path.exists(img_name) and sum(sample['left_hand']['2D'][c]['occluded']) <= 5: calib_data = cam[seq_name][c] # kp process left_hand_landmark = np.array(sample['left_hand']['landmarks']).reshape(-1, 3) left_kp_2d, camera_3d = project(left_hand_landmark, calib_data) angle = hand_angle(camera_3d) # image process img = cv2.imread(img_name) x_min = min(left_kp_2d[:, 0]) x_max = max(left_kp_2d[:, 0]) y_min = min(left_kp_2d[:, 1]) y_max = max(left_kp_2d[:, 1]) crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2))) crop_x_max = min(img.shape[1] - 1, int(x_max + ((x_max - x_min) / 2))) crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2))) crop_y_max = min(img.shape[0] - 1, int(y_max + ((y_max - y_min) / 2))) img = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max] left_kp_2d[:, 0] = left_kp_2d[:, 0] - crop_x_min left_kp_2d[:, 1] = left_kp_2d[:, 1] - crop_y_min plothand(img, left_kp_2d) if savefig: outpath_img = os.path.join(outdir, '00_{:02d}_{}.jpg'.format(c, frame_str)) cv2.imwrite(outpath_img, img) else: cv2.imshow('00_{:02d}_{}.jpg'.format(c, frame_str), img) cv2.waitKey(0) cv2.destroyAllWindows() if 'right_hand' in sample: # this means left hands exists, then we project 3d to 31 different camera outdir = os.path.join(outdir_parent, 'right') if savefig: if not os.path.exists(outdir): os.makedirs(outdir) for c in range(31): img_name = os.path.join(frame_path, '00_{:02d}_{}.jpg'.format(c, frame_str)) # since some camera image is missing, we need to judge whether image exists if os.path.exists(img_name) and sum(sample['right_hand']['2D'][c]['occluded']) <= 5: calib_data = cam[seq_name][c] # kp process right_hand_landmark = np.array(sample['right_hand']['landmarks']).reshape(-1, 3) right_kp_2d, camera_3d = project(right_hand_landmark, calib_data) angle = hand_angle(camera_3d) # image process img = cv2.imread(img_name) x_min = min(right_kp_2d[:, 0]) x_max = max(right_kp_2d[:, 0]) y_min = min(right_kp_2d[:, 1]) y_max = max(right_kp_2d[:, 1]) crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2))) crop_x_max = min(img.shape[1] - 1, int(x_max + ((x_max - x_min) / 2))) crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2))) crop_y_max = min(img.shape[0] - 1, int(y_max + ((y_max - y_min) / 2))) img = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max] right_kp_2d[:, 0] = right_kp_2d[:, 0] - crop_x_min right_kp_2d[:, 1] = right_kp_2d[:, 1] - crop_y_min plothand(img, right_kp_2d) if savefig: outpath_img = os.path.join(outdir, '00_{:02d}_{}.jpg'.format(c, frame_str)) cv2.imwrite(outpath_img, img) else: cv2.imshow('00_{:02d}_{}.jpg'.format(c, frame_str), img) cv2.waitKey(0) cv2.destroyAllWindows()
def viz_sample(camera_list, inpath, outpath, save_fig=False, put_angle=False, is_left=True): """ viz_sample is a function that visualize key points annotation of a single hand from this dataset Args: :param camera_list: camera calibration matrix :param inpath: input path of a particular frame, such as 'HUMBI/subject_1/hand/00000001' :param outpath: output path of the visualized image, if save_fig is True :param save_fig: whether to save the visualized image. default to False :param put_angle: whether to present hand angle info :param is_left: whether the hand is left hand. (1-left hand, 0-right hand, -1-unknown) Returns: :return: None """ if is_left: indir = inpath + '/image_cropped/left/' kp_path = inpath + '/reconstruction/keypoints_l.txt' else: indir = inpath + '/image_cropped/right/' kp_path = inpath + '/reconstruction/keypoints_r.txt' kp_3d = read_kp_3d(kp_path) with open(indir + 'list.txt') as file: file_content = file.read() crop_info = file_content.split('\n') for info in crop_info: if info != '': id = int(info.split(' ')[0]) img_name = 'image%07d.png' % id bbox = [float(num) for num in info.split(' ')[1:5]] scale_x = (bbox[1] - bbox[0] + 1) / 250 scale_y = (bbox[3] - bbox[2] + 1) / 250 # find corresponding camera matrix for cam in (item for item in camera_list if item['id'] == id): camera_param = cam M = camera_param['project'] C = camera_param['C'] R = camera_param['R'] K = camera_param['intrinsic'] T = -np.matmul(R, C) ex = np.hstack((R, T)) # 3*4 M2 = np.matmul(K, ex) kp_2d = project2d(kp_3d, M2) # 21*2 kp_camera = project_camera(kp_3d, ex) # 21*3 angle = hand_angle(kp_camera) kp_2d[:, 0] = (kp_2d[:, 0] - bbox[0]) / scale_x kp_2d[:, 1] = (kp_2d[:, 1] - bbox[2]) / scale_y img = cv2.imread(indir + img_name) plothand(img, kp_2d) if put_angle: cv2.putText(img, 'angle: %.2f' % angle, (15, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (0, 0, 255), 1) if save_fig: outpath_img = outpath + 'image%07d.jpg' % id cv2.imwrite(outpath_img, img) else: cv2.imshow(str(id), img) cv2.waitKey(0) cv2.destroyAllWindows()
def viz(data, ann_index, regressor, save_fig=False, db_root='D:\Hand-data\YouTube-3D-Hands/data/', outpath='./data_train/'): """ viz_sample is a function that visualize key points annotation of a single hand from this dataset Args: :param data: dataset loaded from json annotation files :param ann_index: annotation index :param regressor: regressor of mesh points :param save_fig: whether to save the visualized image. default to False :param db_root: path to the database :param outpath: output path of the visualized image, if save_fig is True :return: None """ import numpy as np from os.path import join ann, img = retrieve_sample(data, ann_index) video_id = img['name'].split('/')[1] if video_id not in unavailable_list: inpath_img = join(db_root, img['name']) if os.path.exists(inpath_img): img = cv2.imread(inpath_img) vertices = np.array(ann['vertices']) x_min = min(vertices[:, 0]) x_max = max(vertices[:, 0]) y_min = min(vertices[:, 1]) y_max = max(vertices[:, 1]) crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2))) crop_x_max = min(img.shape[1] - 1, int(x_max + ((x_max - x_min) / 2))) crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2))) crop_y_max = min(img.shape[0] - 1, int(y_max + ((y_max - y_min) / 2))) img_crop = img[crop_y_min:crop_y_max, crop_x_min:crop_x_max] # compute 2d keypoints info temp = vertices + np.array([0.0, 0.0, 0.3]) Jtr_x = np.matmul(regressor, temp[:, 0]) Jtr_y = np.matmul(regressor, temp[:, 1]) Jtr_z = np.matmul(regressor, temp[:, 2]) kp = np.vstack([Jtr_x, Jtr_y, Jtr_z]).T # 16*3 vertices[:, 0] = vertices[:, 0] - crop_x_min vertices[:, 1] = vertices[:, 1] - crop_y_min kp[:, 0] = kp[:, 0] - crop_x_min kp[:, 1] = kp[:, 1] - crop_y_min pts = get_keypoints_from_mesh_ch(vertices, kp) # 21*3 plothand(img_crop, pts) if save_fig: outpath_img = outpath + str(ann['id']) + '.jpg' cv2.imwrite(outpath_img, img_crop) else: cv2.imshow(ann['id'], img_crop) cv2.waitKey(0) cv2.destroyAllWindows() else: print('missing: ' + inpath_img)
def viz(savfig=False): """ viz is a function that crop and visualize keypoint annotation on image Args: :param savfig : whether to save the visualized image. default to False Returns: :return None """ # output data paths. Replace your own output path here outpath = './sample/' # Input data paths. Replace your own input path here # paths = ['synth1/', 'synth2/', 'synth3', 'synth4'] paths = ['manual_test/', 'manual_train/'] inpath = paths[0] files = sorted([f for f in os.listdir(inpath) if f.endswith('.json')]) for f in files[-2:-1]: with open(inpath + f, 'r') as fid: dat = json.load(fid) # Each file contains 1 hand annotation, with 21 points in # 'hand_pts' of size 21x3, following this scheme: # https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/doc/output.md#hand-output-format # The 3rd column is 1 if valid: pts = np.array(dat['hand_pts']) invalid = pts[:, 2] != 1 im = cv2.imread(inpath + f[0:-5] + '.jpg') # find bounding box for each img (hand) x_max = 0 y_max = 0 x_min = 10000 y_min = 10000 for p in range(pts.shape[0]): if pts[p, 2] != 0: if pts[p, 0] < x_min: x_min = pts[p, 0] if pts[p, 0] > x_max: x_max = pts[p, 0] if pts[p, 1] < y_min: y_min = pts[p, 1] if pts[p, 1] > y_max: y_max = pts[p, 1] crop_x_min = max(0, int(x_min - ((x_max - x_min) / 2))) crop_x_max = min(im.shape[1] - 1, int(x_max + ((x_max - x_min) / 2))) crop_y_min = max(0, int(y_min - ((y_max - y_min) / 2))) crop_y_max = min(im.shape[0] - 1, int(y_max + ((y_max - y_min) / 2))) im_crop = im[crop_y_min:crop_y_max, crop_x_min:crop_x_max] pts[:, 0] = pts[:, 0] - crop_x_min pts[:, 1] = pts[:, 1] - crop_y_min # Plot annotations plothand(im_crop, pts) if savfig: if not os.path.isdir(outpath): os.makedirs(outpath) cv2.imwrite(outpath + f[0:-5] + '.jpg', im_crop) else: cv2.imshow(f[0:-5], im_crop) cv2.waitKey(0) cv2.destroyAllWindows()
def viz(inpath, save_fig=False): """ normdat is a function that convert this dataset to standard ezxr format output Args: :param inpath: local path to the STB dataset :param save_fig: whether to save the visualized image. default to False Returns: :return: None """ root = inpath out_dir = out_path dirs = os.listdir(root) dirs = [ i for i in dirs if i.startswith('B6') and os.path.isdir(os.path.join(root, i)) ] labels_dir = os.path.join(root, 'labels') # exclude_dir = ['B1Counting', 'B1Random', 'B2Counting', 'B2Random'] # dirs = [x for x in dirs if x not in exclude_dir] for img_dir in dirs: imgs_name = os.listdir(os.path.join(root, img_dir)) imgs_name = [i for i in imgs_name if i.endswith('.png')] left_imgs = [i for i in imgs_name if 'BB_left' in i] right_imgs = [i for i in imgs_name if 'BB_right' in i] sk_color_imgs = [i for i in imgs_name if 'SK_color' in i] outpath = os.path.join(out_dir, img_dir) lr_mat_data = sio.loadmat(os.path.join( labels_dir, img_dir + '_BB.mat'))['handPara'] # 3*21*1500 sk_mat_data = sio.loadmat(os.path.join(labels_dir, img_dir + '_SK.mat'))['handPara'] left_sample = left_imgs right_sample = right_imgs sk_sample = sk_color_imgs for i in range(len(left_imgs)): if i > 1: break # left left_img_path = os.path.join(root, img_dir, left_sample[i]) ind_left = int((left_sample[i].strip('.png')).split('_')[-1]) img = cv2.imread(left_img_path) label = lr_mat_data[..., ind_left] pts = trans_uvd(label) plothand(img, pts) if save_fig: if not os.path.exists(outpath): os.mkdir(outpath) outpath_img = outpath + '/' + left_sample[i][:-4] + '.jpg' cv2.imwrite(outpath_img, img) else: cv2.imshow(left_sample[i][:-4], img) cv2.waitKey(0) cv2.destroyAllWindows() # right right_img_path = os.path.join(root, img_dir, right_sample[i]) ind_right = int((right_sample[i].strip('.png')).split('_')[-1]) img = cv2.imread(right_img_path) right_label = lr_mat_data[..., ind_right] label = right_label - np.mat(lr_trans).reshape([3, 1]) pts = trans_uvd(label) plothand(img, pts) if save_fig: if not os.path.exists(outpath): os.mkdir(outpath) outpath_img = outpath + '/' + right_sample[i][:-4] + '.jpg' cv2.imwrite(outpath_img, img) else: cv2.imshow(right_sample[i][:-4], img) cv2.waitKey(0) cv2.destroyAllWindows() #sk sk_img_path = os.path.join(root, img_dir, sk_sample[i]) ind_sk = int((sk_sample[i].strip('.png')).split('_')[-1]) img = cv2.imread(sk_img_path) # depth = cv2.imread(sk_img_path.replace('color', 'depth')) sk_label = sk_mat_data[..., ind_sk] label = np.mat(sk_rot).transpose() * ( sk_label - np.mat(sk_trans).reshape([3, 1])) pts = trans_uvd(label) plothand(img, pts) if save_fig: if not os.path.exists(outpath): os.mkdir(outpath) outpath_img = outpath + '/' + sk_sample[i][:-4] + '.jpg' cv2.imwrite(outpath_img, img) else: cv2.imshow(sk_sample[i][:-4], img) cv2.waitKey(0) cv2.destroyAllWindows()