def h36m_train_extract(dataset_path, out_path, extract_img=False): # convert joints to global order h36m_idx = [11, 6, 7, 8, 1, 2, 3, 12, 24, 14, 15, 17, 18, 19, 25, 26, 27] global_idx = [14, 3, 4, 5, 2, 1, 0, 16, 12, 17, 18, 9, 10, 11, 8, 7, 6] # structs we use imgnames_, scales_, centers_, parts_, Ss_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] # users in validation set user_list = [1, 5, 6, 7, 8] # go over each user for user_i in user_list: user_name = 'S%d' % user_i # bbox_path = os.path.join(dataset_path, user_name, 'MySegmentsMat', 'ground_truth_bb') # path with GT 3D pose pose_path = os.path.join(dataset_path, user_name, 'Poses_D3_Positions_mono') # path with GT 2D pose pose2d_path = os.path.join(dataset_path, user_name, 'Poses_D2_Positions') # path with videos # vid_path = os.path.join(dataset_path, user_name, 'Videos') # go over all the sequences of each user seq_list = glob.glob(os.path.join(pose_path, '*.cdf')) seq_list.sort() for seq_i in seq_list: print('processing: {}'.format(seq_i)) # sequence info seq_name = seq_i.split('/')[-1] action, camera, _ = seq_name.split('.') action = action.replace(' ', '_') # irrelevant sequences if action == '_ALL': continue # 3D pose file poses_3d = pycdf.CDF(seq_i)['Pose'][0] #Nx96 #load 2D pose file seq_i_2D_pose = os.path.join(pose2d_path, os.path.basename(seq_i)) poses_2d = pycdf.CDF(seq_i_2D_pose)['Pose'][0] #Nx64 poses_2d = np.reshape(poses_2d, (-1, 32, 2)) # # bbox file # bbox_file = os.path.join(bbox_path, seq_name.replace('cdf', 'mat')) # bbox_h5py = h5py.File(bbox_file) # video file if extract_img: vid_file = os.path.join(vid_path, seq_name.replace('cdf', 'mp4')) imgs_path = os.path.join(dataset_path, 'images') vidcap = cv2.VideoCapture(vid_file) success, image = vidcap.read() # go over each frame of the sequence for frame_i in range(poses_3d.shape[0]): # read video frame if extract_img: success, image = vidcap.read() if not success: break # check if you can keep this frame if frame_i % 5 == 0: # image name imgname = '%s_%s.%s_%06d.jpg' % (user_name, action, camera, frame_i + 1) #Read img # read GT 3D pose Sall = np.reshape(poses_3d[frame_i, :], [-1, 3]) / 1000. #[32,3] S17 = Sall[h36m_idx] S17 -= S17[0] # root-centered S24 = np.zeros([24, 4]) S24[global_idx, :3] = S17 S24[global_idx, 3] = 1 # # read GT bounding box # mask = bbox_h5py[bbox_h5py['Masks'][frame_i,0]].value.T # ys, xs = np.where(mask==1) # bbox = np.array([np.min(xs), np.min(ys), np.max(xs)+1, np.max(ys)+1]) curPose_2d = poses_2d[frame_i, :] min_pt = np.min(curPose_2d, axis=0) max_pt = np.max(curPose_2d, axis=0) bbox = [min_pt[0], min_pt[1], max_pt[0], max_pt[1]] #Skeleton if False: from renderer import glViewer from renderer import viewer2D image = viewer2D.Vis_Bbox_minmaxPt( image, min_pt, max_pt) viewer2D.ImShow(image, waitTime=1) S17s_ = np.array(S17) * 100 skelVis = S17s_.ravel()[:, np.newaxis] glViewer.setSkeleton([skelVis]) glViewer.show() continue center = [(bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2] scale = 0.9 * max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 200. # store data imgnames_.append(os.path.join('images', imgname)) centers_.append(center) scales_.append(scale) Ss_.append(S24) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, 'h36m_train.npz') np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, S=Ss_)
def exportOursToSpin(cocoPose3DAll, out_path): scaleFactor = 1.2 # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_ , shapes_, skel3D_, has_smpl_ = [], [] ,[], [] # for imgSample in cocoPose3DAll: imgNum = len(cocoPose3DAll) totalSampleNum = [ len(cocoPose3DAll[imgSample]) for imgSample in cocoPose3DAll ] totalSampleNum = sum(totalSampleNum) print("\n\n### ImageNum: {}, SampleNum: {} ###".format(imgNum, totalSampleNum)) # for imgSample in cocoPose3DAll: # for key_imgId, imgSample in sorted(cocoPose3DAll.items()): for key_imgId, imgSample in sorted(cocoPose3DAll.items()): #load image imgPathFull = imgSample[0]['imgId'] fileName = os.path.basename(imgPathFull) fileName_saved = os.path.join(os.path.basename(os.path.dirname(imgPathFull)), fileName) #start from train2014 for sample in imgSample: validJointNum = np.sum(sample['pose2D_validity'][::2]) if validJointNum<4: continue if np.isnan(sample['pose3DParam']['camScale']): continue gt_skel = np.reshape(sample['pose2D_gt'],(26,-1)) #(26,2) This is from data gt_validity = np.reshape(sample['pose2D_validity'],(26,-1)) #(26,2) # Filtering ######################################################################################################## if True: requiredJoints= [0,1,2, 3,4,5, 6,7,8, 9,10,11] #In Total26 if np.min(gt_validity[requiredJoints,0])== False: continue min_pt = np.min(gt_skel[gt_validity[:,0]], axis=0) max_pt = np.max(gt_skel[gt_validity[:,0]], axis=0) # bbox= [ min_pt[0], min_pt[1], max_pt[0], max_pt[1] ] bbox= [ min_pt[0], min_pt[1], max_pt[0] - min_pt[0], max_pt[1] - min_pt[1]] center = [bbox[0] + bbox[2]/2, bbox[1] + bbox[3]/2] scale = scaleFactor*max(bbox[2], bbox[3])/200 #Save data imgnames_.append(os.path.join('images',fileName_saved)) openposes_.append(np.zeros([25,3])) #blank centers_.append(center) scales_.append(scale) has_smpl_.append(1) poses_.append(sample['pose3DParam']['pose']) #(72,) shapes_.append(sample['pose3DParam']['shape']) #(10,) #2D keypoints (total26 -> SPIN24) poseidx_spin24 = [0,1,2, 3,4,5, 6,7,8, 9,10,11, 19,20,21,22,23] poseidx_total26 = [0,1,2, 3,4,5, 6,7,8, 9,10,11, 14, 15, 16, 17, 18 ] part = np.zeros([24,3]) part[poseidx_spin24,:2] = gt_skel[poseidx_total26] #(52,) totalGT26 type part[poseidx_spin24,2] = 1*gt_validity[poseidx_total26,0] parts_.append(part) #3D joint S = np.zeros([24,4]) S[poseidx_spin24,:3] = sample['pose3D_pred'][poseidx_total26,:] * 0.001 #Scaling skeleton 3D (currently mm) -> meter S[poseidx_spin24,3] = 1 skel3D_.append(S) #Debug 2D Visualize if False: img = cv2.imread( os.path.join( '/run/media/hjoo/disk/data/mpii_human_pose_v1/images',imgnames_[-1]) ) img = viewer2D.Vis_Skeleton_2D_smplCOCO(gt_skel, pt2d_visibility = gt_validity[:,0], image =img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape(data3D_h36m, (data3D_h36m.shape[0],-1)).transpose() #(Dim, F) data3D_h36m_vis *=0.001 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton( [ data3D_h36m_vis] ,jointType='smplcoco') glViewer.show() # keypoints # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format( len(imgnames_))) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) # out_file = os.path.join(out_path, '1031-mpii3D_train_44257_all.npz') out_file = os.path.join(out_path, '1031-mpii3D_train_44257_validlimbs.npz') np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, pose=poses_, shape=shapes_, has_smpl=has_smpl_, S=skel3D_)
def LoadAllH36mdata_wSMPL_perSeq(out_path): # data_dir = '/home/hjoo/data/h36m-fetch/human36m_50fps/' # data_dir = '/home/hjoo/data/h36m-fetch/human36m_10fps/' list_skel2Ds_h36m = [] list_skel3Ds_h36m = [] list_smplPose = [] list_smplShape = [] list_openpose = [] list_imgNames = [] list_scale = [] list_center = [] # list_joint2d_spin24 = [] # list_joint3d_spin24 = [] TRAIN_SUBJECTS = [1, 5, 6, 7, 8] actionList = [ "Directions", "Discussion", "Eating", "Greeting", "Phoning", "Photo", "Posing", "Purchases", "Sitting", "SittingDown", "Smoking", "Waiting", "WalkDog", "Walking", "WalkTogether" ] subjectList = TRAIN_SUBJECTS for subId in subjectList: for action in actionList: gtPathList = sorted( glob.glob('{}/S{}/{}_*/*/gt_poses_coco_smpl.pkl'.format( h36mraw_dir, subId, action))) print("S{} - {}: {} files".format(subId, action, len(gtPathList))) for gtPath in gtPathList: with open(gtPath, 'rb') as f: gt_data = pickle.load(f, encoding='latin1') #Get Image List imgDir = os.path.dirname(gtPath) imgList_original = sorted( glob.glob(os.path.join(imgDir, '*.png'))) folderLeg = len(h36mraw_dir) + 1 imgList = [n[folderLeg:] for n in imgList_original] data2D_h36m = np.array(gt_data['2d']) #List -> (N,17,2) data3D_h36m = np.array(gt_data['3d']) #List -> (N,17,3) data3D_smplParams_pose = np.array( gt_data['smplParms']['poses_camCoord']) #List -> (N,72) data3D_smplParams_shape = np.array( gt_data['smplParms']['betas']) #(10,) N = data3D_smplParams_pose.shape[0] data3D_smplParams_shape = np.repeat( data3D_smplParams_shape[np.newaxis, :], N, axis=0) #List -> (N,10) #Scaling skeleton 3D (currently mm) -> meter data3D_h36m *= 0.001 #optional (centering) data3D_h36m = data3D_h36m - data3D_h36m[:, 0:1, :] scalelist = [] centerlist = [] bboxlist = [] #Generate BBox for i in range(len(data2D_h36m)): min_pt = np.min(data2D_h36m[i], axis=0) max_pt = np.max(data2D_h36m[i], axis=0) bbox = [min_pt[0], min_pt[1], max_pt[0], max_pt[1]] center = [(bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2] scale = scaleFactor * max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 200 bboxlist.append(bbox) centerlist.append(center) scalelist.append(scale) bDraw = True if bDraw: rawImg = cv2.imread(imgFullPath) # bbox_xyxy = conv_bboxinfo_centerscale_to_bboxXYXY(center, scale) # rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg,bbox_xyxy[:2], bbox_xyxy[2:]) croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo( rawImg, center, scale, (constants.IMG_RES, constants.IMG_RES)) #Visualize image if False: rawImg = viewer2D.Vis_Skeleton_2D_SPIN49( data['keypoint2d'][0][:, :2], pt2d_visibility=data['keypoint2d'][0][:, 2], image=rawImg) viewer2D.ImShow(rawImg, name='rawImg') viewer2D.ImShow(croppedImg, name='croppedImg') b = 0 ############### Visualize Mesh ############### camParam_scale = pred_camera_vis[b, 0] camParam_trans = pred_camera_vis[b, 1:] pred_vert_vis = ours_vertices[b].copy() pred_vert_vis = convert_smpl_to_bbox( pred_vert_vis, camParam_scale, camParam_trans) #From cropped space to original pred_vert_vis = convert_bbox_to_oriIm( pred_vert_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) #Generate multi-level BBOx bbox_list = multilvel_bbox_crop_gen(rawImg, pred_vert_vis, center, scale, bDebug=False) if False: pred_meshes = { 'ver': pred_vert_vis, 'f': smpl.faces } glViewer.setMeshData([pred_meshes], bComputeNormal=True) # ################ Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = ours_joints_3d[ b, :, :3].copy() #(N,3) pred_joints_vis = convert_smpl_to_bbox( pred_joints_vis, camParam_scale, camParam_trans) pred_joints_vis = convert_bbox_to_oriIm( pred_joints_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) glViewer.setBackgroundTexture(rawImg) glViewer.setWindowSize(rawImg.shape[1], rawImg.shape[0]) glViewer.SetOrthoCamera(True) glViewer.show(1) assert len(imgList) == len(data2D_h36m) assert len(imgList) == len(data3D_h36m) assert len(imgList) == len(data3D_smplParams_pose) assert len(imgList) == len(data3D_smplParams_shape) assert len(imgList) == len(scalelist) assert len(imgList) == len(centerlist) assert len(imgList) == len(bboxlist) list_skel2Ds_h36m.append(data2D_h36m) list_skel3Ds_h36m.append(data3D_h36m) list_smplPose.append(data3D_smplParams_pose) list_smplShape.append(data3D_smplParams_shape) list_imgNames += imgList list_scale += scalelist list_center += centerlist blankopenpose = np.zeros([N, 25, 3]) list_openpose.append(blankopenpose) #Debug 2D Visualize if True: for idx in range(data2D_h36m.shape[0]): img = cv2.imread(imgList_original[idx]) img = viewer2D.Vis_Skeleton_2D_H36m(data2D_h36m[idx], image=img) img = viewer2D.Vis_Bbox_minmaxPt( img, bboxlist[idx][:2], bboxlist[idx][2:]) viewer2D.ImShow(img) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 100 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # break #debug # break #debug list_skel2Ds_h36m = np.vstack( list_skel2Ds_h36m) #List of (N,17,2) -> (NM, 17, 2) list_skel3Ds_h36m = np.vstack( list_skel3Ds_h36m) #List of (N,17,3) -> (NM, 17, 3) list_smplPose = np.vstack(list_smplPose) #List of (N,72) -> (NM, 72) list_smplShape = np.vstack(list_smplShape) #List of (N,10) -> (NM, 10/) list_openpose = np.vstack(list_openpose) #List of (N,10) -> (NM, 10/) assert len(list_imgNames) == list_skel2Ds_h36m.shape[0] assert len(list_imgNames) == list_skel3Ds_h36m.shape[0] assert len(list_imgNames) == list_smplPose.shape[0] assert len(list_imgNames) == list_smplShape.shape[0] assert len(list_imgNames) == list_openpose.shape[0] assert len(list_imgNames) == len(list_scale) assert len(list_imgNames) == len(list_center) #Convert H36M -> SPIN24 # convert joints to global order # h36m_idx = [11, 6, 7, 8, 1, 2, 3, 12, 24, 14, 15, 17, 18, 19, 25, 26, 27] h36m_idx = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] global_idx = [14, 3, 4, 5, 2, 1, 0, 16, 12, 17, 18, 9, 10, 11, 8, 7, 6] sampleNum = len(list_imgNames) joint2d_spin24 = np.zeros((sampleNum, 24, 3)) joint2d_spin24[:, global_idx, :2] = list_skel2Ds_h36m[:, h36m_idx, :] joint2d_spin24[:, global_idx, 2] = 1 joint3d_spin24 = np.zeros((sampleNum, 24, 4)) joint3d_spin24[:, global_idx, :3] = list_skel3Ds_h36m[:, h36m_idx, :] joint3d_spin24[:, global_idx, 3] = 1 list_has_smpl = np.ones((sampleNum, ), dtype=np.uint8) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, 'h36m_training_fair_meter.npz') print("output: {}".format(out_file)) np.savez(out_file, imgname=list_imgNames, center=list_center, scale=list_scale, part=joint2d_spin24, pose=list_smplPose, shape=list_smplShape, has_smpl=list_has_smpl, S=joint3d_spin24, openpose=list_openpose)
def exportOursToSpin(out_path): scaleFactor = 1.2 imgDir = '/run/media/hjoo/disk/data/panoptic_mtc/a4_release/hdImgs/' with open( '/run/media/hjoo/disk/data/panoptic_mtc/a4_release/annotation.pkl', 'rb') as f: data = pickle.load(f) with open( '/run/media/hjoo/disk/data/panoptic_mtc/a4_release/camera_data.pkl', 'rb') as f: cam = pickle.load(f) # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] rhand_2d_list, rhand_3d_list, lhand_2d_list, lhand_3d_list = [], [], [], [] for training_testing, mode_data in data.items(): len_mode = len(mode_data) for i, sample in enumerate(tqdm(mode_data)): seqName = sample['seqName'] # print(seqName) frame_str = sample['frame_str'] frame_path = '{}/{}'.format(seqName, frame_str) assert 'body' in sample if 'right_hand' not in sample or 'left_hand' not in sample: continue body_landmark = np.array(sample['body']['landmarks']).reshape( -1, 3) #19, 3 #SMC19 order rhand_landmark = np.array( sample['right_hand']['landmarks']).reshape( -1, 3) #19, 3 #SMC19 order lhand_landmark = np.array( sample['left_hand']['landmarks']).reshape( -1, 3) #19, 3 #SMC19 order # randomly project the skeleton to a viewpoint for c in range(0, 30): if c in [1, 2, 4, 6, 7, 13, 17, 19, 28]: #Exclude top views continue calib_data = cam[seqName][c] skeleton_3d_camview = applyExtrinsic(body_landmark, calib_data) #19,3 rhand_3d_camview = applyExtrinsic(rhand_landmark, calib_data) #21,3 lhand_3d_camview = applyExtrinsic(lhand_landmark, calib_data) #21,3 skeleton_2d = project2D(body_landmark, calib_data) #19,2 rhand_2d = project2D(rhand_landmark, calib_data) #19,2 lhand_2d = project2D(lhand_landmark, calib_data) #19,2 imgName = os.path.join(frame_path, '00_{:02d}_{}.jpg'.format(c, frame_str)) # print(imgName) imgFullPath = os.path.join(imgDir, imgName) if os.path.exists(imgFullPath) == False: continue # print(imgName) #Visulaize 3D if False: img = cv2.imread(imgFullPath) # img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) # viewer2D.ImShow(img, waitTime=1) skeleton_3d_camview = skeleton_3d_camview.ravel()[:, np. newaxis] rhand_3d_camview = rhand_3d_camview.ravel()[:, np.newaxis] lhand_3d_camview = lhand_3d_camview.ravel()[:, np.newaxis] glViewer.setSkeleton([ skeleton_3d_camview, rhand_3d_camview, lhand_3d_camview ]) glViewer.setBackgroundTexture(img) glViewer.SetOrthoCamera(True) glViewer.show(0) min_pt = np.min(skeleton_2d, axis=0) min_pt[0] = max(min_pt[0], 0) min_pt[1] = max(min_pt[1], 0) max_pt = np.max(skeleton_2d, axis=0) max_pt[0] = min(max_pt[0], 1920) max_pt[1] = min(max_pt[1], 1080) # bbox= [ min_pt[0], min_pt[1], max_pt[0], max_pt[1] ] bbox = [ min_pt[0], min_pt[1], max_pt[0] - min_pt[0], max_pt[1] - min_pt[1] ] center = [bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2] scale = scaleFactor * max(bbox[2], bbox[3]) / 200 #Save data # imgnames_.append(os.path.join('train',fileName_saved)) # imgnames_.append(os.path.join('train',fileName_saved)) imgnames_.append(imgName) openposes_.append(np.zeros([25, 3])) #blank centers_.append(center) scales_.append(scale) # has_smpl_.append(1) # poses_.append(sample['pose3DParam']['pose']) #(72,) # shapes_.append(sample['pose3DParam']['shape']) #(10,) #2D keypoints (total26 -> SPIN24) poseidx_spin24 = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 12, 19, 20, 22, 21, 23 ] poseidx_smc19 = [ 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 2, 0, 1, 15, 16, 17, 18 ] part = np.zeros([24, 3]) part[poseidx_spin24, :2] = skeleton_2d[ poseidx_smc19] #(52,) totalGT26 type part[poseidx_spin24, 2] = 1 parts_.append(part) #3D joint S = np.zeros([24, 4]) S[poseidx_spin24, :3] = skeleton_3d_camview[ poseidx_smc19, :] * 0.01 #Scaling skeleton 3D (currently cm) -> meter S[poseidx_spin24, 3] = 1 skel3D_.append(S) rhand_2d_list.append(rhand_2d) rhand_3d_list.append(rhand_3d_camview * 0.01) lhand_2d_list.append(lhand_2d) lhand_3d_list.append(lhand_3d_camview * 0.01) #Add hand joints #Debug 2D Visualize if False: img = cv2.imread(imgFullPath) # img = cv2.imread( os.path.join( '/run/media/hjoo/disk/data/mpii_human_pose_v1/images',imgnames_[-1]) ) img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(rhand_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(lhand_2d, image=img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 0.001 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format(len(imgnames_))) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) # out_file = os.path.join(out_path, '1031-mpii3D_train_44257_all.npz') out_file = os.path.join(out_path, 'panopticDB.npz') print(f"Save to {out_file}") np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, S=skel3D_, rhand_3d=rhand_3d_list, rhand_2d=rhand_2d_list, lhand_3d=lhand_3d_list, lhand_2d=lhand_2d_list)
def pw3d_extract(dataset_path, out_path): pw3d_multicrop_info = {} # pw3d_crop_info[ (image_name, pid)] # E.g., ('downtown_sitOnStairs_00/image_00000.jpg', 0) # return an array with 8 level of bbox (0 face, 7 whole body) # bbox_list[0] # {'bbox_xyhw': [995.412413595738, 374.69671840965594, 98.54587305319353, 81.94162583240131], 'center': [1044.6853501223347, 415.6675313258566], 'ratio_bbox_over_face': 1.0, 'scale': 0.5912752383191612} # scale factor scaleFactor = 1.2 # structs we use imgnames_, scales_, centers_, parts_ = [], [], [], [] poses_, shapes_, genders_ = [], [], [] multilevel_bboxinfo_ = [] # get a list of .pkl files in the directory dataset_path = os.path.join(dataset_path, 'sequenceFiles', 'test') files = [ os.path.join(dataset_path, f) for f in os.listdir(dataset_path) if f.endswith('.pkl') ] # go through all the .pkl files for filename in files: with open(filename, 'rb') as f: print(f"processing: {filename}") data = pickle.load(f, encoding='latin1') smpl_pose = data['poses'] smpl_betas = data['betas'] poses2d = data['poses2d'] #(N, 3, 18) global_poses = data['cam_poses'] genders = data['genders'] valid = np.array(data['campose_valid']).astype(np.bool) # if False: #Temporal. To export all 3DPW data # for ii in range(len(valid)): # valid[ii][:] =True num_people = len(smpl_pose) num_frames = len(smpl_pose[0]) seq_name = str(data['sequence']) img_names = np.array([ 'imageFiles/' + seq_name + '/image_%s.jpg' % str(i).zfill(5) for i in range(num_frames) ]) smpl_trans = data['trans'] # get through all the people in the sequence for p_id in range(num_people): valid_pose = smpl_pose[p_id][valid[p_id]] valid_betas = np.tile(smpl_betas[p_id][:10].reshape(1, -1), (num_frames, 1)) valid_betas = valid_betas[valid[p_id]] valid_keypoints_2d = poses2d[p_id][valid[p_id]] valid_img_names = img_names[valid[p_id]] valid_global_poses = global_poses[valid[p_id]] valid_smpl_trans = smpl_trans[p_id][valid[p_id]] gender = genders[p_id] assert (gender == 'm') # consider only valid frames for valid_i in tqdm(range(valid_pose.shape[0])): part = valid_keypoints_2d[valid_i, :, :].T cur_img_name = valid_img_names[valid_i] #Disable lower bodies (openpose COCO18 index) # part[ [9,10,12,13], 2] = 0 #Upper body only by ignoring # bHeadOnly = False # if bHeadOnly: # part[ [4,7, 3,6, 8, 11], 2] = 0 target_joint = [ 10, 9, 8, 11, 12, 13, 4, 3, 2, 5, 6, 7, 0, 0 ] #From VIBE valid_joint_cnt = part[target_joint, 2] > 0.3 valid_joint_cnt[12:] = 0 if vibe_protocol and sum( valid_joint_cnt) <= 6: #Following VIBE's prop # reject_cnt+=1 continue part = part[part[:, 2] > 0, :] bbox = [ min(part[:, 0]), min( part[:, 1] ), #Tight bbox from keypoint, minX, minY, maxX, maxY max(part[:, 0]), max(part[:, 1]) ] # if bHeadOnly: #To cover head top # bbox[1] -= abs(bbox[3] - bbox[1]) center = [(bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2] scale = scaleFactor * max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 200 # transform global pose pose = valid_pose[valid_i].copy() extrinsics = valid_global_poses[valid_i][:3, :3] pose[:3] = cv2.Rodrigues( np.dot(extrinsics, cv2.Rodrigues(pose[:3])[0]))[0].T[0] imgnames_.append(valid_img_names[valid_i]) centers_.append(center) scales_.append(scale) poses_.append(pose) shapes_.append(valid_betas[valid_i]) genders_.append(gender) poseidx_spin24 = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 19, 20, 21, 22, 23 ] poseidx_openpose18 = [ 10, 9, 8, 11, 12, 13, 4, 3, 2, 5, 6, 7, 0, 15, 14, 17, 16 ] part = np.zeros([24, 3]) openpose_pt2d = valid_keypoints_2d[valid_i, :, :].T #18,3 part[poseidx_spin24, :] = openpose_pt2d[ poseidx_openpose18, :] part[poseidx_spin24, 2] = 1 * (part[poseidx_spin24, 2] > 0.3) parts_.append(part) #2D keypoints (total26 -> SPIN24) if False: imgName = os.path.join( '/run/media/hjoo/disk/data/3dpw', valid_img_names[valid_i]) rawImg = cv2.imread(imgName) # viewer2D.ImShow(rawImg) # rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox[:2], bbox[2:]) # rawImg = viewer2D.Vis_Skeleton_2D_Openpose18(openpose_pt2d[:,:2].ravel(), image= rawImg, pt2d_visibility=openpose_pt2d[:,2]>0.2) # rawImg = viewer2D.Vis_Skeleton_2D_Openpose18(openpose_pt2d[:,:2].ravel(), image= rawImg, pt2d_visibility=openpose_pt2d[:,2]>0.2) # rawImg = viewer2D.Vis_Skeleton_2D_SPIN49(openpose_pt2d[:,:2], image= rawImg, pt2d_visibility=openpose_pt2d[:,2]>0.2) # viewer2D.ImShow(rawImg) #Draw Mesh SMPL bDebugVis = False if True: imgName = os.path.join( '/run/media/hjoo/disk/data/3dpw', valid_img_names[valid_i]) rawImg = cv2.imread(imgName) bbox_list = [] # cam_ext = data['cam_poses'][valid_i] #4x4 cam_int = data['cam_intrinsics'] #3x3 import torch import glViewer valid_betas_vis = torch.from_numpy( shapes_[-1][np.newaxis, :]).float() valid_pose_vis = torch.from_numpy( valid_pose[valid_i].copy()[np.newaxis, :]).float() smpl_out = smpl_male( betas=valid_betas_vis, body_pose=valid_pose_vis[:, 3:], global_orient=valid_pose_vis[:, :3]) ours_vertices = smpl_out.vertices.detach().cpu().numpy( )[0] ours_vertices += valid_smpl_trans[valid_i] #Projection ver_3d_camview = np.matmul( valid_global_poses[valid_i, :3, :3], ours_vertices.transpose()).transpose( ) + valid_global_poses[valid_i, :3, 3] ver_2d = np.matmul( cam_int, ver_3d_camview.transpose()).transpose() ver_2d[:, 0] = ver_2d[:, 0] / ver_2d[:, 2] ver_2d[:, 1] = ver_2d[:, 1] / ver_2d[:, 2] ver_2d = ver_2d[:, :2] #Find face bbox, tight human bbox bbox_xyxy_full = np.array([ min(ver_2d[:, 0]), min(ver_2d[:, 1]), max(ver_2d[:, 0]), max(ver_2d[:, 1]) ]) # Get face bbox (min size) headVerIdx = g_smpl_facepart['head'] headVert = ver_2d[headVerIdx] minPt = [min(headVert[:, 0]), min(headVert[:, 1])] maxPt = [max(headVert[:, 0]), max(headVert[:, 1])] bbox_xyxy_small = np.array( [minPt[0], minPt[1], maxPt[0], maxPt[1]]) # rawImg= viewer2D.Vis_Pt2ds(ver_2d,rawImg) # rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy_full[:2], bbox_xyxy_full[2:], color=(255,255,0)) # rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy_small[:2], bbox_xyxy_small[2:] ,color=(255,255,0)) #Interpolation minPt_d = bbox_xyxy_full[:2] - bbox_xyxy_small[:2] maxPt_d = bbox_xyxy_full[2:] - bbox_xyxy_small[2:] for i in range(8): crop_level = i # if True: # i = crop_level cur_minPt = bbox_xyxy_small[:2] + minPt_d * i / 7.0 cur_maxPt = bbox_xyxy_small[2:] + maxPt_d * i / 7.0 bbox_xyhw = [ cur_minPt[0], cur_minPt[1], cur_maxPt[0] - cur_minPt[0], cur_maxPt[1] - cur_minPt[1] ] cur_center, cur_scale = conv_bboxinfo_bboxXYHW_to_centerscale( bbox_xyhw) cur_scale *= 1.2 #Scaling factor cur_new_bboxXYXY = conv_bboxinfo_centerscale_to_bboxXYXY( cur_center, cur_scale) if is_export_imgs and crop_level in [7]: #[1,2,4]: #export cropped image into files """Process rgb image and do augmentation.""" cropped_img = crop(rawImg, cur_center, cur_scale, [224, 224], rot=0) # viewer2D.ImShow(cropped_img,waitTime=0,name="cropped") export_img_name = seq_name + '_' + os.path.basename( imgName)[:-4] + f'_pid{p_id}.jpg' export_img_path = os.path.join( export_root, f'croplev_{crop_level}', export_img_name) cv2.imwrite(export_img_path, cropped_img) #Compute face to cur bbox ratio cur_scale / face_scale if i == 0: ratio_bbox_over_face = 1.0 else: ratio_bbox_over_face = cur_scale / bbox_list[ 0]['scale'] bbox_list.append({ "scale": cur_scale, "center": cur_center, "ratio_bbox_over_face": ratio_bbox_over_face, "bbox_xyhw": bbox_xyhw }) if bDebugVis: #Draw full size bbox print( f"{i}: {cur_scale}, {center}, {ratio_bbox_over_face}" ) # tempImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, cur_minPt, cur_maxPt ,color=(255,255,255)) if i in [1, 2, 4]: tempImg = viewer2D.Vis_Bbox_minmaxPt( rawImg, cur_new_bboxXYXY[:2], cur_new_bboxXYXY[2:], color=(0, 255, 255)) else: tempImg = viewer2D.Vis_Bbox_minmaxPt( rawImg, cur_new_bboxXYXY[:2], cur_new_bboxXYXY[2:], color=(255, 0, 0)) viewer2D.ImShow(tempImg, name="bboxGen", waitTime=0) # viewer2D.ImShow(rawImg) multilevel_bboxinfo_.append(bbox_list) key_name = (cur_img_name[11:], p_id) assert key_name not in pw3d_multicrop_info.keys() pw3d_multicrop_info[key_name] = [ dt['bbox_xyhw'] for dt in bbox_list ] # if valid_i==5: # break # store data if not os.path.isdir(out_path): os.makedirs(out_path) multicrop_out_path = os.path.join( out_path, 'pw3d_multicrop_info_sample{}_py2.pkl'.format(len(imgnames_))) with open(multicrop_out_path, 'wb') as f: pickle.dump(pw3d_multicrop_info, f, protocol=2) # pickle.dump(your_object, your_file, protocol=2) for level in range(8): scales_ = [] centers_ = [] for i in range(len(multilevel_bboxinfo_)): scales_.append(multilevel_bboxinfo_[i][level]['scale']) centers_.append(multilevel_bboxinfo_[i][level]['center']) out_file = os.path.join(out_path, f'3dpw_test_multilevel_{level}.npz') np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, pose=poses_, shape=shapes_, gender=genders_, part=parts_)
def multilvel_bbox_crop_gen(rawImg, smpl_vert, fullsize_center, fullsize_scale, bDebug=True): """ Generate bbox from smallest size(face) to full size args: fullsize_center, fullsize_scale: bbox given by original annotation (full body or maximum size) smpl_vert: """ bbox_list = [] bbox_xyxy_full = conv_bboxinfo_centerscale_to_bboxXYXY( fullsize_center, fullsize_scale) # Get face bbox (min size) headVerIdx = g_smpl_facepart['head'] headVert = smpl_vert[headVerIdx] minPt = [ min(headVert[:, 0]) + rawImg.shape[1] * 0.5, min(headVert[:, 1]) + rawImg.shape[0] * 0.5 ] maxPt = [ max(headVert[:, 0]) + rawImg.shape[1] * 0.5, max(headVert[:, 1]) + rawImg.shape[0] * 0.5 ] bbox_xyxy_small = [minPt[0], minPt[1], maxPt[0], maxPt[1]] if bDebug: #Draw full size bbox rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg.copy(), bbox_xyxy_full[:2], bbox_xyxy_full[2:], color=(255, 255, 0)) rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy_small[:2], bbox_xyxy_small[2:], color=(255, 255, 0)) viewer2D.ImShow(rawImg, name="bboxGen") #Interpolation minPt_d = bbox_xyxy_full[:2] - bbox_xyxy_small[:2] maxPt_d = bbox_xyxy_full[2:] - bbox_xyxy_small[2:] for i in range(8): cur_minPt = bbox_xyxy_small[:2] + minPt_d * i / 7.0 cur_maxPt = bbox_xyxy_small[2:] + maxPt_d * i / 7.0 bbox_xyhw = [ cur_minPt[0], cur_minPt[1], cur_maxPt[0] - cur_minPt[0], cur_maxPt[1] - cur_minPt[1] ] cur_center, cur_scale = conv_bboxinfo_bboxXYHW_to_centerscale( bbox_xyhw) #Compute face to cur bbox ratio cur_scale / face_scale if i == 0: ratio_bbox_over_face = 1.0 else: ratio_bbox_over_face = cur_scale / bbox_list[0]['scale'] bbox_list.append({ "scale": cur_scale, "center": cur_center, "ratio_bbox_over_face": ratio_bbox_over_face }) if bDebug: #Draw full size bbox print(f"{i}: {cur_scale}, {center}, {ratio_bbox_over_face}") tempImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, cur_minPt, cur_maxPt, color=(255, 255, 255)) viewer2D.ImShow(tempImg, name="bboxGen", waitTime=0) if bDebug: #Draw full size bbox for b in bbox_list: bbox_xyxy = conv_bboxinfo_centerscale_to_bboxXYXY( b['center'], b['scale']) vis = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy[:2], bbox_xyxy[2:], color=(0, 255, 255)) viewer2D.ImShow(vis, name="bboxGen", waitTime=0) return bbox_list
def LoadMocap(dbDir, imgRoot): blackList = [ '0001', '0003', '0026', '0034', '1005', '1021', '1024', '1032' ] from scipy.io import loadmat annotations_all = [] # seqPathList = sorted(glob.glob('{0}/*.json'.format(dbDir)) ) seqPathList = sorted(glob.glob('{0}/*.mat'.format(dbDir))) #Iterate for each person for i, gtPath in enumerate(seqPathList): imageInfo = [] seqName = os.path.basename(gtPath)[:4] if seqName in blackList: continue # seqName = os.path.basename(jsonData)[:-5] print('{}: {}'.format(seqName, i)) mocap = loadmat(gtPath) # N x 13 vis = mocap['visibility'] x = mocap['x'] y = mocap['y'] skel2d_tracked = np.dstack((x, y, vis)) #(N, 13, 3) subjectId = 'pennaction_{}-id{:03d}'.format( seqName, 0) #only single person in pennaction frameLeng = len(skel2d_tracked) for idx in range(frameLeng): skel2d = skel2d_tracked[idx] #13,3 if idx >= len(mocap['bbox']): print("out of range for bbox") break bbox = mocap['bbox'][idx] annot = {} #Current annotation annot['keypoints'] = skel2d #13,3 annot['subjectId'] = subjectId imgPathFull = "{0}/{1}/{2:06d}.jpg".format(imgRoot, seqName, idx + 1) annot['imgname'] = imgPathFull annot['bbox_xyxy'] = bbox if False: inputImg = cv2.imread(imgPathFull) inputImg = viewer2D.Vis_Skeleton_2D_pennaction(skel2d[:, :2], skel2d[:, 2], image=inputImg) inputImg = viewer2D.Vis_Bbox_minmaxPt(inputImg, annot['bbox_xyxy'][:2], annot['bbox_xyxy'][2:]) viewer2D.ImShow(inputImg, waitTime=0) annotations_all.append(annot) # if(np.sum(j2d_validity_coco19)==0): # print("No valid annotations") # continue # trackid = annot['track_id'] # bbox = annot['bbox'] # image_id = annot['image_id'] # annot_id = annot['id'] # keypoints = annot['keypoints'] #Img path, 2D keypoint, bbox return annotations_all
ours_vertices = ours_output.vertices.detach().cpu().numpy() rawImg = cv2.imread(imgFullPath) # croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo(rawImg, center, scale, (224, 224) ) # camParam_scale = pred_camera_vis[0,0] # camParam_trans = pred_camera_vis[0,1:] # vertices_bbox = convert_smpl_to_bbox(ours_vertices[0], camParam_scale, camParam_trans) # vertices_img = convert_bbox_to_oriIm(vertices_bbox, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) if bDraw: # rawImg = cv2.imread(imgFullPath) bbox_xyxy = conv_bboxinfo_centerscale_to_bboxXYXY( center, scale) rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy[:2], bbox_xyxy[2:]) # croppedImg = crop(rawImg, center, scale, # [constants.IMG_RES, constants.IMG_RES]) croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo( rawImg, center, scale, (constants.IMG_RES, constants.IMG_RES)) #Visualize image if False: rawImg = viewer2D.Vis_Skeleton_2D_SPIN49( data['keypoint2d'][0][:, :2], pt2d_visibility=data['keypoint2d'][0][:, 2], image=rawImg) viewer2D.ImShow(rawImg, name='rawImg') viewer2D.ImShow(croppedImg, name='croppedImg')
def visEFT_singleSubject(renderer): MAGNIFY_RATIO = 3 #onbbox only. To magnify the rendered image size bStopForEachSample = args.waitforkeys #if True, it will wait for any key pressed to move to the next sample bShowTurnTable = args.turntable inputData = args.fit_data imgDir = args.img_dir #Load SMPL model smplModelPath = args.smpl_dir + '/basicModel_neutral_lbs_10_207_0_v1.0.0.pkl' smpl = SMPL_19(smplModelPath, batch_size=1, create_transl=False) #Load EFT fitting data print(f"Loading EFT data from {inputData}") if os.path.exists(inputData): with open(inputData,'r') as f: eft_data = json.load(f) print("EFT data: ver {}".format(eft_data['ver'])) eft_data_all = eft_data['data'] else: print(f"ERROR:: Cannot find EFT data: {inputData}") assert False #Visualize each EFT Fitting output for idx, eft_data in enumerate(tqdm(eft_data_all)): #Get raw image path imgFullPath = eft_data['imageName'] # imgName = os.path.basename(imgFullPath) imgName = imgFullPath imgFullPath =os.path.join(imgDir, imgName) if os.path.exists(imgFullPath) ==False: print(f"Img path is not valid: {imgFullPath}") assert False rawImg = cv2.imread(imgFullPath) print(f'Input image: {imgFullPath}') #EFT data bbox_scale = eft_data['bbox_scale'] bbox_center = eft_data['bbox_center'] pred_camera = np.array(eft_data['parm_cam']) pred_betas = np.reshape(np.array( eft_data['parm_shape'], dtype=np.float32), (1,10) ) #(10,) pred_betas = torch.from_numpy(pred_betas) pred_pose_rotmat = np.reshape( np.array( eft_data['parm_pose'], dtype=np.float32), (1,24,3,3) ) #(24,3,3) pred_pose_rotmat = torch.from_numpy(pred_pose_rotmat) keypoint_2d_validity = eft_data['joint_validity_openpose18'] #COCO only. Annotation index if 'annotId' in eft_data.keys(): print("COCO annotId: {}".format(eft_data['annotId'])) #Get SMPL mesh and joints from SMPL parameters smpl_output = smpl(betas=pred_betas, body_pose=pred_pose_rotmat[:,1:], global_orient=pred_pose_rotmat[:,[0]], pose2rot=False) smpl_vertices = smpl_output.vertices.detach().cpu().numpy()[0] smpl_joints_3d = smpl_output.joints.detach().cpu().numpy()[0] #Crop image using cropping information croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo(rawImg, bbox_center, bbox_scale, (BBOX_IMG_RES, BBOX_IMG_RES) ) if MAGNIFY_RATIO>1: croppedImg = cv2.resize(croppedImg, (croppedImg.shape[1]*MAGNIFY_RATIO, croppedImg.shape[0]*MAGNIFY_RATIO) ) ######################## # Visualization ######################## # Visualize 2D image if True: viewer2D.ImShow(rawImg, name='rawImg', waitTime=1) #You should press any key viewer2D.ImShow(croppedImg, name='croppedImg', waitTime=1) #Convert bbox_center, bbox_scale --> bbox_xyxy bbox_xyxy = conv_bboxinfo_bboxXYXY(bbox_scale,bbox_center) img_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg.copy(),bbox_xyxy[:2], bbox_xyxy[2:]) viewer2D.ImShow(img_bbox, name='img_bbox', waitTime=1) # Visualization Mesh if True: camParam_scale = pred_camera[0] camParam_trans = pred_camera[1:] pred_vert_vis = smpl_vertices smpl_joints_3d_vis = smpl_joints_3d if args.onbbox: pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans) smpl_joints_3d_vis = convert_smpl_to_bbox(smpl_joints_3d_vis, camParam_scale, camParam_trans) renderer.setBackgroundTexture(croppedImg) renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) pred_vert_vis *=MAGNIFY_RATIO else: #Covert SMPL to BBox first pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans) smpl_joints_3d_vis = convert_smpl_to_bbox(smpl_joints_3d_vis, camParam_scale, camParam_trans) #From cropped space to original pred_vert_vis = convert_bbox_to_oriIm(pred_vert_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) smpl_joints_3d_vis = convert_bbox_to_oriIm(smpl_joints_3d_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) renderer.setBackgroundTexture(rawImg) renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0]) #In orthographic model. XY of 3D is just 2D projection smpl_joints_2d_vis = conv_3djoint_2djoint(smpl_joints_3d_vis,rawImg.shape ) # image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_smpl45(smpl_joints_2d_vis, image=rawImg.copy(),color=(0,255,255)) image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_Openpose18(smpl_joints_2d_vis, image=rawImg.copy(),color=(255,0,0)) #All 2D joint image_2dkeypoint_pred = viewer2D.Vis_Skeleton_2D_Openpose18(smpl_joints_2d_vis, pt2d_visibility=keypoint_2d_validity, image=image_2dkeypoint_pred,color=(0,255,255)) #Only valid viewer2D.ImShow(image_2dkeypoint_pred, name='keypoint_2d_pred', waitTime=1) pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} v = pred_meshes['ver'] f = pred_meshes['f'] #Visualize in the original image space renderer.set_mesh(v,f) renderer.showBackground(True) renderer.setWorldCenterBySceneCenter() renderer.setCameraViewMode("cam") #Set image size for rendering if args.onbbox: renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) else: renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0]) renderer.display() renderImg = renderer.get_screen_color_ibgr() viewer2D.ImShow(renderImg,waitTime=1) # Visualize multi-level cropped bbox if args.multi_bbox: from demo.multi_bbox_gen import multilvel_bbox_crop_gen bbox_list = multilvel_bbox_crop_gen(rawImg, pred_vert_vis, bbox_center, bbox_scale) #Visualize BBox for b_idx, b in enumerate(bbox_list): # bbox_xyxy= conv_bboxinfo_centerscale_to_bboxXYXY(b['center'], b['scale']) bbox_xyxy= b['bbox_xyxy'] if b_idx==0: img_multi_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy[:2], bbox_xyxy[2:] ,color=(0,255,0)) else: img_multi_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg, bbox_xyxy[:2], bbox_xyxy[2:] ,color=(0,255,255)) viewer2D.ImShow(img_multi_bbox, name='multi_bbox', waitTime=1) # for bbox in bbox_list: # Visualization Mesh on side view if True: renderer.showBackground(False) renderer.setWorldCenterBySceneCenter() # renderer.setCameraViewMode("side") #To show the object in side vie renderer.setCameraViewMode("free") renderer.setViewAngle(90,20) #Set image size for rendering if args.onbbox: renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) else: renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0]) renderer.display() sideImg = renderer.get_screen_color_ibgr() #Overwite on rawImg viewer2D.ImShow(sideImg,waitTime=1) sideImg = cv2.resize(sideImg, (renderImg.shape[1], renderImg.shape[0]) ) # renderImg = cv2.resize(renderImg, (sideImg.shape[1], sideImg.shape[0]) ) # Visualization Mesh on side view if True: renderer.showBackground(False) renderer.setWorldCenterBySceneCenter() # renderer.setCameraViewMode("side") #To show the object in side vie renderer.setCameraViewMode("free") renderer.setViewAngle(-60,50) #Set image size for rendering if args.onbbox: renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) else: renderer.setViewportSize(rawImg.shape[1], rawImg.shape[0]) renderer.display() sideImg_2 = renderer.get_screen_color_ibgr() #Overwite on rawImg viewer2D.ImShow(sideImg_2,waitTime=1) sideImg_2 = cv2.resize(sideImg_2, (renderImg.shape[1], renderImg.shape[0]) ) # renderImg = cv2.resize(renderImg, (sideImg.shape[1], sideImg.shape[0]) ) #Visualize camera view and side view saveImg = np.concatenate( (renderImg,sideImg), axis =1) # saveImg = np.concatenate( (croppedImg, renderImg,sideImg, sideImg_2), axis =1) if bStopForEachSample: viewer2D.ImShow(saveImg,waitTime=0) #waitTime=0 means that it will wait for any key pressed else: viewer2D.ImShow(saveImg,waitTime=1) #Render Mesh on the rotating view if bShowTurnTable: renderer.showBackground(False) renderer.setWorldCenterBySceneCenter() renderer.setCameraViewMode("free") for i in range(90): renderer.setViewAngle(i*4,0) renderer.display() sideImg = renderer.get_screen_color_ibgr() #Overwite on rawImg viewer2D.ImShow(sideImg,waitTime=1,name="turn_table") if False: #If you want to save this into files render_output_path = args.render_dir + '/turntable_{}_{:08d}.jpg'.format(os.path.basename(imgName),i) cv2.imwrite(render_output_path, sideImg) #Save the rendered image to files if True: if os.path.exists(args.render_dir) == False: os.mkdir(args.render_dir) render_output_path = args.render_dir + '/render_{}_eft{:08d}.jpg'.format(imgName[:-4],idx) print(f"Save to {render_output_path}") cv2.imwrite(render_output_path, saveImg)
def exportOursToSpin(eftDir, out_path): # scaleFactor = 1.2 # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] pose3DList = os.listdir(eftDir) # for imgSample in cocoPose3DAll: sampleNum = len(pose3DList) # totalSampleNum = [ len(cocoPose3DAll[imgSample]) for imgSample in cocoPose3DAll ] # totalSampleNum = sum(totalSampleNum) print("\n\n### SampleNum: {} ###".format(sampleNum)) maxDiff = 0 for fname in tqdm(sorted(pose3DList)): fname_path = os.path.join(eftDir, fname) pose3d = pickle.load(open(fname_path, 'rb')) #load image imgPathFull = pose3d['imageName'][0] fileName = os.path.basename(imgPathFull) fileName_saved = os.path.join( os.path.basename(os.path.dirname(imgPathFull)), fileName) #start from train2014 center = pose3d['center'][0] scale = pose3d['scale'][0] smpl_shape = pose3d['pred_shape'].ravel() smpl_pose_mat = torch.from_numpy( pose3d['pred_pose_rotmat'][0]) #24,3,3 pred_rotmat_hom = torch.cat([ smpl_pose_mat.view(-1, 3, 3), torch.tensor( [0, 0, 0], dtype=torch.float32, ).view(1, 3, 1).expand(24, -1, -1) ], dim=-1) smpl_pose = tgm.rotation_matrix_to_angle_axis( pred_rotmat_hom).contiguous().view(-1, 72) #verification if True: recon_mat = batch_rodrigues(smpl_pose.view( -1, 3)) #24,3... axis -> rotmat diff = abs(recon_mat.numpy() - pose3d['pred_pose_rotmat'][0]) #2.1234155e-07 # print(np.max(diff)) maxDiff = max(maxDiff, np.max(diff)) smpl_pose = smpl_pose.numpy().ravel() openpose2d = pose3d['keypoint2d'][0][:25] #25,3 spin2d_skel24 = pose3d['keypoint2d'][0][25:] #24,3 #Save data imgnames_.append(fileName_saved) centers_.append(center) scales_.append(scale) has_smpl_.append(1) poses_.append(smpl_pose) #(72,) shapes_.append(smpl_shape) #(10,) openposes_.append(openpose2d) #blank # print(openpose2d)/ parts_.append(spin2d_skel24) #3D joint S = np.zeros( [24, 4]) #blank for 3d. TODO: may need to add valid data for this skel3D_.append(S) #Debug 2D Visualize if False: img = cv2.imread( os.path.join('/run/media/hjoo/disk/data/coco', imgnames_[-1])) img = viewer2D.Vis_Skeleton_2D_smplCOCO( gt_skel, pt2d_visibility=gt_validity[:, 0], image=img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 100 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # keypoints # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format(len(imgnames_))) print("maxDiff in rot conv.: {}".format(maxDiff)) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, os.path.basename(eftDir) + '.npz') print(f"Save to {out_file}") np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, pose=poses_, shape=shapes_, has_smpl=has_smpl_, S=skel3D_)
def visEFT_singleSubject(renderer): bStopForEachSample = args.waitforkeys #if True, it will wait for any key pressed to move to the next sample inputData = args.fit_data imgDir = args.img_dir #Load SMPL model smplModelPath = args.smpl_dir + '/basicModel_neutral_lbs_10_207_0_v1.0.0.pkl' smpl = SMPL(smplModelPath, batch_size=1, create_transl=False) print("Loading coco annotation from:{}".format(args.cocoAnnotFile)) assert os.path.exists(args.cocoAnnotFile) cocoAnnotDic = loadCOCOAnnot(args.cocoAnnotFile) #Load EFT fitting data print(f"Loading EFT data from {inputData}") if os.path.exists(inputData): with open(inputData, 'r') as f: eft_data = json.load(f) print("EFT data: ver {}".format(eft_data['ver'])) eft_data_all = eft_data['data'] else: print(f"ERROR:: Cannot find EFT data: {inputData}") assert False #Visualize each EFT Fitting output for idx, eft_data in enumerate(eft_data_all): #Get raw image path imgFullPath = eft_data['imageName'] imgName = os.path.basename(imgFullPath) imgFullPath = os.path.join(imgDir, imgName) if os.path.exists(imgFullPath) == False: print(f"Img path is not valid: {imgFullPath}") assert False rawImg = cv2.imread(imgFullPath) print(f'Input image: {imgFullPath}') #EFT data bbox_scale = eft_data['bbox_scale'] bbox_center = eft_data['bbox_center'] pred_camera = np.array(eft_data['parm_cam']) pred_betas = np.reshape( np.array(eft_data['parm_shape'], dtype=np.float32), (1, 10)) #(10,) pred_betas = torch.from_numpy(pred_betas) pred_pose_rotmat = np.reshape( np.array(eft_data['parm_pose'], dtype=np.float32), (1, 24, 3, 3)) #(24,3,3) pred_pose_rotmat = torch.from_numpy(pred_pose_rotmat) keypoint_2d_validity = eft_data['joint_validity_openpose18'] #COCO only. Annotation index print("COCO annotId: {}".format(eft_data['annotId'])) annot = cocoAnnotDic[eft_data['annotId']] print(annot['bbox']) ######################## #Visualize COCO annotation annot_keypoint = np.reshape( np.array(annot['keypoints'], dtype=np.float32), (-1, 3)) #17,3 rawImg = viewer2D.Vis_Skeleton_2D_coco(annot_keypoint[:, :2], annot_keypoint[:, 2], image=rawImg) rawImg = viewer2D.Vis_Bbox(rawImg, annot['bbox'], color=(0, 255, 0)) #Get SMPL mesh and joints from SMPL parameters smpl_output = smpl(betas=pred_betas, body_pose=pred_pose_rotmat[:, 1:], global_orient=pred_pose_rotmat[:, [0]], pose2rot=False) smpl_vertices = smpl_output.vertices.detach().cpu().numpy()[0] smpl_joints_3d = smpl_output.joints.detach().cpu().numpy()[0] #Crop image using cropping information croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo( rawImg, bbox_center, bbox_scale, (BBOX_IMG_RES, BBOX_IMG_RES)) ######################## # Visualization of EFT ######################## # Visualize 2D image if True: viewer2D.ImShow(rawImg, name='rawImg', waitTime=1) #You should press any key viewer2D.ImShow(croppedImg, name='croppedImg', waitTime=1) #Convert bbox_center, bbox_scale --> bbox_xyxy bbox_xyxy = conv_bboxinfo_bboxXYXY(bbox_scale, bbox_center) img_bbox = viewer2D.Vis_Bbox_minmaxPt(rawImg.copy(), bbox_xyxy[:2], bbox_xyxy[2:]) viewer2D.ImShow(img_bbox, name='img_bbox', waitTime=1) # Visualization Mesh if True: camParam_scale = pred_camera[0] camParam_trans = pred_camera[1:] pred_vert_vis = smpl_vertices smpl_joints_3d_vis = smpl_joints_3d if True: #args.onbbox: pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans) smpl_joints_3d_vis = convert_smpl_to_bbox( smpl_joints_3d_vis, camParam_scale, camParam_trans) renderer.setBackgroundTexture(croppedImg) renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} v = pred_meshes['ver'] f = pred_meshes['f'] #Visualize in the original image space renderer.set_mesh(v, f) renderer.showBackground(True) renderer.setWorldCenterBySceneCenter() renderer.setCameraViewMode("cam") renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) renderer.display() renderImg = renderer.get_screen_color_ibgr() viewer2D.ImShow(renderImg, waitTime=1) # Visualization Mesh on side view if True: renderer.showBackground(False) renderer.setWorldCenterBySceneCenter() renderer.setCameraViewMode("side") renderer.setViewportSize(croppedImg.shape[1], croppedImg.shape[0]) renderer.display() sideImg = renderer.get_screen_color_ibgr() #Overwite on rawImg viewer2D.ImShow(sideImg, waitTime=1) sideImg = cv2.resize(sideImg, (renderImg.shape[1], renderImg.shape[0])) #Visualize camera view and side view saveImg = np.concatenate((renderImg, sideImg), axis=1) if bStopForEachSample: viewer2D.ImShow( saveImg, waitTime=0 ) #waitTime=0 means that it will wait for any key pressed else: viewer2D.ImShow(saveImg, waitTime=1) #Save the rendered image to files if False: if os.path.exists(args.render_dir) == False: os.mkdir(args.render_dir) render_output_path = args.render_dir + '/render_{:08d}.jpg'.format( idx) print(f"Save to {render_output_path}") cv2.imwrite(render_output_path, saveImg)
def exportOursToSpin(out_path): scaleFactor = 1.2 hagglingDBdir = '/home/hjoo/data/pytorch_motionSynth/motionsynth_data/data/processed_panoptic' haggling_files = os.listdir(hagglingDBdir + '/panopticDB_pkl_hagglingProcessed') # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] rhand_2d_list, rhand_3d_list, lhand_2d_list, lhand_3d_list = [], [], [], [] subject_id_list = [] for mocapName in haggling_files: seqname = mocapName[:18] #'170221_haggling_b1' if "170221_haggling_b" not in seqname and "170228_haggling_b" not in seqname: #testing set continue print(f"processing{mocapName}") groupid = mocapName[24:-4] # 3 print(f"{mocapName}, {seqname}, {groupid}") # groupname = f"group{groupid}" # mocapName =f'{seqname}_{groupname}.pkl' imgDir = f'/home/hjoo/data/panoptic-toolbox/{seqname}/hdImgs/00_00' calibdata = f'/home/hjoo/data/panoptic-toolbox/{seqname}/calibration_{seqname}.json' cam = json.load(open(calibdata, "r"))['cameras'] cam = cam[479:] #479 is 00_00 # with open('/run/media/hjoo/disk/data/panoptic_mtc/a4_release/annotation.pkl', 'rb') as f: # data = pickle.load(f) # with open('/run/media/hjoo/disk/data/panoptic_mtc/a4_release/camera_data.pkl', 'rb') as f: # cam = pickle.load(f) bodydatapath = f'{hagglingDBdir}/panopticDB_pkl_hagglingProcessed/' + mocapName bodydata = pickle.load(open(bodydatapath, "rb"), encoding='latin1') handdatapath = f'{hagglingDBdir}/panopticDB_hand_pkl_hagglingProcessed/' + mocapName handdata = pickle.load(open(handdatapath, "rb"), encoding='latin1') # if True: # for subj in bodydata['subjects']: # startFrame = subj['startFrame'] # for n in range(subj['joints19'].shape[1]): # imgpath = os.path.join(imgDir,"00_00_%08d.jpg" % (startFrame+n)) # img = cv2.imread(imgpath) # viewer2D.ImShow(img) groupStartFrame = bodydata['startFrame'] subNum = len(bodydata['subjects']) for subid in range(subNum): #[1]: subj_body = bodydata['subjects'][subid]['joints19'] if len(handdata['hand_right']) <= subid: continue if len(handdata['hand_left']) <= subid: continue subj_righthand = handdata['hand_right'][subid]['hand21'] subj_lefthand = handdata['hand_left'][subid]['hand21'] #Validity Score Right subj_righthand_validity = handdata['hand_right'][subid][ 'bValidFrame'] # subj_righthand_score = handdata['hand_right'][subid]['scores'] localStart = groupStartFrame - handdata['hand_right'][subid][ 'startFrame'] #This subj_righthand_validity = subj_righthand_validity[localStart:] # subj_righthand_score = subj_righthand_score[: , localStart:] #Validity Score Left subj_lefthand_validity = handdata['hand_left'][subid][ 'bValidFrame'] # subj_lefthand_score = handdata['hand_left'][subid]['scores'] localStart = groupStartFrame - handdata['hand_left'][subid][ 'startFrame'] #This subj_lefthand_validity = subj_lefthand_validity[localStart:] # subj_lefthand_score = subj_lefthand_score[:, localStart:] print("valid: {}/{}".format( sum(handdata['hand_right'][subid]['bValidFrame']), len(handdata['hand_right'][subid]['bValidFrame']))) print("valid: {}/{}".format( sum(handdata['hand_left'][subid]['bValidFrame']), len(handdata['hand_left'][subid]['bValidFrame']))) startFrame = bodydata['startFrame'] # assert subj_body.shape[1] == subj_righthand.shape[1] frameLeng = subj_body.shape[1] for ii in range(frameLeng): bVis = False frameid = startFrame + ii subject_id = "{}_g{}_s{}_{:08d}".format( seqname, groupid, subid, frameid) #seqname, groupid. subid, frameid # print(subject_id) imgFullPath = os.path.join(imgDir, "00_00_%08d.jpg" % (frameid)) body_landmark = np.array(subj_body[:, ii]).reshape( -1, 3) #19, 3 #SMC19 order # body_landmark = np.array(sample['body']['landmarks']).reshape(-1, 3) #19, 3 #SMC19 order if subj_righthand.shape[1] <= ii: rhand_landmark = np.zeros( (21, 3)) #21, 3 #SMC19 order else: rhand_landmark = np.array(subj_righthand[:, ii]).reshape( -1, 3) #21, 3 #SMC19 order if subj_lefthand.shape[1] <= ii: lhand_landmark = np.zeros( (21, 3)) #21, 3 #SMC19 order else: lhand_landmark = np.array(subj_lefthand[:, ii]).reshape( -1, 3) #21, 3 #SMC19 order calib_data = cam[0] #00 for c in calib_data: calib_data[c] = np.array(calib_data[c]) skeleton_3d_camview = applyExtrinsic(body_landmark, calib_data) #19,3 rhand_3d_camview = applyExtrinsic(rhand_landmark, calib_data) #21,3 lhand_3d_camview = applyExtrinsic(lhand_landmark, calib_data) #21,3 skeleton_2d = project2D(body_landmark, calib_data) #19,2 rhand_2d = project2D(rhand_landmark, calib_data) #19,2 lhand_2d = project2D(lhand_landmark, calib_data) #19,2 imgName = "{}/hdImgs/00_00/00_00_{:08d}.jpg".format( seqname, frameid) # print(imgName) # imgFullPath = os.path.join(imgDir, imgName) # if os.path.exists(imgFullPath) == False: # continue # print(imgName) #Visulaize 3D if False: img = cv2.imread(imgFullPath) img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) viewer2D.ImShow(img, waitTime=1) skeleton_3d_camview = skeleton_3d_camview.ravel()[:, np. newaxis] rhand_3d_camview = rhand_3d_camview.ravel()[:, np.newaxis] lhand_3d_camview = lhand_3d_camview.ravel()[:, np.newaxis] glViewer.setSkeleton([ skeleton_3d_camview, rhand_3d_camview, lhand_3d_camview ]) # glViewer.setSkeleton([skeleton_3d_camview]) glViewer.setBackgroundTexture(img) glViewer.SetOrthoCamera(True) glViewer.show(0) continue min_pt = np.min(skeleton_2d, axis=0) min_pt[0] = max(min_pt[0], 0) min_pt[1] = max(min_pt[1], 0) max_pt = np.max(skeleton_2d, axis=0) max_pt[0] = min(max_pt[0], 1920) max_pt[1] = min(max_pt[1], 1080) # bbox= [ min_pt[0], min_pt[1], max_pt[0], max_pt[1] ] bbox = [ min_pt[0], min_pt[1], max_pt[0] - min_pt[0], max_pt[1] - min_pt[1] ] center = [bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2] scale = scaleFactor * max(bbox[2], bbox[3]) / 200 #Save data # imgnames_.append(os.path.join('train',fileName_saved)) # imgnames_.append(os.path.join('train',fileName_saved)) imgnames_.append(imgName) openposes_.append(np.zeros([25, 3])) #blank centers_.append(center) scales_.append(scale) # has_smpl_.append(1) # poses_.append(sample['pose3DParam']['pose']) #(72,) # shapes_.append(sample['pose3DParam']['shape']) #(10,) #2D keypoints (total26 -> SPIN24) poseidx_spin24 = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 12, 19, 20, 22, 21, 23 ] poseidx_smc19 = [ 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 2, 0, 1, 15, 16, 17, 18 ] part = np.zeros([24, 3]) part[poseidx_spin24, :2] = skeleton_2d[ poseidx_smc19] #(52,) totalGT26 type part[poseidx_spin24, 2] = 1 parts_.append(part) #3D joint S = np.zeros([24, 4]) S[poseidx_spin24, :3] = skeleton_3d_camview[ poseidx_smc19, :] * 0.01 #Scaling skeleton 3D (currently cm) -> meter S[poseidx_spin24, 3] = 1 skel3D_.append(S) rhand_2d_list.append(rhand_2d) if len(subj_righthand_validity) <= ii: rhand_validity = 0 else: rhand_validity = subj_righthand_validity[ii] # rhand_score = subj_righthand_score[:,ii].mean() rhand_3d_camview = rhand_3d_camview * 0.01 if rhand_validity == 1: rhand_3d_camview = np.concatenate( [rhand_3d_camview, np.ones((21, 1))], axis=1) else: rhand_3d_camview = np.concatenate( [rhand_3d_camview, np.zeros((21, 1))], axis=1) bVis = True rhand_3d_list.append(rhand_3d_camview) lhand_2d_list.append(lhand_2d) if len(subj_lefthand_validity) <= ii: lhand_validity = 0 else: lhand_validity = subj_lefthand_validity[ii] # lhand_validity = subj_lefthand_validity[ii] lhand_3d_camview = lhand_3d_camview * 0.01 if lhand_validity == 1: lhand_3d_camview = np.concatenate( [lhand_3d_camview, np.ones((21, 1))], axis=1) else: lhand_3d_camview = np.concatenate( [lhand_3d_camview, np.zeros((21, 1))], axis=1) bVis = True lhand_3d_list.append(lhand_3d_camview) subject_id_list.append(subject_id) #Add hand joints # print("right: {} : left: {}".format(rhand_score, lhand_score)) # print("right: {} : left: {}".format(rhand_validity, lhand_validity)) #Debug 2D Visualize if False: #bVis: img = cv2.imread(imgFullPath) # img = cv2.imread( os.path.join( '/run/media/hjoo/disk/data/mpii_human_pose_v1/images',imgnames_[-1]) ) img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(rhand_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(lhand_2d, image=img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 0.001 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format(len(imgnames_))) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) # out_file = os.path.join(out_path, '1031-mpii3D_train_44257_all.npz') out_file = os.path.join(out_path, f'panoptic_{mocapName[:-4]}') print(f"Save to {out_file}") np.savez( out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, S=skel3D_, rhand_3d=rhand_3d_list, rhand_2d=rhand_2d_list, lhand_3d=lhand_3d_list, lhand_2d=lhand_2d_list, subjectid= subject_id_list #To handle sequence data and track the same person in output )