def _visualize_gui_naive(self, meshList, skelList=None, body_bbox_list=None, img_original=None, normal_compute=True): """ args: meshList: list of {'ver': pred_vertices, 'f': smpl.faces} skelList: list of [JointNum*3, 1] (where 1 means num. of frames in glviewer) bbr_list: list of [x,y,w,h] """ if body_bbox_list is not None: for bbr in body_bbox_list: viewer2D.Vis_Bbox(img_original, bbr) # viewer2D.ImShow(img_original) glViewer.setWindowSize(img_original.shape[1], img_original.shape[0]) # glViewer.setRenderOutputSize(inputImg.shape[1],inputImg.shape[0]) glViewer.setBackgroundTexture(img_original) glViewer.SetOrthoCamera(True) glViewer.setMeshData(meshList, bComputeNormal= normal_compute) # meshes = {'ver': pred_vertices, 'f': smplWrapper.f} if skelList is not None: glViewer.setSkeleton(skelList) if True: #Save to File if True: #Cam view rendering # glViewer.setSaveFolderName(overlaidImageFolder) glViewer.setNearPlane(50) glViewer.setWindowSize(img_original.shape[1], img_original.shape[0]) # glViewer.show_SMPL(bSaveToFile = True, bResetSaveImgCnt = False, countImg = False, mode = 'camera') glViewer.show(1) if False: #Side view rendering # glViewer.setSaveFolderName(sideImageFolder) glViewer.setNearPlane(50) glViewer.setWindowSize(img_original.shape[1], img_original.shape[0]) glViewer.show_SMPL(bSaveToFile = True, bResetSaveImgCnt = False, countImg = True, zoom=1108, mode = 'youtube')
def visualize_gui_smplpose_basic(self, smpl, poseParamList, shapeParamList=None, colorList=None, isRotMat=False, scalingFactor=300, waittime=1): ''' Visualize SMPL vertices from SMPL pose parameters This can be used as a quick visualize function if you have a pose parameters args: poseParamList: list of pose parameters (numpy array) in angle axis (72,) by default or rot matrix (24,3,3) with isRotMat==True shapeParamList: (optional) list of shape parameters (numpy array) (10,). If not provided, use a zero vector colorList: (optional) list of color RGB values e.g., (255,0,0) for red ''' zero_betas = torch.from_numpy(np.zeros((1, 10), dtype=np.float32)) default_color = glViewer.g_colorSet['eft'] meshList = [] for i, poseParam in enumerate(poseParamList): if shapeParamList is not None: shapeParam = torch.from_numpy(shapeParamList[i][np.newaxis, :]) else: shapeParam = zero_betas #.copy() if colorList is not None: color = colorList[i] else: color = default_color poseParam_tensor = torch.from_numpy( poseParam[np.newaxis, :]).float() if isRotMat: #rot mat pred_output = smpl(betas=shapeParam, body_pose=poseParam_tensor[:, 1:], global_orient=poseParam_tensor[:, [0]], pose2rot=False) else: #angle axis pred_output = smpl(betas=shapeParam, body_pose=poseParam_tensor[:, 3:], global_orient=poseParam_tensor[:, :3], pose2rot=True) nn_vertices = pred_output.vertices[0].numpy() * scalingFactor tempMesh = {'ver': nn_vertices, 'f': smpl.faces, 'color': color} meshList.append(tempMesh) # glViewer.setRenderOutputSize(inputImg.shape[1],inputImg.shape[0]) # glViewer.setBackgroundTexture(img_original) # glViewer.SetOrthoCamera(True) glViewer.setMeshData( meshList, bComputeNormal=True ) # meshes = {'ver': pred_vertices, 'f': smplWrapper.f} # glViewer.setSkeleton(skelList) glViewer.show(waittime) #Press q to escape
def visSMPLoutput_bboxSpace(smpl, pred_output, image=None, bUseSMPLX=False, waittime=-1, winsizeScale=4, color=None): """ From prediction output, obtain smpl mesh and joint TODO: Currently just assume single batch Input: pred_output['pred_shape'] pred_output['pred_rotmat'] or pred_output['pred_pose'] pred_output['pred_camera'] if waittime <0, do not call glViwer.show() Example: visSMPLoutput(self.smpl, {"pred_rotmat":pred_rotmat, "pred_shape":pred_betas, "pred_camera":pred_camera }, image = images[0]) """ smpl_output, smpl_output_bbox = getSMPLoutput_bboxSpace( smpl, pred_output, bUseSMPLX) if color is not None: smpl_output_bbox['body_mesh']['color'] = color glViewer.setMeshData([smpl_output_bbox['body_mesh']], bComputeNormal=True) glViewer.setSkeleton( [np.reshape(smpl_output_bbox['body_joints'], (-1, 1))], colorRGB=glViewer.g_colorSet['spin']) if image is not None: if type(image) == torch.Tensor: image = denormImg(image) smpl_output_bbox['img'] = image glViewer.setBackgroundTexture(image) glViewer.setWindowSize(image.shape[1] * winsizeScale, image.shape[0] * winsizeScale) glViewer.SetOrthoCamera(True) if waittime >= 0: glViewer.show(waittime) return smpl_output, smpl_output_bbox
def h36m_train_extract(dataset_path, out_path, extract_img=False): # convert joints to global order h36m_idx = [11, 6, 7, 8, 1, 2, 3, 12, 24, 14, 15, 17, 18, 19, 25, 26, 27] global_idx = [14, 3, 4, 5, 2, 1, 0, 16, 12, 17, 18, 9, 10, 11, 8, 7, 6] # structs we use imgnames_, scales_, centers_, parts_, Ss_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] # users in validation set user_list = [1, 5, 6, 7, 8] # go over each user for user_i in user_list: user_name = 'S%d' % user_i # bbox_path = os.path.join(dataset_path, user_name, 'MySegmentsMat', 'ground_truth_bb') # path with GT 3D pose pose_path = os.path.join(dataset_path, user_name, 'Poses_D3_Positions_mono') # path with GT 2D pose pose2d_path = os.path.join(dataset_path, user_name, 'Poses_D2_Positions') # path with videos # vid_path = os.path.join(dataset_path, user_name, 'Videos') # go over all the sequences of each user seq_list = glob.glob(os.path.join(pose_path, '*.cdf')) seq_list.sort() for seq_i in seq_list: print('processing: {}'.format(seq_i)) # sequence info seq_name = seq_i.split('/')[-1] action, camera, _ = seq_name.split('.') action = action.replace(' ', '_') # irrelevant sequences if action == '_ALL': continue # 3D pose file poses_3d = pycdf.CDF(seq_i)['Pose'][0] #Nx96 #load 2D pose file seq_i_2D_pose = os.path.join(pose2d_path, os.path.basename(seq_i)) poses_2d = pycdf.CDF(seq_i_2D_pose)['Pose'][0] #Nx64 poses_2d = np.reshape(poses_2d, (-1, 32, 2)) # # bbox file # bbox_file = os.path.join(bbox_path, seq_name.replace('cdf', 'mat')) # bbox_h5py = h5py.File(bbox_file) # video file if extract_img: vid_file = os.path.join(vid_path, seq_name.replace('cdf', 'mp4')) imgs_path = os.path.join(dataset_path, 'images') vidcap = cv2.VideoCapture(vid_file) success, image = vidcap.read() # go over each frame of the sequence for frame_i in range(poses_3d.shape[0]): # read video frame if extract_img: success, image = vidcap.read() if not success: break # check if you can keep this frame if frame_i % 5 == 0: # image name imgname = '%s_%s.%s_%06d.jpg' % (user_name, action, camera, frame_i + 1) #Read img # read GT 3D pose Sall = np.reshape(poses_3d[frame_i, :], [-1, 3]) / 1000. #[32,3] S17 = Sall[h36m_idx] S17 -= S17[0] # root-centered S24 = np.zeros([24, 4]) S24[global_idx, :3] = S17 S24[global_idx, 3] = 1 # # read GT bounding box # mask = bbox_h5py[bbox_h5py['Masks'][frame_i,0]].value.T # ys, xs = np.where(mask==1) # bbox = np.array([np.min(xs), np.min(ys), np.max(xs)+1, np.max(ys)+1]) curPose_2d = poses_2d[frame_i, :] min_pt = np.min(curPose_2d, axis=0) max_pt = np.max(curPose_2d, axis=0) bbox = [min_pt[0], min_pt[1], max_pt[0], max_pt[1]] #Skeleton if False: from renderer import glViewer from renderer import viewer2D image = viewer2D.Vis_Bbox_minmaxPt( image, min_pt, max_pt) viewer2D.ImShow(image, waitTime=1) S17s_ = np.array(S17) * 100 skelVis = S17s_.ravel()[:, np.newaxis] glViewer.setSkeleton([skelVis]) glViewer.show() continue center = [(bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2] scale = 0.9 * max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 200. # store data imgnames_.append(os.path.join('images', imgname)) centers_.append(center) scales_.append(scale) Ss_.append(S24) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, 'h36m_train.npz') np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, S=Ss_)
def LoadAllH36mdata_wSMPL_perSeq(out_path): # data_dir = '/home/hjoo/data/h36m-fetch/human36m_50fps/' # data_dir = '/home/hjoo/data/h36m-fetch/human36m_10fps/' list_skel2Ds_h36m = [] list_skel3Ds_h36m = [] list_smplPose = [] list_smplShape = [] list_openpose = [] list_imgNames = [] list_scale = [] list_center = [] # list_joint2d_spin24 = [] # list_joint3d_spin24 = [] TRAIN_SUBJECTS = [1, 5, 6, 7, 8] actionList = [ "Directions", "Discussion", "Eating", "Greeting", "Phoning", "Photo", "Posing", "Purchases", "Sitting", "SittingDown", "Smoking", "Waiting", "WalkDog", "Walking", "WalkTogether" ] subjectList = TRAIN_SUBJECTS for subId in subjectList: for action in actionList: gtPathList = sorted( glob.glob('{}/S{}/{}_*/*/gt_poses_coco_smpl.pkl'.format( h36mraw_dir, subId, action))) print("S{} - {}: {} files".format(subId, action, len(gtPathList))) for gtPath in gtPathList: with open(gtPath, 'rb') as f: gt_data = pickle.load(f, encoding='latin1') #Get Image List imgDir = os.path.dirname(gtPath) imgList_original = sorted( glob.glob(os.path.join(imgDir, '*.png'))) folderLeg = len(h36mraw_dir) + 1 imgList = [n[folderLeg:] for n in imgList_original] data2D_h36m = np.array(gt_data['2d']) #List -> (N,17,2) data3D_h36m = np.array(gt_data['3d']) #List -> (N,17,3) data3D_smplParams_pose = np.array( gt_data['smplParms']['poses_camCoord']) #List -> (N,72) data3D_smplParams_shape = np.array( gt_data['smplParms']['betas']) #(10,) N = data3D_smplParams_pose.shape[0] data3D_smplParams_shape = np.repeat( data3D_smplParams_shape[np.newaxis, :], N, axis=0) #List -> (N,10) #Scaling skeleton 3D (currently mm) -> meter data3D_h36m *= 0.001 #optional (centering) data3D_h36m = data3D_h36m - data3D_h36m[:, 0:1, :] scalelist = [] centerlist = [] bboxlist = [] #Generate BBox for i in range(len(data2D_h36m)): min_pt = np.min(data2D_h36m[i], axis=0) max_pt = np.max(data2D_h36m[i], axis=0) bbox = [min_pt[0], min_pt[1], max_pt[0], max_pt[1]] center = [(bbox[2] + bbox[0]) / 2, (bbox[3] + bbox[1]) / 2] scale = scaleFactor * max(bbox[2] - bbox[0], bbox[3] - bbox[1]) / 200 bboxlist.append(bbox) centerlist.append(center) scalelist.append(scale) bDraw = True if bDraw: rawImg = cv2.imread(imgFullPath) # bbox_xyxy = conv_bboxinfo_centerscale_to_bboxXYXY(center, scale) # rawImg = viewer2D.Vis_Bbox_minmaxPt(rawImg,bbox_xyxy[:2], bbox_xyxy[2:]) croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo( rawImg, center, scale, (constants.IMG_RES, constants.IMG_RES)) #Visualize image if False: rawImg = viewer2D.Vis_Skeleton_2D_SPIN49( data['keypoint2d'][0][:, :2], pt2d_visibility=data['keypoint2d'][0][:, 2], image=rawImg) viewer2D.ImShow(rawImg, name='rawImg') viewer2D.ImShow(croppedImg, name='croppedImg') b = 0 ############### Visualize Mesh ############### camParam_scale = pred_camera_vis[b, 0] camParam_trans = pred_camera_vis[b, 1:] pred_vert_vis = ours_vertices[b].copy() pred_vert_vis = convert_smpl_to_bbox( pred_vert_vis, camParam_scale, camParam_trans) #From cropped space to original pred_vert_vis = convert_bbox_to_oriIm( pred_vert_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) #Generate multi-level BBOx bbox_list = multilvel_bbox_crop_gen(rawImg, pred_vert_vis, center, scale, bDebug=False) if False: pred_meshes = { 'ver': pred_vert_vis, 'f': smpl.faces } glViewer.setMeshData([pred_meshes], bComputeNormal=True) # ################ Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = ours_joints_3d[ b, :, :3].copy() #(N,3) pred_joints_vis = convert_smpl_to_bbox( pred_joints_vis, camParam_scale, camParam_trans) pred_joints_vis = convert_bbox_to_oriIm( pred_joints_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) glViewer.setBackgroundTexture(rawImg) glViewer.setWindowSize(rawImg.shape[1], rawImg.shape[0]) glViewer.SetOrthoCamera(True) glViewer.show(1) assert len(imgList) == len(data2D_h36m) assert len(imgList) == len(data3D_h36m) assert len(imgList) == len(data3D_smplParams_pose) assert len(imgList) == len(data3D_smplParams_shape) assert len(imgList) == len(scalelist) assert len(imgList) == len(centerlist) assert len(imgList) == len(bboxlist) list_skel2Ds_h36m.append(data2D_h36m) list_skel3Ds_h36m.append(data3D_h36m) list_smplPose.append(data3D_smplParams_pose) list_smplShape.append(data3D_smplParams_shape) list_imgNames += imgList list_scale += scalelist list_center += centerlist blankopenpose = np.zeros([N, 25, 3]) list_openpose.append(blankopenpose) #Debug 2D Visualize if True: for idx in range(data2D_h36m.shape[0]): img = cv2.imread(imgList_original[idx]) img = viewer2D.Vis_Skeleton_2D_H36m(data2D_h36m[idx], image=img) img = viewer2D.Vis_Bbox_minmaxPt( img, bboxlist[idx][:2], bboxlist[idx][2:]) viewer2D.ImShow(img) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 100 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # break #debug # break #debug list_skel2Ds_h36m = np.vstack( list_skel2Ds_h36m) #List of (N,17,2) -> (NM, 17, 2) list_skel3Ds_h36m = np.vstack( list_skel3Ds_h36m) #List of (N,17,3) -> (NM, 17, 3) list_smplPose = np.vstack(list_smplPose) #List of (N,72) -> (NM, 72) list_smplShape = np.vstack(list_smplShape) #List of (N,10) -> (NM, 10/) list_openpose = np.vstack(list_openpose) #List of (N,10) -> (NM, 10/) assert len(list_imgNames) == list_skel2Ds_h36m.shape[0] assert len(list_imgNames) == list_skel3Ds_h36m.shape[0] assert len(list_imgNames) == list_smplPose.shape[0] assert len(list_imgNames) == list_smplShape.shape[0] assert len(list_imgNames) == list_openpose.shape[0] assert len(list_imgNames) == len(list_scale) assert len(list_imgNames) == len(list_center) #Convert H36M -> SPIN24 # convert joints to global order # h36m_idx = [11, 6, 7, 8, 1, 2, 3, 12, 24, 14, 15, 17, 18, 19, 25, 26, 27] h36m_idx = [0, 4, 5, 6, 1, 2, 3, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] global_idx = [14, 3, 4, 5, 2, 1, 0, 16, 12, 17, 18, 9, 10, 11, 8, 7, 6] sampleNum = len(list_imgNames) joint2d_spin24 = np.zeros((sampleNum, 24, 3)) joint2d_spin24[:, global_idx, :2] = list_skel2Ds_h36m[:, h36m_idx, :] joint2d_spin24[:, global_idx, 2] = 1 joint3d_spin24 = np.zeros((sampleNum, 24, 4)) joint3d_spin24[:, global_idx, :3] = list_skel3Ds_h36m[:, h36m_idx, :] joint3d_spin24[:, global_idx, 3] = 1 list_has_smpl = np.ones((sampleNum, ), dtype=np.uint8) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, 'h36m_training_fair_meter.npz') print("output: {}".format(out_file)) np.savez(out_file, imgname=list_imgNames, center=list_center, scale=list_scale, part=joint2d_spin24, pose=list_smplPose, shape=list_smplShape, has_smpl=list_has_smpl, S=joint3d_spin24, openpose=list_openpose)
def run_evaluation(model, dataset_name, dataset, result_file, batch_size=32, img_res=224, num_workers=32, shuffle=False, log_freq=50, bVerbose= True): """Run evaluation on the datasets and metrics we report in the paper. """ device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu') # # Transfer model to the GPU # model.to(device) # Load SMPL model global g_smpl_neutral, g_smpl_male, g_smpl_female if g_smpl_neutral is None: g_smpl_neutral = SMPL(config.SMPL_MODEL_DIR, create_transl=False).to(device) g_smpl_male = SMPL(config.SMPL_MODEL_DIR, gender='male', create_transl=False).to(device) g_smpl_female = SMPL(config.SMPL_MODEL_DIR, gender='female', create_transl=False).to(device) smpl_neutral = g_smpl_neutral smpl_male = g_smpl_male smpl_female = g_smpl_female else: smpl_neutral = g_smpl_neutral smpl_male = g_smpl_male smpl_female = g_smpl_female # renderer = PartRenderer() # Regressor for H36m joints J_regressor = torch.from_numpy(np.load(config.JOINT_REGRESSOR_H36M)).float() save_results = result_file is not None # Disable shuffling if you want to save the results if save_results: shuffle=False # Create dataloader for the dataset data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) # Pose metrics # MPJPE and Reconstruction error for the non-parametric and parametric shapes # mpjpe = np.zeros(len(dataset)) # recon_err = np.zeros(len(dataset)) quant_mpjpe = {}#np.zeros(len(dataset)) quant_recon_err = {}#np.zeros(len(dataset)) mpjpe = np.zeros(len(dataset)) recon_err = np.zeros(len(dataset)) mpjpe_smpl = np.zeros(len(dataset)) recon_err_smpl = np.zeros(len(dataset)) # Shape metrics # Mean per-vertex error shape_err = np.zeros(len(dataset)) shape_err_smpl = np.zeros(len(dataset)) # Mask and part metrics # Accuracy accuracy = 0. parts_accuracy = 0. # True positive, false positive and false negative tp = np.zeros((2,1)) fp = np.zeros((2,1)) fn = np.zeros((2,1)) parts_tp = np.zeros((7,1)) parts_fp = np.zeros((7,1)) parts_fn = np.zeros((7,1)) # Pixel count accumulators pixel_count = 0 parts_pixel_count = 0 # Store SMPL parameters output_pred_pose = np.zeros((len(dataset), 72)) output_pred_betas = np.zeros((len(dataset), 10)) output_pred_camera = np.zeros((len(dataset), 3)) output_pred_joints = np.zeros((len(dataset), 14, 3)) output_gt_pose = np.zeros((len(dataset), 72)) output_gt_betas = np.zeros((len(dataset), 10)) output_gt_joints = np.zeros((len(dataset), 14, 3)) output_error_MPJPE = np.zeros((len(dataset))) output_error_recon = np.zeros((len(dataset))) output_imgNames =[] output_cropScale = np.zeros((len(dataset))) output_cropCenter = np.zeros((len(dataset), 2)) outputStartPointer = 0 eval_pose = False eval_masks = False eval_parts = False # Choose appropriate evaluation for each dataset if dataset_name == 'h36m-p1' or dataset_name == 'h36m-p2' or dataset_name == '3dpw' or dataset_name == 'mpi-inf-3dhp': eval_pose = True elif dataset_name == 'lsp': eval_masks = True eval_parts = True annot_path = config.DATASET_FOLDERS['upi-s1h'] joint_mapper_h36m = constants.H36M_TO_J17 if dataset_name == 'mpi-inf-3dhp' else constants.H36M_TO_J14 joint_mapper_gt = constants.J24_TO_J17 if dataset_name == 'mpi-inf-3dhp' else constants.J24_TO_J14 # Iterate over the entire dataset for step, batch in enumerate(tqdm(data_loader, desc='Eval', total=len(data_loader))): # Get ground truth annotations from the batch imgName = batch['imgname'][0] seqName = os.path.basename ( os.path.dirname(imgName) ) gt_pose = batch['pose'].to(device) gt_betas = batch['betas'].to(device) gt_vertices = smpl_neutral(betas=gt_betas, body_pose=gt_pose[:, 3:], global_orient=gt_pose[:, :3]).vertices images = batch['img'].to(device) gender = batch['gender'].to(device) curr_batch_size = images.shape[0] with torch.no_grad(): pred_rotmat, pred_betas, pred_camera = model(images) pred_output = smpl_neutral(betas=pred_betas, body_pose=pred_rotmat[:,1:], global_orient=pred_rotmat[:,0].unsqueeze(1), pose2rot=False) pred_vertices = pred_output.vertices # 3D pose evaluation if eval_pose: # Regressor broadcasting J_regressor_batch = J_regressor[None, :].expand(pred_vertices.shape[0], -1, -1).to(device) # Get 14 ground truth joints if 'h36m' in dataset_name or 'mpi-inf' in dataset_name: gt_keypoints_3d = batch['pose_3d'].cuda() gt_keypoints_3d = gt_keypoints_3d[:, joint_mapper_gt, :-1] # For 3DPW get the 14 common joints from the rendered shape else: gt_vertices = smpl_male(global_orient=gt_pose[:,:3], body_pose=gt_pose[:,3:], betas=gt_betas).vertices gt_vertices_female = smpl_female(global_orient=gt_pose[:,:3], body_pose=gt_pose[:,3:], betas=gt_betas).vertices gt_vertices[gender==1, :, :] = gt_vertices_female[gender==1, :, :] gt_keypoints_3d = torch.matmul(J_regressor_batch, gt_vertices) gt_pelvis = gt_keypoints_3d[:, [0],:].clone() gt_keypoints_3d = gt_keypoints_3d[:, joint_mapper_h36m, :] gt_keypoints_3d = gt_keypoints_3d - gt_pelvis if False: from renderer import viewer2D from renderer import glViewer import humanModelViewer batchNum = gt_pose.shape[0] for i in range(batchNum): smpl_face = humanModelViewer.GetSMPLFace() meshes_gt = {'ver': gt_vertices[i].cpu().numpy()*100, 'f': smpl_face} meshes_pred = {'ver': pred_vertices[i].cpu().numpy()*100, 'f': smpl_face} glViewer.setMeshData([meshes_gt, meshes_pred], bComputeNormal= True) glViewer.show(5) # Get 14 predicted joints from the mesh pred_keypoints_3d = torch.matmul(J_regressor_batch, pred_vertices) # if save_results: # pred_joints[step * batch_size:step * batch_size + curr_batch_size, :, :] = pred_keypoints_3d.cpu().numpy() pred_pelvis = pred_keypoints_3d[:, [0],:].clone() pred_keypoints_3d = pred_keypoints_3d[:, joint_mapper_h36m, :] pred_keypoints_3d = pred_keypoints_3d - pred_pelvis #Visualize GT mesh and SPIN output mesh if False: from renderer import viewer2D from renderer import glViewer import humanModelViewer gt_keypoints_3d_vis = gt_keypoints_3d.cpu().numpy() gt_keypoints_3d_vis = np.reshape(gt_keypoints_3d_vis, (gt_keypoints_3d_vis.shape[0],-1)) #N,14x3 gt_keypoints_3d_vis = np.swapaxes(gt_keypoints_3d_vis, 0,1) *100 pred_keypoints_3d_vis = pred_keypoints_3d.cpu().numpy() pred_keypoints_3d_vis = np.reshape(pred_keypoints_3d_vis, (pred_keypoints_3d_vis.shape[0],-1)) #N,14x3 pred_keypoints_3d_vis = np.swapaxes(pred_keypoints_3d_vis, 0,1) *100 # output_sample = output_sample[ : , np.newaxis]*0.1 # gt_sample = gt_sample[: , np.newaxis]*0.1 # (skelNum, dim, frames) glViewer.setSkeleton( [gt_keypoints_3d_vis, pred_keypoints_3d_vis] ,jointType='smplcoco')#(skelNum, dim, frames) glViewer.show() # Absolute error (MPJPE) error = torch.sqrt(((pred_keypoints_3d - gt_keypoints_3d) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy() # mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error # Reconstuction_error r_error = reconstruction_error(pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) # recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error for ii, p in enumerate(batch['imgname'][:len(r_error)]): seqName = os.path.basename( os.path.dirname(p)) # quant_mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error if seqName not in quant_mpjpe.keys(): quant_mpjpe[seqName] = [] quant_recon_err[seqName] = [] quant_mpjpe[seqName].append(error[ii]) quant_recon_err[seqName].append(r_error[ii]) # Reconstuction_error # quant_recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) if bVerbose: print(">>> {} : MPJPE {:.02f} mm, error: {:.02f} mm | Total MPJPE {:.02f} mm, error {:.02f} mm".format(seqName, np.mean(error)*1000, np.mean(r_error)*1000, np.hstack(list_mpjpe).mean()*1000, np.hstack(list_reconError).mean()*1000) ) # print("MPJPE {}, error: {}".format(np.mean(error)*100, np.mean(r_error)*100)) # If mask or part evaluation, render the mask and part images # if eval_masks or eval_parts: # mask, parts = renderer(pred_vertices, pred_camera) # Mask evaluation (for LSP) if eval_masks: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() # Dimensions of original image orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): # After rendering, convert imate back to original resolution pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0 # Load gt mask gt_mask = cv2.imread(os.path.join(annot_path, batch['maskname'][i]), 0) > 0 # Evaluation consistent with the original UP-3D code accuracy += (gt_mask == pred_mask).sum() pixel_count += np.prod(np.array(gt_mask.shape)) for c in range(2): cgt = gt_mask == c cpred = pred_mask == c tp[c] += (cgt & cpred).sum() fp[c] += (~cgt & cpred).sum() fn[c] += (cgt & ~cpred).sum() f1 = 2 * tp / (2 * tp + fp + fn) # Part evaluation (for LSP) if eval_parts: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i]) # Load gt part segmentation gt_parts = cv2.imread(os.path.join(annot_path, batch['partname'][i]), 0) # Evaluation consistent with the original UP-3D code # 6 parts + background for c in range(7): cgt = gt_parts == c cpred = pred_parts == c cpred[gt_parts == 255] = 0 parts_tp[c] += (cgt & cpred).sum() parts_fp[c] += (~cgt & cpred).sum() parts_fn[c] += (cgt & ~cpred).sum() gt_parts[gt_parts == 255] = 0 pred_parts[pred_parts == 255] = 0 parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn) parts_accuracy += (gt_parts == pred_parts).sum() parts_pixel_count += np.prod(np.array(gt_parts.shape)) # Print intermediate results during evaluation if bVerbose: if step % log_freq == log_freq - 1: if eval_pose: print('MPJPE: ' + str(1000 * mpjpe[:step * batch_size].mean())) print('Reconstruction Error: ' + str(1000 * recon_err[:step * batch_size].mean())) print() if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() if save_results: rot_pad = torch.tensor([0,0,1], dtype=torch.float32, device=device).view(1,3,1) rotmat = torch.cat((pred_rotmat.view(-1, 3, 3), rot_pad.expand(curr_batch_size * 24, -1, -1)), dim=-1) pred_pose = tgm.rotation_matrix_to_angle_axis(rotmat).contiguous().view(-1, 72) output_pred_pose[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_pose.cpu().numpy() output_pred_betas[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_betas.cpu().numpy() output_pred_camera[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_camera.cpu().numpy() output_pred_pose[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_pose.cpu().numpy() output_pred_betas[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_betas.cpu().numpy() output_pred_camera[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_camera.cpu().numpy() output_pred_joints[outputStartPointer:outputStartPointer+curr_batch_size, :] = pred_keypoints_3d.cpu().numpy() output_gt_pose[outputStartPointer:outputStartPointer+curr_batch_size, :] = gt_pose.cpu().numpy() output_gt_betas[outputStartPointer:outputStartPointer+curr_batch_size, :] = gt_betas.cpu().numpy() output_gt_joints[outputStartPointer:outputStartPointer+curr_batch_size, :] = gt_keypoints_3d.cpu().numpy() output_error_MPJPE[outputStartPointer:outputStartPointer+curr_batch_size,] = error *1000 output_error_recon[outputStartPointer:outputStartPointer+curr_batch_size] = r_error*1000 output_cropScale[outputStartPointer:outputStartPointer+curr_batch_size] = batch['scale'].cpu().numpy() output_cropCenter[outputStartPointer:outputStartPointer+curr_batch_size, :] = batch['center'].cpu().numpy() output_imgNames +=batch['imgname'] outputStartPointer +=curr_batch_size # if outputStartPointer>100: #Debug # break # if len(output_imgNames) < output_pred_pose.shape[0]: output ={} finalLen = len(output_imgNames) output['imageNames'] = output_imgNames output['pred_pose'] = output_pred_pose[:finalLen] output['pred_betas'] = output_pred_betas[:finalLen] output['pred_camera'] = output_pred_camera[:finalLen] output['pred_joints'] = output_pred_joints[:finalLen] output['gt_pose'] = output_gt_pose[:finalLen] output['gt_betas'] = output_gt_betas[:finalLen] output['gt_joints'] = output_gt_joints[:finalLen] output['error_MPJPE'] = output_error_MPJPE[:finalLen] output['error_recon'] = output_error_recon[:finalLen] output['cropScale'] = output_cropScale[:finalLen] output['cropCenter'] = output_cropCenter[:finalLen] # Save reconstructions to a file for further processing if save_results: import pickle # np.savez(result_file, pred_joints=pred_joints, pred_pose=pred_pose, pred_betas=pred_betas, pred_camera=pred_camera) with open(result_file,'wb') as f: pickle.dump(output, f) f.close() print("Saved to:{}".format(result_file)) # Print final results during evaluation if bVerbose: print('*** Final Results ***') print() if eval_pose: # if bVerbose: # print('MPJPE: ' + str(1000 * mpjpe.mean())) # print('Reconstruction Error: ' + str(1000 * recon_err.mean())) # print() list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) output_str ='SeqNames; ' for seq in quant_mpjpe: output_str += seq + ';' output_str +='\n MPJPE; ' quant_mpjpe_avg_mm = np.hstack(list_mpjpe).mean()*1000 output_str += "Avg {:.02f} mm; ".format( quant_mpjpe_avg_mm) for seq in quant_mpjpe: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_mpjpe[seq]).mean()) output_str +='\n Recon Error; ' quant_recon_error_avg_mm = np.hstack(list_reconError).mean()*1000 output_str +="Avg {:.02f}mm; ".format( quant_recon_error_avg_mm ) for seq in quant_recon_err: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_recon_err[seq]).mean()) if bVerbose: print(output_str) else: print(">>> Test on 3DPW: MPJPE: {} | quant_recon_error_avg_mm: {}".format(quant_mpjpe_avg_mm, quant_recon_error_avg_mm) ) return quant_mpjpe_avg_mm, quant_recon_error_avg_mm if bVerbose: if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() return -1 #Should return something
def exportOursToSpin(cocoPose3DAll, out_path): scaleFactor = 1.2 # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_ , shapes_, skel3D_, has_smpl_ = [], [] ,[], [] # for imgSample in cocoPose3DAll: imgNum = len(cocoPose3DAll) totalSampleNum = [ len(cocoPose3DAll[imgSample]) for imgSample in cocoPose3DAll ] totalSampleNum = sum(totalSampleNum) print("\n\n### ImageNum: {}, SampleNum: {} ###".format(imgNum, totalSampleNum)) # for imgSample in cocoPose3DAll: # for key_imgId, imgSample in sorted(cocoPose3DAll.items()): for key_imgId, imgSample in sorted(cocoPose3DAll.items()): #load image imgPathFull = imgSample[0]['imgId'] fileName = os.path.basename(imgPathFull) fileName_saved = os.path.join(os.path.basename(os.path.dirname(imgPathFull)), fileName) #start from train2014 for sample in imgSample: validJointNum = np.sum(sample['pose2D_validity'][::2]) if validJointNum<4: continue if np.isnan(sample['pose3DParam']['camScale']): continue gt_skel = np.reshape(sample['pose2D_gt'],(26,-1)) #(26,2) This is from data gt_validity = np.reshape(sample['pose2D_validity'],(26,-1)) #(26,2) # Filtering ######################################################################################################## if True: requiredJoints= [0,1,2, 3,4,5, 6,7,8, 9,10,11] #In Total26 if np.min(gt_validity[requiredJoints,0])== False: continue min_pt = np.min(gt_skel[gt_validity[:,0]], axis=0) max_pt = np.max(gt_skel[gt_validity[:,0]], axis=0) # bbox= [ min_pt[0], min_pt[1], max_pt[0], max_pt[1] ] bbox= [ min_pt[0], min_pt[1], max_pt[0] - min_pt[0], max_pt[1] - min_pt[1]] center = [bbox[0] + bbox[2]/2, bbox[1] + bbox[3]/2] scale = scaleFactor*max(bbox[2], bbox[3])/200 #Save data imgnames_.append(os.path.join('images',fileName_saved)) openposes_.append(np.zeros([25,3])) #blank centers_.append(center) scales_.append(scale) has_smpl_.append(1) poses_.append(sample['pose3DParam']['pose']) #(72,) shapes_.append(sample['pose3DParam']['shape']) #(10,) #2D keypoints (total26 -> SPIN24) poseidx_spin24 = [0,1,2, 3,4,5, 6,7,8, 9,10,11, 19,20,21,22,23] poseidx_total26 = [0,1,2, 3,4,5, 6,7,8, 9,10,11, 14, 15, 16, 17, 18 ] part = np.zeros([24,3]) part[poseidx_spin24,:2] = gt_skel[poseidx_total26] #(52,) totalGT26 type part[poseidx_spin24,2] = 1*gt_validity[poseidx_total26,0] parts_.append(part) #3D joint S = np.zeros([24,4]) S[poseidx_spin24,:3] = sample['pose3D_pred'][poseidx_total26,:] * 0.001 #Scaling skeleton 3D (currently mm) -> meter S[poseidx_spin24,3] = 1 skel3D_.append(S) #Debug 2D Visualize if False: img = cv2.imread( os.path.join( '/run/media/hjoo/disk/data/mpii_human_pose_v1/images',imgnames_[-1]) ) img = viewer2D.Vis_Skeleton_2D_smplCOCO(gt_skel, pt2d_visibility = gt_validity[:,0], image =img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape(data3D_h36m, (data3D_h36m.shape[0],-1)).transpose() #(Dim, F) data3D_h36m_vis *=0.001 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton( [ data3D_h36m_vis] ,jointType='smplcoco') glViewer.show() # keypoints # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format( len(imgnames_))) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) # out_file = os.path.join(out_path, '1031-mpii3D_train_44257_all.npz') out_file = os.path.join(out_path, '1031-mpii3D_train_44257_validlimbs.npz') np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, pose=poses_, shape=shapes_, has_smpl=has_smpl_, S=skel3D_)
def visEFT_singleSubject(inputDir, imDir, smplModelDir, bUseSMPLX): if bUseSMPLX: smpl = SMPLX(smplModelDir, batch_size=1, create_transl=False) else: smpl = SMPL(smplModelDir, batch_size=1, create_transl=False) fileList = listdir(inputDir) #Check all fitting files print(">> Found {} files in the fitting folder {}".format(len(fileList), inputDir)) totalCnt =0 erroneousCnt =0 # fileList =['00_00_00008422_0.pkl', '00_00_00008422_1731.pkl', '00_00_00008422_3462.pkl'] #debug for f in sorted(fileList): #Load fileFullPath = join(inputDir, f) with open(fileFullPath,'rb') as f: dataDict = pickle.load(f) print(f"Loaded :{fileFullPath}") if 'imageName' in dataDict.keys(): #If this pkl has only one instance. Made this to hand panoptic output where pkl has multi instances dataDict = {0:dataDict} for jj, k in enumerate(dataDict): if jj%50 !=0: continue data = dataDict[k] # print(data['subjectId']) # continue if 'smpltype' in data: if (data['smpltype'] =='smpl' and bUseSMPLX) or (data['smpltype'] =='smplx' and bUseSMPLX==False): print("SMPL type mismatch error") assert False imgFullPathOri = data['imageName'][0] imgFullPath = os.path.join(imDir, os.path.basename(imgFullPathOri)) data['subjectId'] =0 #TODO debug fileName = "{}_{}".format(data['subjectId'], os.path.basename(imgFullPathOri)[:-4]) if args.bRenderToFiles and os.path.exists(os.path.join(render_dirName, fileName+".jpg")): continue if True: #Additional path checking, if not valid if os.path.exists(imgFullPath) == False: imgFullPath =getpath_level(imDir, imgFullPathOri ,1) if os.path.exists(imgFullPath) == False: imgFullPath =getpath_level(imDir, imgFullPathOri,2) if os.path.exists(imgFullPath) == False: imgFullPath =getpath_level(imDir, imgFullPathOri, 3 ) scale = data['scale'][0] center = data['center'][0] # print(data['annotId']) ours_betas = torch.from_numpy(data['pred_shape']) ours_pose_rotmat = torch.from_numpy(data['pred_pose_rotmat']) # spin_betas = torch.from_numpy(data['opt_beta']) #Compute 2D reprojection error # if not (data['loss_keypoints_2d']<0.0001 or data['loss_keypoints_2d']>0.001 : # continue maxBeta = abs(torch.max( abs(ours_betas)).item()) if data['loss_keypoints_2d']>0.0005 or maxBeta>3: erroneousCnt +=1 print(">>> loss2d: {}, maxBeta: {}".format( data['loss_keypoints_2d'],maxBeta) ) # spin_pose = torch.from_numpy(data['opt_pose']) pred_camera_vis = data['pred_camera'] if os.path.exists(imgFullPath) == False: print(imgFullPath) assert os.path.exists(imgFullPath) rawImg = cv2.imread(imgFullPath) print(imgFullPath) croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo(rawImg, center, scale, (BBOX_IMG_RES, BBOX_IMG_RES) ) #Visualize 2D image if args.bRenderToFiles ==False: viewer2D.ImShow(rawImg, name='rawImg', waitTime=10) #You should press any key viewer2D.ImShow(croppedImg, name='croppedImg', waitTime=10) ours_output = smpl(betas=ours_betas, body_pose=ours_pose_rotmat[:,1:], global_orient=ours_pose_rotmat[:,0].unsqueeze(1), pose2rot=False ) ours_vertices = ours_output.vertices.detach().cpu().numpy() ours_joints_3d = ours_output.joints.detach().cpu().numpy() #Visualize 3D mesh and 3D skeleton in BBox Space if True: b =0 camParam_scale = pred_camera_vis[b,0] camParam_trans = pred_camera_vis[b,1:] ############### Visualize Mesh ############### pred_vert_vis = ours_vertices[b].copy() pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans) pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} glViewer.setMeshData([pred_meshes], bComputeNormal= True) ################ Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = ours_joints_3d[b,:,:3].copy() #(N,3) pred_joints_vis = convert_smpl_to_bbox(pred_joints_vis, camParam_scale, camParam_trans) glViewer.setSkeleton( [pred_joints_vis.ravel()[:,np.newaxis]]) ################ Other 3D setup############### glViewer.setBackgroundTexture(croppedImg) glViewer.setWindowSize(croppedImg.shape[1]*args.windowscale, croppedImg.shape[0]*args.windowscale) glViewer.SetOrthoCamera(True) print("Press 'q' in the 3D window to go to the next sample") glViewer.show(0) #Visualize 3D mesh and 3D skeleton on original image space if True: b =0 camParam_scale = pred_camera_vis[b,0] camParam_trans = pred_camera_vis[b,1:] ############### Visualize Mesh ############### pred_vert_vis = ours_vertices[b].copy() pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans) #From cropped space to original pred_vert_vis = convert_bbox_to_oriIm(pred_vert_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} glViewer.setMeshData([pred_meshes], bComputeNormal= True) # ################ Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = ours_joints_3d[b,:,:3].copy() #(N,3) pred_joints_vis = convert_smpl_to_bbox(pred_joints_vis, camParam_scale, camParam_trans) pred_joints_vis = convert_bbox_to_oriIm(pred_joints_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) glViewer.setSkeleton( [pred_joints_vis.ravel()[:,np.newaxis]]) glViewer.setBackgroundTexture(rawImg) glViewer.setWindowSize(rawImg.shape[1]*args.magnifyFactor, rawImg.shape[0]*args.magnifyFactor) glViewer.SetOrthoCamera(True) print("Press 'q' in the 3D window to go to the next sample") if args.bRenderToFiles: #Export rendered files if os.path.exists(render_dirName) == False: #make a output folder if necessary os.mkdir(render_dirName) # subjId = data['subjectId'][22:24] fileName = "{}_{}".format(data['subjectId'], os.path.basename(imgFullPathOri)[:-4]) # rawImg = cv2.putText(rawImg,data['subjectId'],(100,100), cv2.FONT_HERSHEY_PLAIN, 2, (255,255,0),2) glViewer.render_on_image(render_dirName, fileName, rawImg) print(f"Render to {fileName}")
def visEFT_multiSubjects(inputDir, imDir, smplModelDir, bUseSMPLX = False): if bUseSMPLX: smpl = SMPLX(smplModelDir, batch_size=1, create_transl=False) else: smpl = SMPL(smplModelDir, batch_size=1, create_transl=False) fileList = listdir(inputDir) #Check all fitting files print(">> Found {} files in the fitting folder {}".format(len(fileList), inputDir)) totalCnt =0 erroneousCnt =0 #Merge sample from the same image data_perimage ={} for f in sorted(fileList): if "_init" in f: continue #Load imageName = f[:f.rfind('_')] if imageName not in data_perimage.keys(): data_perimage[imageName] =[] data_perimage[imageName].append(f) for imgName in data_perimage: eftFileNames = data_perimage[imgName] meshData =[] skelData =[] for f in eftFileNames: fileFullPath = join(inputDir, f) with open(fileFullPath,'rb') as f: data = pickle.load(f) imgFullPathOri = data['imageName'][0] imgFullPath = os.path.join(imDir, os.path.basename(imgFullPathOri)) if True: #Additional path checking, if not valid if os.path.exists(imgFullPath) == False: imgFullPath =getpath_level(imDir, imgFullPathOri ,1) if os.path.exists(imgFullPath) == False: imgFullPath =getpath_level(imDir, imgFullPathOri,2) if os.path.exists(imgFullPath) == False: imgFullPath =getpath_level(imDir, imgFullPathOri, 3 ) scale = data['scale'][0] center = data['center'][0] ours_betas = torch.from_numpy(data['pred_shape']) ours_pose_rotmat = torch.from_numpy(data['pred_pose_rotmat']) # spin_betas = torch.from_numpy(data['opt_beta']) #Compute 2D reprojection error # if not (data['loss_keypoints_2d']<0.0001 or data['loss_keypoints_2d']>0.001 : # continue maxBeta = abs(torch.max( abs(ours_betas)).item()) if data['loss_keypoints_2d']>0.0005 or maxBeta>3: erroneousCnt +=1 print(">>> loss2d: {}, maxBeta: {}".format( data['loss_keypoints_2d'],maxBeta) ) # spin_pose = torch.from_numpy(data['opt_pose']) pred_camera_vis = data['pred_camera'] assert os.path.exists(imgFullPath) rawImg = cv2.imread(imgFullPath) print(imgFullPath) croppedImg, boxScale_o2n, bboxTopLeft = crop_bboxInfo(rawImg, center, scale, (constants.IMG_RES, constants.IMG_RES) ) #Visualize 2D image if args.bRenderToFiles ==False: viewer2D.ImShow(rawImg, name='rawImg', waitTime=10) #You should press any key viewer2D.ImShow(croppedImg, name='croppedImg', waitTime=10) if bUseSMPLX: ours_output = smpl(betas=ours_betas, body_pose=ours_pose_rotmat[:,1:-2], global_orient=ours_pose_rotmat[:,0].unsqueeze(1), pose2rot=False ) # ours_output = smpl() #Default test else: ours_output = smpl(betas=ours_betas, body_pose=ours_pose_rotmat[:,1:], global_orient=ours_pose_rotmat[:,0].unsqueeze(1), pose2rot=False ) # ours_output = smpl() #Default test ours_vertices = ours_output.vertices.detach().cpu().numpy() ours_joints_3d = ours_output.joints.detach().cpu().numpy() if False: #Debugging # ours_vertices = ours_vertices - ours_joints_3d[0,12,:] save_mesh_obj(ours_vertices[0], smpl.faces, 'test.obj') #Visualize 3D mesh and 3D skeleton on original image space if True: b =0 camParam_scale = pred_camera_vis[b,0] camParam_trans = pred_camera_vis[b,1:] ############### Visualize Mesh ############### pred_vert_vis = ours_vertices[b].copy() pred_vert_vis = convert_smpl_to_bbox(pred_vert_vis, camParam_scale, camParam_trans) #From cropped space to original pred_vert_vis = convert_bbox_to_oriIm(pred_vert_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} # glViewer.setMeshData([pred_meshes], bComputeNormal= True) # ################ Visualize Skeletons ############### #Vis pred-SMPL joint # pred_joints_vis = ours_joints_3d[b,-9:,:3].copy() #(N,3) #Debuggin pred_joints_vis = ours_joints_3d[b,:,:3].copy() #(N,3) pred_joints_vis = convert_smpl_to_bbox(pred_joints_vis, camParam_scale, camParam_trans) pred_joints_vis = convert_bbox_to_oriIm(pred_joints_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) meshData.append(pred_meshes) skelData.append(pred_joints_vis.ravel()[:,np.newaxis]) # glViewer.setSkeleton( [pred_joints_vis.ravel()[:,np.newaxis]]) glViewer.setBackgroundTexture(rawImg) glViewer.setWindowSize(rawImg.shape[1]*args.magnifyFactor, rawImg.shape[0]*args.magnifyFactor) glViewer.SetOrthoCamera(True) # print("Press 'q' in the 3D window to go to the next sample") # glViewer.show(0) glViewer.setSkeleton(skelData) glViewer.setMeshData(meshData, bComputeNormal= True) if args.bRenderToFiles: #Export rendered files if os.path.exists(render_dirName) == False: #make a output folder if necessary os.mkdir(render_dirName) fileName = imgFullPathOri[:-4].replace("/","_") glViewer.render_on_image(render_dirName, fileName, rawImg) print(f"render to {fileName}") glViewer.show(args.displaytime)
# pred_vert_vis *=pred_camera_vis_init[b,0] # pred_vert_vis[:,0] += pred_camera_vis_init[b,1] #no need +1 (or 112). Rendernig has this offset already # pred_vert_vis[:,1] += pred_camera_vis_init[b,2] #no need +1 (or 112). Rendernig has this offset already # pred_vert_vis*=112 # spin_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} glViewer.setMeshData([pred_meshes], bComputeNormal=True) # glViewer.setMeshData([pred_meshes, spin_meshes], bComputeNormal= True) # glViewer.setMeshData([pred_meshes], bComputeNormal= True) # glViewer.SetMeshColor('red') glViewer.setBackgroundTexture(croppedImg) glViewer.setWindowSize(croppedImg.shape[1], croppedImg.shape[0]) glViewer.SetOrthoCamera(True) glViewer.show(0) #Save data if dbName == 'mpii' or dbName == 'lsp': imgnames_.append( os.path.join('images', os.path.basename(imgName))) elif dbName == 'coco': imgnames_.append( os.path.join('train2014', os.path.basename(imgName))) elif dbName == 'coco_semmap': imgnames_.append(os.path.basename(imgName)) #No folder name elif dbName == 'pennaction' or dbName == 'posetrack' or dbName == 'posetrack_train': imgname_wDir = os.path.join( os.path.basename(os.path.dirname(imgName)),
def train_exemplar_step(self, input_batch): self.model.train() if self.options.bExemplarMode: self.exemplerTrainingMode() # Get data from the batch images = input_batch['img'] # input image gt_keypoints_2d = input_batch[ 'keypoints'] # 2D keypoints #[N,49,3] gt_pose = input_batch[ 'pose'] # SMPL pose parameters #[N,72] gt_betas = input_batch[ 'betas'] # SMPL beta parameters #[N,10] gt_joints = input_batch[ 'pose_3d'] # 3D pose #[N,24,4] has_smpl = input_batch['has_smpl'].byte( ) == 1 # flag that indicates whether SMPL parameters are valid has_pose_3d = input_batch['has_pose_3d'].byte( ) == 1 # flag that indicates whether 3D pose is valid is_flipped = input_batch[ 'is_flipped'] # flag that indicates whether image was flipped during data augmentation rot_angle = input_batch[ 'rot_angle'] # rotation angle used for data augmentation dataset_name = input_batch[ 'dataset_name'] # name of the dataset the image comes from indices = input_batch[ 'sample_index'] # index of example inside its dataset batch_size = images.shape[0] # Get GT vertices and model joints # Note that gt_model_joints is different from gt_joints as it comes from SMPL gt_out = self.smpl(betas=gt_betas, body_pose=gt_pose[:, 3:], global_orient=gt_pose[:, :3]) gt_model_joints = gt_out.joints #[N, 49, 3] #Note this is different from gt_joints! gt_vertices = gt_out.vertices # Get current best fits from the dictionary opt_pose, opt_betas, opt_validity = self.fits_dict[(dataset_name, indices.cpu(), rot_angle.cpu(), is_flipped.cpu())] opt_pose = opt_pose.to(self.device) opt_betas = opt_betas.to(self.device) opt_output = self.smpl(betas=opt_betas, body_pose=opt_pose[:, 3:], global_orient=opt_pose[:, :3]) opt_vertices = opt_output.vertices # opt_joints = opt_output.joints opt_joints = opt_output.joints.detach() #for smpl-x #assuer that non valid opt has GT values if len(has_smpl[opt_validity == 0]) > 0: assert min(has_smpl[opt_validity == 0]) #All should be True # else: #Assue 3D DB! # opt_pose = gt_pose # opt_betas = gt_betas # opt_vertices = gt_vertices # opt_joints = gt_model_joints # De-normalize 2D keypoints from [-1,1] to pixel space gt_keypoints_2d_orig = gt_keypoints_2d.clone() gt_keypoints_2d_orig[:, :, :-1] = 0.5 * self.options.img_res * ( gt_keypoints_2d_orig[:, :, :-1] + 1) #49: (25+24) x 3 # Estimate camera translation given the model joints and 2D keypoints # by minimizing a weighted least squares loss # gt_cam_t = estimate_translation(gt_model_joints, gt_keypoints_2d_orig, focal_length=self.focal_length, img_size=self.options.img_res) opt_cam_t = estimate_translation(opt_joints, gt_keypoints_2d_orig, focal_length=self.focal_length, img_size=self.options.img_res) opt_joint2d_loss = self.smplify.get_fitting_loss( opt_pose, opt_betas, opt_cam_t, #opt_pose (N,72) (N,10) opt_cam_t: (N,3) 0.5 * self.options.img_res * torch.ones(batch_size, 2, device=self.device), #(N,2) (112, 112) gt_keypoints_2d_orig).mean(dim=-1) # Feed images in the network to predict camera and SMPL parameters pred_rotmat, pred_betas, pred_camera = self.model(images) pred_output = self.smpl(betas=pred_betas, body_pose=pred_rotmat[:, 1:], global_orient=pred_rotmat[:, 0].unsqueeze(1), pose2rot=False) pred_vertices = pred_output.vertices pred_joints_3d = pred_output.joints # # Convert Weak Perspective Camera [s, tx, ty] to camera translation [tx, ty, tz] in 3D given the bounding box size # # This camera translation can be used in a full perspective projection # pred_cam_t = torch.stack([pred_camera[:,1], # pred_camera[:,2], # 2*self.focal_length/(self.options.img_res * pred_camera[:,0] +1e-9)],dim=-1) # camera_center = torch.zeros(batch_size, 2, device=self.device) # pred_keypoints_2d = perspective_projection(pred_joints_3d, # rotation=torch.eye(3, device=self.device).unsqueeze(0).expand(batch_size, -1, -1), # translation=pred_cam_t, # focal_length=self.focal_length, # camera_center=camera_center) # # Normalize keypoints to [-1,1] # pred_keypoints_2d = pred_keypoints_2d / (self.options.img_res / 2.) ### Centering Vertex..model... more complicated! if True: pred_pelvis = (pred_joints_3d[:, 27:28, :3] + pred_joints_3d[:, 28:29, :3]) / 2 pred_joints_3d[:, :, : 3] = pred_joints_3d[:, :, : 3] - pred_pelvis #centering pred_vertices = pred_vertices - pred_pelvis gt_pelvis = (gt_joints[:, 2:3, :3] + gt_joints[:, 2:3, :3]) / 2 gt_joints[:, :, :3] = gt_joints[:, :, :3] - gt_pelvis #centering gt_model_pelvis = (gt_model_joints[:, 27:28, :3] + gt_model_joints[:, 28:29, :3]) / 2 gt_model_joints[:, :, : 3] = gt_model_joints[:, :, : 3] - gt_model_pelvis #centering gt_vertices = gt_vertices - gt_model_pelvis # Replace the optimized parameters with the ground truth parameters, if available opt_vertices[has_smpl, :, :] = gt_vertices[has_smpl, :, :] # opt_cam_t[has_smpl, :] = gt_cam_t[has_smpl, :] opt_joints[has_smpl, :, :] = gt_model_joints[has_smpl, :, :] opt_pose[has_smpl, :] = gt_pose[has_smpl, :] opt_betas[has_smpl, :] = gt_betas[has_smpl, :] #Weak projection pred_keypoints_2d = weakProjection_gpu(pred_joints_3d, pred_camera[:, 0], pred_camera[:, 1:]) #N, 49, 2 # Assert whether a fit is valid by comparing the joint loss with the threshold valid_fit = (opt_joint2d_loss < self.options.smplify_threshold).to( self.device) # Add the examples with GT parameters to the list of valid fits valid_fit = valid_fit | has_smpl # opt_keypoints_2d = perspective_projection(opt_joints, # rotation=torch.eye(3, device=self.device).unsqueeze(0).expand(batch_size, -1, -1), # translation=opt_cam_t, # focal_length=self.focal_length, # camera_center=camera_center) # opt_keypoints_2d = opt_keypoints_2d / (self.options.img_res / 2.) # Compute loss on SMPL parameters loss_regr_pose, loss_regr_betas = self.smpl_losses( pred_rotmat, pred_betas, opt_pose, opt_betas, valid_fit) # loss_regr_pose, loss_regr_betas = self.smpl_losses(pred_rotmat, pred_betas, gt_pose, gt_betas, valid_fit) # Compute 2D reprojection loss for the keypoints loss_keypoints_2d = self.keypoint2d_loss( pred_keypoints_2d, gt_keypoints_2d, self.options.openpose_train_weight, self.options.gt_train_weight) # Compute 3D keypoint loss # loss_keypoints_3d = self.keypoint_3d_loss(pred_joints_3d, gt_joints, has_pose_3d) loss_keypoints_3d = self.keypoint_3d_loss_modelSkel( pred_joints_3d, gt_model_joints[:, 25:, :], has_pose_3d) # Per-vertex loss for the shape loss_shape = self.shape_loss(pred_vertices, opt_vertices, valid_fit) # loss_shape = self.shape_loss(pred_vertices, gt_vertices, valid_fit) # Compute total loss # The last component is a loss that forces the network to predict positive depth values loss = self.options.shape_loss_weight * loss_shape +\ self.options.keypoint_loss_weight * loss_keypoints_2d +\ self.options.keypoint_loss_weight * loss_keypoints_3d +\ loss_regr_pose + self.options.beta_loss_weight * loss_regr_betas +\ ((torch.exp(-pred_camera[:,0]*10)) ** 2 ).mean() #NOte!! Loss is defined below again! if True: #Exemplar Loss. 2D only + Keep the original shape + camera regularization #At this moment just use opt_betas. Ideally, we can use the beta estimated from direct result betaMax = abs(torch.max(abs(opt_betas)).item()) pred_betasMax = abs(torch.max(abs(pred_betas)).item()) # print(pred_betasMax) if False: #betaMax<2.0: loss_regr_betas_noReject = self.criterion_regr( pred_betas, opt_betas) else: loss_regr_betas_noReject = torch.mean(pred_betas**2) #Prevent bending knee? # red_rotmat[0,6,:,:] - loss = self.options.keypoint_loss_weight * loss_keypoints_2d + \ self.options.beta_loss_weight * loss_regr_betas_noReject + \ ((torch.exp(-pred_camera[:,0]*10)) ** 2 ).mean() # print(loss_regr_betas) loss *= 60 # print("loss2D: {}, loss3D: {}".format( self.options.keypoint_loss_weight * loss_keypoints_2d,self.options.keypoint_loss_weight * loss_keypoints_3d ) ) # Do backprop self.optimizer.zero_grad() loss.backward() self.optimizer.step() # Pack output arguments for tensorboard logging output = { 'pred_vertices': 0, #pred_vertices.detach(), 'opt_vertices': 0, 'pred_cam_t': 0, #pred_cam_t.detach(), 'opt_cam_t': 0 } #Save result output = {} output['pred_pose_rotmat'] = pred_rotmat.detach().cpu().numpy() output['pred_shape'] = pred_betas.detach().cpu().numpy() output['pred_camera'] = pred_camera.detach().cpu().numpy() output['opt_pose'] = opt_pose.detach().cpu().numpy() output['opt_beta'] = opt_betas.detach().cpu().numpy() output['sampleIdx'] = input_batch['sample_index'].detach().cpu().numpy( ) #To use loader directly output['imageName'] = input_batch['imgname'] output['scale'] = input_batch['scale'].detach().cpu().numpy() output['center'] = input_batch['center'].detach().cpu().numpy() #To save new db file output['keypoint2d'] = input_batch['keypoints_original'].detach().cpu( ).numpy() losses = { 'loss': loss.detach().item(), 'loss_keypoints': loss_keypoints_2d.detach().item(), 'loss_keypoints_3d': loss_keypoints_3d.detach().item(), 'loss_regr_pose': loss_regr_pose.detach().item(), 'loss_regr_betas': loss_regr_betas.detach().item(), 'loss_shape': loss_shape.detach().item() } if self.options.bDebug_visEFT: #g_debugVisualize: #Debug Visualize input for b in range(batch_size): #denormalizeImg curImgVis = images[b] #3,224,224 curImgVis = self.de_normalize_img(curImgVis).cpu().numpy() curImgVis = np.transpose(curImgVis, (1, 2, 0)) * 255.0 curImgVis = curImgVis[:, :, [2, 1, 0]] #Denormalize image curImgVis = np.ascontiguousarray(curImgVis, dtype=np.uint8) originalImgVis = curImgVis.copy() viewer2D.ImShow(curImgVis, name='rawIm') curImgVis = viewer2D.Vis_Skeleton_2D_general( gt_keypoints_2d_orig[b, :, :2].cpu().numpy(), gt_keypoints_2d_orig[b, :, 2], bVis=False, image=curImgVis) pred_keypoints_2d_vis = pred_keypoints_2d[ b, :, :2].detach().cpu().numpy() pred_keypoints_2d_vis = 0.5 * self.options.img_res * ( pred_keypoints_2d_vis + 1) #49: (25+24) x 3 if glViewer.g_bShowSkeleton: curImgVis = viewer2D.Vis_Skeleton_2D_general( pred_keypoints_2d_vis, bVis=False, image=curImgVis) viewer2D.ImShow(curImgVis, scale=2.0, waitTime=1) #Vis Opt-SMPL joint if False: smpl_jointsVis = opt_joints[ b, :, :3].cpu().numpy() * 100 #[N,49,3] smpl_jointsVis = smpl_jointsVis.ravel()[:, np.newaxis] glViewer.setSkeleton([smpl_jointsVis]) # glViewer.show() #Vis Opt-Vertex meshVertVis = opt_vertices[b].cpu().numpy() * 100 meshes = {'ver': meshVertVis, 'f': self.smpl.faces} glViewer.setMeshData([meshes], bComputeNormal=True) #Get camera pred_params pred_camera_vis = pred_camera.detach().cpu().numpy() ############### Visualize Mesh ############### pred_vert_vis = pred_vertices[b].detach().cpu().numpy() # meshVertVis = gt_vertices[b].detach().cpu().numpy() # meshVertVis = meshVertVis-pelvis #centering pred_vert_vis *= pred_camera_vis[b, 0] pred_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already pred_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already pred_vert_vis *= 112 pred_meshes = {'ver': pred_vert_vis, 'f': self.smpl.faces} if has_smpl[b] == False: opt_vertices[has_smpl, :, :] = gt_vertices[has_smpl, :, :] opt_joints[has_smpl, :, :] = gt_model_joints[ has_smpl, :, :] opt_model_pelvis = (opt_joints[:, 27:28, :3] + opt_joints[:, 28:29, :3]) / 2 opt_vertices = opt_vertices - opt_model_pelvis opt_vert_vis = opt_vertices[b].detach().cpu().numpy() opt_vert_vis *= pred_camera_vis[b, 0] opt_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already opt_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already opt_vert_vis *= 112 opt_meshes = {'ver': opt_vert_vis, 'f': self.smpl.faces} # glViewer.setMeshData([pred_meshes, opt_meshes], bComputeNormal= True) # glViewer.setMeshData([pred_meshes], bComputeNormal= True) glViewer.setMeshData([pred_meshes, opt_meshes], bComputeNormal=True) ############### Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = pred_joints_3d[ b, :, :3].detach().cpu().numpy() #[N,49,3] pred_joints_vis = pred_joints_vis.ravel()[:, np.newaxis] #Weak-perspective projection pred_joints_vis *= pred_camera_vis[b, 0] pred_joints_vis[::3] += pred_camera_vis[b, 1] pred_joints_vis[1::3] += pred_camera_vis[b, 2] pred_joints_vis *= 112 #112 == 0.5*224 glViewer.setSkeleton([pred_joints_vis]) #Vis SMPL's Skeleton # gt_smpljointsVis = gt_model_joints[b,:,:3].cpu().numpy() #[N,49,3] # # gt_pelvis = (gt_smpljointsVis[ 25+2,:] + gt_smpljointsVis[ 25+3,:]) / 2 # # gt_smpljointsVis = gt_smpljointsVis- gt_pelvis # gt_smpljointsVis = gt_smpljointsVis.ravel()[:,np.newaxis] # gt_smpljointsVis*=pred_camera_vis[b,0] # gt_smpljointsVis[::3] += pred_camera_vis[b,1] # gt_smpljointsVis[1::3] += pred_camera_vis[b,2] # gt_smpljointsVis*=112 # glViewer.addSkeleton( [gt_smpljointsVis]) # #Vis GT joint (not model (SMPL) joint!!) # if has_pose_3d[b]: # gt_jointsVis = gt_model_joints[b,:,:3].cpu().numpy() #[N,49,3] # # gt_jointsVis = gt_joints[b,:,:3].cpu().numpy() #[N,49,3] # # gt_pelvis = (gt_jointsVis[ 25+2,:] + gt_jointsVis[ 25+3,:]) / 2 # # gt_jointsVis = gt_jointsVis- gt_pelvis # gt_jointsVis = gt_jointsVis.ravel()[:,np.newaxis] # gt_jointsVis*=pred_camera_vis[b,0] # gt_jointsVis[::3] += pred_camera_vis[b,1] # gt_jointsVis[1::3] += pred_camera_vis[b,2] # gt_jointsVis*=112 # glViewer.addSkeleton( [gt_jointsVis]) # # glViewer.show() glViewer.setBackgroundTexture(originalImgVis) glViewer.setWindowSize(curImgVis.shape[1], curImgVis.shape[0]) glViewer.SetOrthoCamera(True) glViewer.show(1) # continue return output, losses
def run_evaluation(model, dataset_name, dataset, result_file, batch_size=1, img_res=224, num_workers=32, shuffle=False, log_freq=50, bVerbose=True): """Run evaluation on the datasets and metrics we report in the paper. """ device = torch.device( 'cuda') if torch.cuda.is_available() else torch.device('cpu') # # Transfer model to the GPU # model.to(device) # Load SMPL model global g_smpl_neutral, g_smpl_male, g_smpl_female if g_smpl_neutral is None: g_smpl_neutral = SMPL(config.SMPL_MODEL_DIR, create_transl=False).to(device) g_smpl_male = SMPL(config.SMPL_MODEL_DIR, gender='male', create_transl=False).to(device) g_smpl_female = SMPL(config.SMPL_MODEL_DIR, gender='female', create_transl=False).to(device) smpl_neutral = g_smpl_neutral smpl_male = g_smpl_male smpl_female = g_smpl_female else: smpl_neutral = g_smpl_neutral smpl_male = g_smpl_male smpl_female = g_smpl_female # renderer = PartRenderer() # Regressor for H36m joints J_regressor = torch.from_numpy(np.load( config.JOINT_REGRESSOR_H36M)).float() save_results = result_file is not None # Disable shuffling if you want to save the results if save_results: shuffle = False # Create dataloader for the dataset data_loader = DataLoader(dataset, batch_size=batch_size, shuffle=shuffle, num_workers=num_workers) # Pose metrics # MPJPE and Reconstruction error for the non-parametric and parametric shapes # mpjpe = np.zeros(len(dataset)) # recon_err = np.zeros(len(dataset)) quant_mpjpe = {} #np.zeros(len(dataset)) quant_recon_err = {} #np.zeros(len(dataset)) mpjpe = np.zeros(len(dataset)) recon_err = np.zeros(len(dataset)) mpjpe_smpl = np.zeros(len(dataset)) recon_err_smpl = np.zeros(len(dataset)) # Shape metrics # Mean per-vertex error shape_err = np.zeros(len(dataset)) shape_err_smpl = np.zeros(len(dataset)) # Mask and part metrics # Accuracy accuracy = 0. parts_accuracy = 0. # True positive, false positive and false negative tp = np.zeros((2, 1)) fp = np.zeros((2, 1)) fn = np.zeros((2, 1)) parts_tp = np.zeros((7, 1)) parts_fp = np.zeros((7, 1)) parts_fn = np.zeros((7, 1)) # Pixel count accumulators pixel_count = 0 parts_pixel_count = 0 # Store SMPL parameters smpl_pose = np.zeros((len(dataset), 72)) smpl_betas = np.zeros((len(dataset), 10)) smpl_camera = np.zeros((len(dataset), 3)) pred_joints = np.zeros((len(dataset), 17, 3)) eval_pose = False eval_masks = False eval_parts = False # Choose appropriate evaluation for each dataset if dataset_name == 'h36m-p1' or dataset_name == 'h36m-p2' or dataset_name == '3dpw' or dataset_name == '3dpw-vibe' or dataset_name == 'mpi-inf-3dhp': eval_pose = True elif dataset_name == 'lsp': eval_masks = True eval_parts = True annot_path = config.DATASET_FOLDERS['upi-s1h'] joint_mapper_h36m = constants.H36M_TO_J17 if dataset_name == 'mpi-inf-3dhp' else constants.H36M_TO_J14 joint_mapper_gt = constants.J24_TO_J17 if dataset_name == 'mpi-inf-3dhp' else constants.J24_TO_J14 # Iterate over the entire dataset # cnt =0 for step, batch in enumerate( tqdm(data_loader, desc='Eval', total=len(data_loader))): # Get ground truth annotations from the batch # imgName = batch['imgname'][0] # seqName = os.path.basename ( os.path.dirname(imgName) ) gt_pose = batch['pose'].to(device) gt_betas = batch['betas'].to(device) gt_vertices = smpl_neutral(betas=gt_betas, body_pose=gt_pose[:, 3:], global_orient=gt_pose[:, :3]).vertices images = batch['img'].to(device) gender = batch['gender'].to(device) curr_batch_size = images.shape[0] bLoadFromFile = True missingPkl = False if bLoadFromFile: # pklDir= '/run/media/hjoo/disk/data/cvpr2020_eft_researchoutput/0_SPIN/0_exemplarOutput/04-22_3dpw_test_with8143_iter10' pklDir = '/private/home/hjoo/spinOut/05-11_3dpw_test_with1336_iter5' pklDir = '/private/home/hjoo/spinOut/05-11_3dpw_test_with1039_iter5' pklDir = '/private/home/hjoo/spinOut/05-11_3dpw_test_with1336_iter10' pklDir = '/private/home/hjoo/spinOut/05-11_3dpw_test_with1336_iter3' pklDir = '/run/media/hjoo/disk/data/cvpr2020_eft_researchoutput/0_SPIN/0_exemplarOutput/05-24_3dpw_test_with1336_iterUpto20' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_with1336_iterUpto50_thr2e4' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_smplify_3dpwtest_from1336' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_smplify_3dpwtest_from7640' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_smplify_3dpwtest_from5992' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_with1644_h36m_thr2e4' #New test with LSP Init pklDir = '/private/home/hjoo/spinOut/05-27_3dpw_test_smplify_3dpwtest_from732_lsp' pklDir = '/private/home/hjoo/spinOut/05-27_3dpw_test_with732_lsp_withHips' pklDir = '/private/home/hjoo/spinOut/05-27_3dpw_test_with732_lsp_noHips' #New test with MPII start #SMPLify pklDir = '/private/home/hjoo/spinOut/05-28_3dpw_test_smplify_3dpwtest_from3097_best' pklDir = '/private/home/hjoo/spinOut/05-31_3dpw_test_smplify_3dpwTest_bestW3DPW_from8653' pklDir = '/private/home/hjoo/spinOut/05-27_3dpw_test_smplify_3dpwtest_from35_mpii' #EFT pklDir = '/private/home/hjoo/spinOut/05-28_3dpw_test_with35_mpii_noHips' pklDir = '/private/home/hjoo/spinOut/05-31_3dpw_test_3dpwTest_bestW3DPW_from8653' pklDir = '/private/home/hjoo/spinOut/05-27_3dpw_test_with35_mpii_withHips' pklDir = '/private/home/hjoo/spinOut/05-27_3dpw_test_with35_mpii_noHips' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_with7640_iterUpto50_thr2e4' pklDir = '/private/home/hjoo/spinOut/05-25_3dpw_test_with5992_iterUpto50_thr2e4' pklDir = '/private/home/hjoo/spinOut/05-28_3dpw_test_byeft_with3097_best_noHips' #Rebuttal Additional Ablation pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_layer4only' #Layer 4 only pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_afterResOnly' #HMR FC part only pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_allResLayers' #Res Layers pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_layer4andLayer' #layer4 + HMR FC part #Rebuttal Additional Ablation (more) pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_layerteset_onlyRes_withconv1' #All resnet pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_layerteset_all' #no freezing pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_layerteset_decOnly' #The last regression layer pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_layerteset_fc2Later' #The last regression layer #Restart Some verification pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_ablation_noFreez' #The last regression layer pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_again_noFreez' #Original. No freeze. For debug pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_all' #Original. No freeze. For debug #Rebuttal: Real Ablation pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_onlyRes_withconv1' #Optimizing Res50. Freeze HMR FC pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_onlyAfterRes' #Optimizing HMR FC part. Freeze Res50 pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_decOnly' #Free all except the last layer of HMR pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_onlyLayer4' #Optimzing only Layer4 of ResNet pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_fc2Later' #HMR FC2 and layer pklDir = '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_from8653_ablation_layerteset_onlyRes50LastConv' #Last Conv of Res50 #Ablation: SMPLify pklDir = '/private/home/hjoo/spinOut/08-10_3dpw_test_smplify_abl_3dpwTest_from8653_noPrior' pklDir = '/private/home/hjoo/spinOut/08-10_3dpw_test_smplify_abl_3dpwTest_from8653_noCamFirst' pklDir = '/private/home/hjoo/spinOut/08-10_3dpw_test_smplify_abl_3dpwTest_from8653_noCamFirst_noPrior' pklDir = '/private/home/hjoo/spinOut/08-10_3dpw_test_smplify_abl_3dpwTest_from8653_noAnglePrior' pklDir = '/private/home/hjoo/spinOut/08-10_3dpw_test_smplify_abl_3dpwTest_from8653_noPosePrior' #CVPR 2021. New Start (old 3DPW) pklDir = '/private/home/hjoo/spinOut/10-31_3dpw_test_with7640_coco3d' pklDir = '/private/home/hjoo/spinOut/10-31_3dpw_test_with8377_cocoAl_h36_inf_3dpw' pklDir = '/private/home/hjoo/spinOut/10-31_3dpw_test_with6814_cocoAl_h36_inf' pklDir = '/private/home/hjoo/spinOut/10-31_3dpw_test_with5992_cocoAl' pklDir = '/private/home/hjoo/spinOut/10-31_3dpw_test_35_mpii' pklDir = '/private/home/hjoo/spinOut/10-31_3dpw_test_1644_h36m' #CVPR 2021. New Start (old 3DPW) pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_with8377_cocoAl_h36_inf_3dpw' pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_with6814_cocoAl_h36_inf' pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_with7640_coco3d' pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_732_lsp' pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_35_mpii' pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_with5992_cocoAl' pklDir = '/private/home/hjoo/spinOut/11-01_3dpw_test__vibe_1644_h36m' #Eval pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with7640_cocopart_iter100' # pklDir= '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with7640_cocopart_iter50' pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with5992_cocoall3d_iter100' pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with8377_cocoall3d_h36m_inf_3dpw_iter100' pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with6814_cocoall3d_h36m_inf_iter100' pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with35_mpii3d_iter100' pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with1644_h36m_iter100' pklDir = '/private/home/hjoo/spinOut/11-02_3dpw_test_smplify_with732_lspet_iter100' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_hmr/croplev_4' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_hmr/croplev_2' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_hmr/croplev_1' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_partial/croplev_1' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_partial/croplev_2' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_partial/croplev_4' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_partial/croplev_7' pklDir = '/checkpoint/hjoo/data/3dpw_crop/output_hmr/croplev_7' # pklDir= '/private/home/hjoo/spinOut/05-31_3dpw_test_3dpwTest_bestW3DPW_from8653' # pklDir= '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_ablation_noFreez20' #The last regression layer # pklDir= '/private/home/hjoo/spinOut/08-08_3dpw_test_abl_3dpwTest_from8653_ablation_layerteset_Layer2Later' #FC2 layer and later # 08-08_3dpw_test_3dpwTest_bestW3DPW_from8653 # # # 08-08_3dpw_test_abl_3dpwTest_from8653_layer4only pred_rotmat_list, pred_betas_list, pred_camera_list = [], [], [] for subjectid, name in zip(batch['subjectId'], batch['imgname']): seqName = os.path.basename(os.path.dirname(name)) subjectid = subjectid.item() imgNameOnly = os.path.basename(name)[:-4] pklfilename = f'{seqName}_{imgNameOnly}_pid{subjectid}.pkl' # pklfilename ='downtown_arguing_00_image_00049_pid0.pkl' pklfilepath = os.path.join(pklDir, pklfilename) # cnt+=1 # assert os.path.exists(pklfilepath) if os.path.exists(pklfilepath) == False: missingPkl = True # print(f"Missing file: {pklfilepath}") continue # break else: with open(pklfilepath, 'rb') as f: data = pkl.load(f, encoding='latin1') cam = data['cam'][0] # cam = data['theta'][0,:3] shape = data['theta'][0, -10:] pose_aa = data['theta'][0, 3:-10] pose_rotmat = angle_axis_to_rotation_matrix( torch.from_numpy(pose_aa).view(24, 3)) #24,4,4 pose_rotmat = pose_rotmat[:, :3, :3].numpy() #24,3,3 pred_rotmat_list.append(pose_rotmat[None, :]) pred_betas_list.append(shape[None, :]) pred_camera_list.append(cam[None, :]) pred_rotmat = torch.from_numpy( np.concatenate(pred_rotmat_list, axis=0)).cuda() pred_betas = torch.from_numpy( np.concatenate(pred_betas_list, axis=0)).cuda() pred_camera = torch.from_numpy( np.concatenate(pred_camera_list, axis=0)).cuda() # continue if missingPkl: with torch.no_grad(): assert False pred_rotmat, pred_betas, pred_camera = model(images) pred_output = smpl_neutral(betas=pred_betas, body_pose=pred_rotmat[:, 1:], global_orient=pred_rotmat[:, 0].unsqueeze(1), pose2rot=False) pred_vertices = pred_output.vertices if save_results: rot_pad = torch.tensor([0, 0, 1], dtype=torch.float32, device=device).view(1, 3, 1) rotmat = torch.cat((pred_rotmat.view( -1, 3, 3), rot_pad.expand(curr_batch_size * 24, -1, -1)), dim=-1) pred_pose = tgm.rotation_matrix_to_angle_axis( rotmat).contiguous().view(-1, 72) smpl_pose[step * batch_size:step * batch_size + curr_batch_size, :] = pred_pose.cpu().numpy() smpl_betas[step * batch_size:step * batch_size + curr_batch_size, :] = pred_betas.cpu().numpy() smpl_camera[step * batch_size:step * batch_size + curr_batch_size, :] = pred_camera.cpu().numpy() # 3D pose evaluation if eval_pose: # Regressor broadcasting J_regressor_batch = J_regressor[None, :].expand( pred_vertices.shape[0], -1, -1).to(device) # Get 14 ground truth joints if 'h36m' in dataset_name or 'mpi-inf' in dataset_name: gt_keypoints_3d = batch['pose_3d'].cuda() gt_keypoints_3d = gt_keypoints_3d[:, joint_mapper_gt, :-1] # For 3DPW get the 14 common joints from the rendered shape else: gt_vertices = smpl_male(global_orient=gt_pose[:, :3], body_pose=gt_pose[:, 3:], betas=gt_betas).vertices gt_vertices_female = smpl_female(global_orient=gt_pose[:, :3], body_pose=gt_pose[:, 3:], betas=gt_betas).vertices gt_vertices[gender == 1, :, :] = gt_vertices_female[gender == 1, :, :] gt_keypoints_3d = torch.matmul(J_regressor_batch, gt_vertices) gt_pelvis = gt_keypoints_3d[:, [0], :].clone() gt_keypoints_3d = gt_keypoints_3d[:, joint_mapper_h36m, :] gt_keypoints_3d = gt_keypoints_3d - gt_pelvis if False: from renderer import viewer2D from renderer import glViewer import humanModelViewer batchNum = gt_pose.shape[0] for i in range(batchNum): smpl_face = humanModelViewer.GetSMPLFace() meshes_gt = { 'ver': gt_vertices[i].cpu().numpy() * 100, 'f': smpl_face } meshes_pred = { 'ver': pred_vertices[i].cpu().numpy() * 100, 'f': smpl_face } glViewer.setMeshData([meshes_gt, meshes_pred], bComputeNormal=True) glViewer.show(5) # Get 14 predicted joints from the mesh pred_keypoints_3d = torch.matmul(J_regressor_batch, pred_vertices) if save_results: pred_joints[ step * batch_size:step * batch_size + curr_batch_size, :, :] = pred_keypoints_3d.cpu().numpy() pred_pelvis = pred_keypoints_3d[:, [0], :].clone() pred_keypoints_3d = pred_keypoints_3d[:, joint_mapper_h36m, :] pred_keypoints_3d = pred_keypoints_3d - pred_pelvis #Visualize GT mesh and SPIN output mesh if False: from renderer import viewer2D from renderer import glViewer import humanModelViewer gt_keypoints_3d_vis = gt_keypoints_3d.cpu().numpy() gt_keypoints_3d_vis = np.reshape( gt_keypoints_3d_vis, (gt_keypoints_3d_vis.shape[0], -1)) #N,14x3 gt_keypoints_3d_vis = np.swapaxes(gt_keypoints_3d_vis, 0, 1) * 100 pred_keypoints_3d_vis = pred_keypoints_3d.cpu().numpy() pred_keypoints_3d_vis = np.reshape( pred_keypoints_3d_vis, (pred_keypoints_3d_vis.shape[0], -1)) #N,14x3 pred_keypoints_3d_vis = np.swapaxes(pred_keypoints_3d_vis, 0, 1) * 100 # output_sample = output_sample[ : , np.newaxis]*0.1 # gt_sample = gt_sample[: , np.newaxis]*0.1 # (skelNum, dim, frames) glViewer.setSkeleton( [gt_keypoints_3d_vis, pred_keypoints_3d_vis], jointType='smplcoco') #(skelNum, dim, frames) glViewer.show() # Absolute error (MPJPE) error = torch.sqrt( ((pred_keypoints_3d - gt_keypoints_3d)**2).sum(dim=-1)).mean(dim=-1).cpu().numpy() error_upper = torch.sqrt( ((pred_keypoints_3d - gt_keypoints_3d)**2).sum(dim=-1)).mean(dim=-1).cpu().numpy() # mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error # Reconstuction_error r_error = reconstruction_error(pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) r_error_upper = reconstruction_error( pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) # recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error for ii, p in enumerate(batch['imgname'][:len(r_error)]): seqName = os.path.basename(os.path.dirname(p)) # quant_mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error if seqName not in quant_mpjpe.keys(): quant_mpjpe[seqName] = [] quant_recon_err[seqName] = [] quant_mpjpe[seqName].append(error[ii]) quant_recon_err[seqName].append(r_error[ii]) # Reconstuction_error # quant_recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error list_mpjpe = np.hstack([quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack( [quant_recon_err[k] for k in quant_recon_err]) if bVerbose: print( ">>> {} : MPJPE {:.02f} mm, error: {:.02f} mm | Total MPJPE {:.02f} mm, error {:.02f} mm" .format(seqName, np.mean(error) * 1000, np.mean(r_error) * 1000, np.hstack(list_mpjpe).mean() * 1000, np.hstack(list_reconError).mean() * 1000)) # print("MPJPE {}, error: {}".format(np.mean(error)*100, np.mean(r_error)*100)) # If mask or part evaluation, render the mask and part images # if eval_masks or eval_parts: # mask, parts = renderer(pred_vertices, pred_camera) # Mask evaluation (for LSP) if eval_masks: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() # Dimensions of original image orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): # After rendering, convert imate back to original resolution pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0 # Load gt mask gt_mask = cv2.imread( os.path.join(annot_path, batch['maskname'][i]), 0) > 0 # Evaluation consistent with the original UP-3D code accuracy += (gt_mask == pred_mask).sum() pixel_count += np.prod(np.array(gt_mask.shape)) for c in range(2): cgt = gt_mask == c cpred = pred_mask == c tp[c] += (cgt & cpred).sum() fp[c] += (~cgt & cpred).sum() fn[c] += (cgt & ~cpred).sum() f1 = 2 * tp / (2 * tp + fp + fn) # Part evaluation (for LSP) if eval_parts: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i]) # Load gt part segmentation gt_parts = cv2.imread( os.path.join(annot_path, batch['partname'][i]), 0) # Evaluation consistent with the original UP-3D code # 6 parts + background for c in range(7): cgt = gt_parts == c cpred = pred_parts == c cpred[gt_parts == 255] = 0 parts_tp[c] += (cgt & cpred).sum() parts_fp[c] += (~cgt & cpred).sum() parts_fn[c] += (cgt & ~cpred).sum() gt_parts[gt_parts == 255] = 0 pred_parts[pred_parts == 255] = 0 parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn) parts_accuracy += (gt_parts == pred_parts).sum() parts_pixel_count += np.prod(np.array(gt_parts.shape)) # Print intermediate results during evaluation if bVerbose: if step % log_freq == log_freq - 1: if eval_pose: print('MPJPE: ' + str(1000 * mpjpe[:step * batch_size].mean())) print('Reconstruction Error: ' + str(1000 * recon_err[:step * batch_size].mean())) print() if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0, 1, 2, 3, 4, 5, 6]].mean()) print() # if step==3: #Debug # break # Save reconstructions to a file for further processing if save_results: np.savez(result_file, pred_joints=pred_joints, pose=smpl_pose, betas=smpl_betas, camera=smpl_camera) # Print final results during evaluation if bVerbose: print('*** Final Results ***') print() if eval_pose: # if bVerbose: # print('MPJPE: ' + str(1000 * mpjpe.mean())) # print('Reconstruction Error: ' + str(1000 * recon_err.mean())) # print() list_mpjpe = np.hstack([quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack( [quant_recon_err[k] for k in quant_recon_err]) output_str = 'SeqNames; ' for seq in quant_mpjpe: output_str += seq + ';' output_str += '\n MPJPE; ' quant_mpjpe_avg_mm = np.hstack(list_mpjpe).mean() * 1000 output_str += "Avg {:.02f} mm; ".format(quant_mpjpe_avg_mm) for seq in quant_mpjpe: output_str += '{:.02f}; '.format( 1000 * np.hstack(quant_mpjpe[seq]).mean()) output_str += '\n Recon Error; ' quant_recon_error_avg_mm = np.hstack(list_reconError).mean() * 1000 output_str += "Avg {:.02f}mm; ".format(quant_recon_error_avg_mm) for seq in quant_recon_err: output_str += '{:.02f}; '.format( 1000 * np.hstack(quant_recon_err[seq]).mean()) if bVerbose: print(output_str) else: print( ">>> Test on 3DPW: MPJPE: {} | quant_recon_error_avg_mm: {}". format(quant_mpjpe_avg_mm, quant_recon_error_avg_mm)) #Export log to json if os.path.exists( "/private/home/hjoo/codes/fairmocap/benchmarks_eval"): targetFile = "evalFromPkl_" + os.path.basename(pklDir) + ".txt" targetFile = os.path.join( '/private/home/hjoo/codes/fairmocap/benchmarks_eval', targetFile) print(f"\n Writing output to:{targetFile}") text_file = open(targetFile, "w") text_file.write(output_str) text_file.write("\n Input Pkl Dir:{}".format(pklDir)) text_file.close() return quant_mpjpe_avg_mm, quant_recon_error_avg_mm if bVerbose: if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0, 1, 2, 3, 4, 5, 6]].mean()) print() return -1 #Should return something
pred_meshes = {'ver': pred_vert_vis, 'f': smpl.faces} glViewer.setMeshData([pred_meshes], bComputeNormal=True) # ################ Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = ours_joints_3d[b, :, :3].copy() #(N,3) pred_joints_vis = convert_smpl_to_bbox( pred_joints_vis, camParam_scale, camParam_trans) pred_joints_vis = convert_bbox_to_oriIm( pred_joints_vis, boxScale_o2n, bboxTopLeft, rawImg.shape[1], rawImg.shape[0]) glViewer.setBackgroundTexture(rawImg) glViewer.setWindowSize(rawImg.shape[1], rawImg.shape[0]) glViewer.SetOrthoCamera(True) glViewer.show(1) #Save data if dbName == 'mpii' or dbName == 'lsp': imgnames_.append( os.path.join('images', os.path.basename(imgName))) elif dbName == 'coco' or dbName == 'cocoall': imgnames_.append( os.path.join('train2014', os.path.basename(imgName))) elif dbName == 'coco_semmap': imgnames_.append(os.path.basename(imgName)) #No folder name elif dbName == 'pennaction' or dbName == 'posetrack' or dbName == 'posetrack_train': imgname_wDir = os.path.join( os.path.basename(os.path.dirname(imgName)),
def renderSMPLoutput(rootDir='/home/hjoo/temp/render_general', rendermode='overlaid', rendertype='mesh', imgname='render'): if os.path.exists(rootDir) == False: os.mkdir(rootDir) if rendermode == 'side': targetFolder = rootDir + '/side' if rendertype == 'mesh': glViewer.g_bShowSkeleton = False glViewer.g_bShowMesh = True targetFolder += "_mesh" elif rendertype == 'skeleton': glViewer.g_bShowSkeleton = True glViewer.g_bShowMesh = False targetFolder += "_skel" else: assert (False) if os.path.exists(targetFolder) == False: os.mkdir(targetFolder) glViewer.show(1) glViewer.setSaveFolderName(targetFolder) glViewer.setSaveImgName(imgname) glViewer.show_SMPL(bSaveToFile=True, bResetSaveImgCnt=False, countImg=True, mode='side', zoom=1600, bShowBG=False) # glViewer.show(0) elif rendermode == 'overlaid': targetFolder = rootDir + '/overlaid' if rendertype == 'mesh': glViewer.g_bShowSkeleton = False glViewer.g_bShowMesh = True targetFolder += "_mesh" elif rendertype == 'skeleton': glViewer.g_bShowSkeleton = True glViewer.g_bShowMesh = False targetFolder += "_skel" elif rendertype == 'raw': glViewer.g_bShowSkeleton = False glViewer.g_bShowMesh = False targetFolder += "_raw" else: assert (False) if os.path.exists(targetFolder) == False: os.mkdir(targetFolder) glViewer.setSaveFolderName(targetFolder) glViewer.setSaveImgName(imgname) glViewer.show_SMPL(bSaveToFile=True, bResetSaveImgCnt=False, countImg=True, mode='camera') else: assert (False)
def run_withWeakProj(self, init_pose, init_betas, init_cameras, camera_center, keypoints_2d, bDebugVis=False, bboxInfo=None, imagevis=None, ablation_smplify_noCamOptFirst=False, ablation_smplify_noPrior=False): """Perform body fitting. Input: init_pose: SMPL pose estimate init_betas: SMPL betas estimate init_cam_t: Camera translation estimate camera_center: Camera center location keypoints_2d: Keypoints used for the optimization Returns: vertices: Vertices of optimized shape joints: 3D joints of optimized shape pose: SMPL pose parameters of optimized shape betas: SMPL beta parameters of optimized shape camera_translation: Camera translation reprojection_loss: Final joint reprojection loss """ # batch_size = init_pose.shape[0] # Make camera translation a learnable parameter # camera_translation = init_cam_t.clone() # Get joint confidence joints_2d = keypoints_2d[:, :, :2] joints_conf = keypoints_2d[:, :, -1] # Split SMPL pose to body pose and global orientation body_pose = init_pose[:, 3:].detach().clone() global_orient = init_pose[:, :3].detach().clone() betas = init_betas.detach().clone() camera = init_cameras.detach().clone() # Step 1: Optimize camera translation and body orientation # Optimize only camera translation and body orientation body_pose.requires_grad = False betas.requires_grad = False global_orient.requires_grad = True camera.requires_grad = True camera_opt_params = [global_orient, camera] camera_optimizer = torch.optim.Adam(camera_opt_params, lr=self.step_size, betas=(0.9, 0.999)) # joints_2d # for i in range(self.num_iters*10): # g_timer.tic() if ablation_smplify_noCamOptFirst == False: for i in range(self.num_iters): smpl_output = self.smpl(global_orient=global_orient, body_pose=body_pose, betas=betas) model_joints = smpl_output.joints loss = camera_fitting_loss_weakperspective( model_joints, camera, init_cameras, joints_2d, joints_conf) camera_optimizer.zero_grad() loss.backward() camera_optimizer.step() # print(loss) #Render if False: from renderer import glViewer body_pose_all = torch.cat([global_orient, body_pose], dim=-1) #[N,72] smpl_output, smpl_output_bbox = visSMPLoutput_bboxSpace( self.smpl, { "pred_pose": body_pose_all, "pred_shape": betas, "pred_camera": camera }, color=glViewer.g_colorSet['spin'], image=imagevis) glViewer.show(1) #Render if False: root_imgname = os.path.basename( bboxInfo['imgname'])[:-4] renderRoot = f'/home/hjoo/temp/render_eft/smplify_{root_imgname}' imgname = '{:04d}'.format(i) renderSMPLoutput(renderRoot, 'overlaid', 'mesh', imgname=imgname) renderSMPLoutput(renderRoot, 'overlaid', 'skeleton', imgname=imgname) renderSMPLoutput(renderRoot, 'side', 'mesh', imgname=imgname) # g_timer.toc(average =True, bPrint=True,title="Single Camera Optimization") # Fix camera translation after optimizing camera # camera.requires_grad = False # Step 2: Optimize body joints # Optimize only the body pose and global orientation of the body body_pose.requires_grad = True betas.requires_grad = True global_orient.requires_grad = True if ablation_smplify_noCamOptFirst == False: #Original from SPIN camera.requires_grad = False body_opt_params = [body_pose, betas, global_orient] else: #New camera.requires_grad = True body_opt_params = [body_pose, betas, global_orient, camera] # For joints ignored during fitting, set the confidence to 0 joints_conf[:, self.ign_joints] = 0. # g_timer.tic() body_optimizer = torch.optim.Adam(body_opt_params, lr=self.step_size, betas=(0.9, 0.999)) for i in range(self.num_iters): smpl_output = self.smpl(global_orient=global_orient, body_pose=body_pose, betas=betas) g_timer.tic() model_joints = smpl_output.joints # loss = body_fitting_loss(body_pose, betas, model_joints, camera, camera_center, # joints_2d, joints_conf, self.pose_prior, # focal_length=self.focal_length) if ablation_smplify_noPrior: # print('ablation_smplify_noPrior') loss, reprojection_loss = body_fitting_loss_weakperspective( body_pose, betas, model_joints, camera, joints_2d, joints_conf, self.pose_prior, angle_prior_weight=0 ) #,pose_prior_weight=0) # pose_prior_weight=0)#, angle_prior_weight=0) else: loss, reprojection_loss = body_fitting_loss_weakperspective( body_pose, betas, model_joints, camera, joints_2d, joints_conf, self.pose_prior) #Stop with sufficiently small pixel error if reprojection_loss.mean() < 2.0: break body_optimizer.zero_grad() loss.backward() body_optimizer.step() g_timer.toc(average=False, bPrint=True, title="SMPLify iter") if bDebugVis: from renderer import glViewer body_pose_all = torch.cat([global_orient, body_pose], dim=-1) #[N,72] smpl_output, smpl_output_bbox = visSMPLoutput_bboxSpace( self.smpl, { "pred_pose": body_pose_all, "pred_shape": betas, "pred_camera": camera }, waittime=1, color=glViewer.g_colorSet['spin'], image=imagevis) # #Render if False: root_imgname = os.path.basename(bboxInfo['imgname'])[:-4] renderRoot = f'/home/hjoo/temp/render_eft/smplify_{root_imgname}' imgname = '{:04d}'.format(i + self.num_iters) renderSMPLoutput(renderRoot, 'overlaid', 'mesh', imgname=imgname) renderSMPLoutput(renderRoot, 'overlaid', 'skeleton', imgname=imgname) renderSMPLoutput(renderRoot, 'side', 'mesh', imgname=imgname) # g_timer.toc(average =True, bPrint=True,title="Whole body optimization") # Get final loss value with torch.no_grad(): smpl_output = self.smpl(global_orient=global_orient, body_pose=body_pose, betas=betas, return_full_pose=True) model_joints = smpl_output.joints # reprojection_loss = body_fitting_loss(body_pose, betas, model_joints, camera_translation, camera_center, # joints_2d, joints_conf, self.pose_prior, # focal_length=self.focal_length, # output='reprojection') if ablation_smplify_noPrior: reprojection_loss = body_fitting_loss_weakperspective( body_pose, betas, model_joints, camera, joints_2d, joints_conf, self.pose_prior, angle_prior_weight= 0, #pose_prior_weight=0, # pose_prior_weight=0 angle_prior_weight=0, output='reprojection') else: #Original reprojection_loss = body_fitting_loss_weakperspective( body_pose, betas, model_joints, camera, joints_2d, joints_conf, self.pose_prior, output='reprojection') vertices = smpl_output.vertices.detach() joints = smpl_output.joints.detach() pose = torch.cat([global_orient, body_pose], dim=-1).detach() betas = betas.detach() if bDebugVis: from renderer import glViewer body_pose_all = torch.cat([global_orient, body_pose], dim=-1) #[N,72] visSMPLoutput_bboxSpace(self.smpl, { "pred_pose": body_pose_all, "pred_shape": betas, "pred_camera": camera }, color=glViewer.g_colorSet['spin']) if False: glViewer.show() elif False: #Render to Files in original image space bboxCenter = bboxInfo['bboxCenter'] bboxScale = bboxInfo['bboxScale'] imgname = bboxInfo['imgname'] #Get Skeletons import cv2 img_original = cv2.imread(imgname) # viewer2D.ImShow(img_original, waitTime=0) imgShape = img_original.shape[:2] smpl_output, smpl_output_bbox, smpl_output_imgspace = getSMPLoutput_imgSpace( self.smpl, { "pred_pose": body_pose_all, "pred_shape": betas, "pred_camera": camera }, bboxCenter, bboxScale, imgShape) glViewer.setBackgroundTexture( img_original) #Vis raw video as background glViewer.setWindowSize(img_original.shape[1] * 2, img_original.shape[0] * 2) #Vis raw video as background smpl_output_imgspace['body_mesh'][ 'color'] = glViewer.g_colorSet['spin'] glViewer.setMeshData( [smpl_output_imgspace['body_mesh']], bComputeNormal=True) #Vis raw video as background glViewer.setSkeleton([]) imgname = os.path.basename(imgname)[:-4] fileName = "smplify_{0}_{1:04d}".format(imgname, 0) # rawImg = cv2.putText(rawImg,data['subjectId'],(100,100), cv2.FONT_HERSHEY_PLAIN, 2, (255,255,0),2) glViewer.render_on_image('/home/hjoo/temp/render_eft', fileName, img_original, scaleFactor=2) glViewer.show() else: root_imgname = os.path.basename(bboxInfo['imgname'])[:-4] # renderRoot=f'/home/hjoo/temp/render_eft/smplify_{root_imgname}' renderRoot = f'/home/hjoo/temp/render_rebuttal/smplify_{root_imgname}' imgname = '{:04d}'.format(i + self.num_iters) renderSMPLoutput(renderRoot, 'overlaid', 'mesh', imgname=imgname) renderSMPLoutput(renderRoot, 'overlaid', 'skeleton', imgname=imgname) renderSMPLoutput(renderRoot, 'side', 'mesh', imgname=imgname) renderSMPLoutput_merge(renderRoot) return vertices, joints, pose, betas, camera, reprojection_loss
def exportOursToSpin(eftDir, out_path): # scaleFactor = 1.2 # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] pose3DList = os.listdir(eftDir) # for imgSample in cocoPose3DAll: sampleNum = len(pose3DList) # totalSampleNum = [ len(cocoPose3DAll[imgSample]) for imgSample in cocoPose3DAll ] # totalSampleNum = sum(totalSampleNum) print("\n\n### SampleNum: {} ###".format(sampleNum)) maxDiff = 0 for fname in tqdm(sorted(pose3DList)): fname_path = os.path.join(eftDir, fname) pose3d = pickle.load(open(fname_path, 'rb')) #load image imgPathFull = pose3d['imageName'][0] fileName = os.path.basename(imgPathFull) fileName_saved = os.path.join( os.path.basename(os.path.dirname(imgPathFull)), fileName) #start from train2014 center = pose3d['center'][0] scale = pose3d['scale'][0] smpl_shape = pose3d['pred_shape'].ravel() smpl_pose_mat = torch.from_numpy( pose3d['pred_pose_rotmat'][0]) #24,3,3 pred_rotmat_hom = torch.cat([ smpl_pose_mat.view(-1, 3, 3), torch.tensor( [0, 0, 0], dtype=torch.float32, ).view(1, 3, 1).expand(24, -1, -1) ], dim=-1) smpl_pose = tgm.rotation_matrix_to_angle_axis( pred_rotmat_hom).contiguous().view(-1, 72) #verification if True: recon_mat = batch_rodrigues(smpl_pose.view( -1, 3)) #24,3... axis -> rotmat diff = abs(recon_mat.numpy() - pose3d['pred_pose_rotmat'][0]) #2.1234155e-07 # print(np.max(diff)) maxDiff = max(maxDiff, np.max(diff)) smpl_pose = smpl_pose.numpy().ravel() openpose2d = pose3d['keypoint2d'][0][:25] #25,3 spin2d_skel24 = pose3d['keypoint2d'][0][25:] #24,3 #Save data imgnames_.append(fileName_saved) centers_.append(center) scales_.append(scale) has_smpl_.append(1) poses_.append(smpl_pose) #(72,) shapes_.append(smpl_shape) #(10,) openposes_.append(openpose2d) #blank # print(openpose2d)/ parts_.append(spin2d_skel24) #3D joint S = np.zeros( [24, 4]) #blank for 3d. TODO: may need to add valid data for this skel3D_.append(S) #Debug 2D Visualize if False: img = cv2.imread( os.path.join('/run/media/hjoo/disk/data/coco', imgnames_[-1])) img = viewer2D.Vis_Skeleton_2D_smplCOCO( gt_skel, pt2d_visibility=gt_validity[:, 0], image=img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 100 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # keypoints # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format(len(imgnames_))) print("maxDiff in rot conv.: {}".format(maxDiff)) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, os.path.basename(eftDir) + '.npz') print(f"Save to {out_file}") np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, pose=poses_, shape=shapes_, has_smpl=has_smpl_, S=skel3D_)
def train_data(dataset_path, openpose_path, out_path, joints_idx, scaleFactor, extract_img=False, fits_3d=None): joints17_idx = [4, 18, 19, 20, 23, 24, 25, 3, 5, 6, 7, 9, 10, 11, 14, 15, 16] h, w = 2048, 2048 imgnames_, scales_, centers_ = [], [], [] parts_, Ss_, openposes_ = [], [], [] S17s_ =[] # training data user_list = range(1,9) seq_list = range(1,3) vid_list = list(range(3)) + list(range(4,9)) counter = 0 for user_i in user_list: for seq_i in seq_list: seq_path = os.path.join(dataset_path, 'S' + str(user_i), 'Seq' + str(seq_i)) # mat file with annotations annot_file = os.path.join(seq_path, 'annot.mat') annot2 = sio.loadmat(annot_file)['annot2'] # annot3 = sio.loadmat(annot_file)['annot3'] # calibration file and camera parameters calib_file = os.path.join(seq_path, 'camera.calibration') Ks, Rs, Ts = read_calibration(calib_file, vid_list) for j, vid_i in enumerate(vid_list): # image folder imgs_path = os.path.join(seq_path, 'imageFrames', 'video_' + str(vid_i)) # extract frames from video file if extract_img: # if doesn't exist if not os.path.isdir(imgs_path): os.makedirs(imgs_path) # video file vid_file = os.path.join(seq_path, 'imageSequence', 'video_' + str(vid_i) + '.avi') vidcap = cv2.VideoCapture(vid_file) # process video frame = 0 while 1: # extract all frames success, image = vidcap.read() if not success: break frame += 1 # image name imgname = os.path.join(imgs_path, 'frame_%06d.jpg' % frame) # save image cv2.imwrite(imgname, image) # per frame cam_aa = cv2.Rodrigues(Rs[j])[0].T[0] pattern = os.path.join(imgs_path, '*.jpg') img_list = glob.glob(pattern) for i, img_i in enumerate(img_list): # for each image we store the relevant annotations img_name = img_i.split('/')[-1] img_view = os.path.join('S' + str(user_i), 'Seq' + str(seq_i), 'imageFrames', 'video_' + str(vid_i), img_name) joints = np.reshape(annot2[vid_i][0][i], (28, 2))[joints17_idx] #2D joint, 17,2 S17 = np.reshape(annot3[vid_i][0][i], (28, 3))/1000 #3D joint. 28, 3 S17 = S17[joints17_idx] - S17[4] # 4 is the root #MPI_INOF -> 17,3 if False: skelVis = np.reshape(S17.ravel(), (-1,1) )*100 glViewer.setSkeleton(skelVis) glViewer.show(5) continue bbox = [min(joints[:,0]), min(joints[:,1]), max(joints[:,0]), max(joints[:,1])] center = [(bbox[2]+bbox[0])/2, (bbox[3]+bbox[1])/2] scale = scaleFactor*max(bbox[2]-bbox[0], bbox[3]-bbox[1])/200 # check that all joints are visible x_in = np.logical_and(joints[:, 0] < w, joints[:, 0] >= 0) y_in = np.logical_and(joints[:, 1] < h, joints[:, 1] >= 0) ok_pts = np.logical_and(x_in, y_in) if np.sum(ok_pts) < len(joints_idx): continue part = np.zeros([24,3]) part[joints_idx] = np.hstack([joints, np.ones([17,1])]) json_file = os.path.join(openpose_path, img_view.replace('.jpg', '_keypoints.json')) openpose = read_openpose(json_file, part, 'mpi_inf_3dhp') if False: from renderer import viewer2D viewer2D.Vis_Skeleton_2D_general(openpose[:,:2]) S = np.zeros([24,4]) S[joints_idx] = np.hstack([S17, np.ones([17,1])]) # because of the dataset size, we only keep every 10th frame counter += 1 if counter % 10 != 1: continue # store the data imgnames_.append(img_view) centers_.append(center) scales_.append(scale) parts_.append(part) #2dJoint 24,3 Ss_.append(S) #3D joint 24,4 S17s_.append(S17) #3D joint 17,4...#Debug openposes_.append(openpose) #openpose 25,3 if len(S17s_)==105: #Debug break#Debug break#Debug break#Debug break #Debug # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) out_file = os.path.join(out_path, 'mpi_inf_3dhp_train.npz') if fits_3d is not None: fits_3d = np.load(fits_3d) if True: #Visualize SMPl from humanModelViewer import smplModelViewer frameLen=100 poses = fits_3d['pose'][:frameLen] shape = fits_3d['shape'][:frameLen] meshes, j_coco19, _ = smplModelViewer(poses, shape,returnUnit='cm', bCentering=False, jointType='smplcoco_total26', modelType='f') glViewer.setMeshData([meshes],bComputeNormal=True) #Skeleton S17s_ = np.array(S17s_[:frameLen]) *100 skelVis = np.reshape(S17s_, (S17s_.shape[0],-1) ) #N,dim skelVis = np.swapaxes(skelVis, 0,1) glViewer.setSkeleton([skelVis]) glViewer.show() np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, pose=fits_3d['pose'], shape=fits_3d['shape'], has_smpl=fits_3d['has_smpl'], S=Ss_, openpose=openposes_) else: np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, S=Ss_, openpose=openposes_)
def exportOursToSpin(out_path): scaleFactor = 1.2 imgDir = '/run/media/hjoo/disk/data/panoptic_mtc/a4_release/hdImgs/' with open( '/run/media/hjoo/disk/data/panoptic_mtc/a4_release/annotation.pkl', 'rb') as f: data = pickle.load(f) with open( '/run/media/hjoo/disk/data/panoptic_mtc/a4_release/camera_data.pkl', 'rb') as f: cam = pickle.load(f) # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] rhand_2d_list, rhand_3d_list, lhand_2d_list, lhand_3d_list = [], [], [], [] for training_testing, mode_data in data.items(): len_mode = len(mode_data) for i, sample in enumerate(tqdm(mode_data)): seqName = sample['seqName'] # print(seqName) frame_str = sample['frame_str'] frame_path = '{}/{}'.format(seqName, frame_str) assert 'body' in sample if 'right_hand' not in sample or 'left_hand' not in sample: continue body_landmark = np.array(sample['body']['landmarks']).reshape( -1, 3) #19, 3 #SMC19 order rhand_landmark = np.array( sample['right_hand']['landmarks']).reshape( -1, 3) #19, 3 #SMC19 order lhand_landmark = np.array( sample['left_hand']['landmarks']).reshape( -1, 3) #19, 3 #SMC19 order # randomly project the skeleton to a viewpoint for c in range(0, 30): if c in [1, 2, 4, 6, 7, 13, 17, 19, 28]: #Exclude top views continue calib_data = cam[seqName][c] skeleton_3d_camview = applyExtrinsic(body_landmark, calib_data) #19,3 rhand_3d_camview = applyExtrinsic(rhand_landmark, calib_data) #21,3 lhand_3d_camview = applyExtrinsic(lhand_landmark, calib_data) #21,3 skeleton_2d = project2D(body_landmark, calib_data) #19,2 rhand_2d = project2D(rhand_landmark, calib_data) #19,2 lhand_2d = project2D(lhand_landmark, calib_data) #19,2 imgName = os.path.join(frame_path, '00_{:02d}_{}.jpg'.format(c, frame_str)) # print(imgName) imgFullPath = os.path.join(imgDir, imgName) if os.path.exists(imgFullPath) == False: continue # print(imgName) #Visulaize 3D if False: img = cv2.imread(imgFullPath) # img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) # viewer2D.ImShow(img, waitTime=1) skeleton_3d_camview = skeleton_3d_camview.ravel()[:, np. newaxis] rhand_3d_camview = rhand_3d_camview.ravel()[:, np.newaxis] lhand_3d_camview = lhand_3d_camview.ravel()[:, np.newaxis] glViewer.setSkeleton([ skeleton_3d_camview, rhand_3d_camview, lhand_3d_camview ]) glViewer.setBackgroundTexture(img) glViewer.SetOrthoCamera(True) glViewer.show(0) min_pt = np.min(skeleton_2d, axis=0) min_pt[0] = max(min_pt[0], 0) min_pt[1] = max(min_pt[1], 0) max_pt = np.max(skeleton_2d, axis=0) max_pt[0] = min(max_pt[0], 1920) max_pt[1] = min(max_pt[1], 1080) # bbox= [ min_pt[0], min_pt[1], max_pt[0], max_pt[1] ] bbox = [ min_pt[0], min_pt[1], max_pt[0] - min_pt[0], max_pt[1] - min_pt[1] ] center = [bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2] scale = scaleFactor * max(bbox[2], bbox[3]) / 200 #Save data # imgnames_.append(os.path.join('train',fileName_saved)) # imgnames_.append(os.path.join('train',fileName_saved)) imgnames_.append(imgName) openposes_.append(np.zeros([25, 3])) #blank centers_.append(center) scales_.append(scale) # has_smpl_.append(1) # poses_.append(sample['pose3DParam']['pose']) #(72,) # shapes_.append(sample['pose3DParam']['shape']) #(10,) #2D keypoints (total26 -> SPIN24) poseidx_spin24 = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 12, 19, 20, 22, 21, 23 ] poseidx_smc19 = [ 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 2, 0, 1, 15, 16, 17, 18 ] part = np.zeros([24, 3]) part[poseidx_spin24, :2] = skeleton_2d[ poseidx_smc19] #(52,) totalGT26 type part[poseidx_spin24, 2] = 1 parts_.append(part) #3D joint S = np.zeros([24, 4]) S[poseidx_spin24, :3] = skeleton_3d_camview[ poseidx_smc19, :] * 0.01 #Scaling skeleton 3D (currently cm) -> meter S[poseidx_spin24, 3] = 1 skel3D_.append(S) rhand_2d_list.append(rhand_2d) rhand_3d_list.append(rhand_3d_camview * 0.01) lhand_2d_list.append(lhand_2d) lhand_3d_list.append(lhand_3d_camview * 0.01) #Add hand joints #Debug 2D Visualize if False: img = cv2.imread(imgFullPath) # img = cv2.imread( os.path.join( '/run/media/hjoo/disk/data/mpii_human_pose_v1/images',imgnames_[-1]) ) img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(rhand_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(lhand_2d, image=img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 0.001 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format(len(imgnames_))) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) # out_file = os.path.join(out_path, '1031-mpii3D_train_44257_all.npz') out_file = os.path.join(out_path, 'panopticDB.npz') print(f"Save to {out_file}") np.savez(out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, S=skel3D_, rhand_3d=rhand_3d_list, rhand_2d=rhand_2d_list, lhand_3d=lhand_3d_list, lhand_2d=lhand_2d_list)
def train_step(self, input_batch): self.model.train() # Get data from the batch images = input_batch['img'] # input image gt_keypoints_2d = input_batch[ 'keypoints'] # 2D keypoints #[N,49,3] gt_pose = input_batch[ 'pose'] # SMPL pose parameters #[N,72] gt_betas = input_batch[ 'betas'] # SMPL beta parameters #[N,10] gt_joints = input_batch[ 'pose_3d'] # 3D pose #[N,24,4] has_smpl = input_batch['has_smpl'].byte( ) == 1 # flag that indicates whether SMPL parameters are valid has_pose_3d = input_batch['has_pose_3d'].byte( ) == 1 # flag that indicates whether 3D pose is valid is_flipped = input_batch[ 'is_flipped'] # flag that indicates whether image was flipped during data augmentation rot_angle = input_batch[ 'rot_angle'] # rotation angle used for data augmentation dataset_name = input_batch[ 'dataset_name'] # name of the dataset the image comes from indices = input_batch[ 'sample_index'] # index of example inside its dataset batch_size = images.shape[0] #Debug temporary scaling for h36m # Get GT vertices and model joints # Note that gt_model_joints is different from gt_joints as it comes from SMPL gt_out = self.smpl(betas=gt_betas, body_pose=gt_pose[:, 3:], global_orient=gt_pose[:, :3]) gt_model_joints = gt_out.joints.detach() #[N, 49, 3] gt_vertices = gt_out.vertices # else: # gt_out = self.smpl(betas=gt_betas, body_pose=gt_pose[:,3:-6], global_orient=gt_pose[:,:3]) # gt_model_joints = gt_out.joints.detach() #[N, 49, 3] # gt_vertices = gt_out.vertices # Get current best fits from the dictionary opt_pose, opt_betas, opt_validity = self.fits_dict[(dataset_name, indices.cpu(), rot_angle.cpu(), is_flipped.cpu())] opt_pose = opt_pose.to(self.device) opt_betas = opt_betas.to(self.device) # if g_smplx == False: opt_output = self.smpl(betas=opt_betas, body_pose=opt_pose[:, 3:], global_orient=opt_pose[:, :3]) opt_vertices = opt_output.vertices opt_joints = opt_output.joints.detach() # else: # opt_output = self.smpl(betas=opt_betas, body_pose=opt_pose[:,3:-6], global_orient=opt_pose[:,:3]) # opt_vertices = opt_output.vertices # opt_joints = opt_output.joints.detach() #assuer that non valid opt has GT values if len(has_smpl[opt_validity == 0]) > 0: assert min(has_smpl[opt_validity == 0]) #All should be True #assuer that non valid opt has GT values if len(has_smpl[opt_validity == 0]) > 0: assert min(has_smpl[opt_validity == 0]) #All should be True # De-normalize 2D keypoints from [-1,1] to pixel space gt_keypoints_2d_orig = gt_keypoints_2d.clone() gt_keypoints_2d_orig[:, :, :-1] = 0.5 * self.options.img_res * ( gt_keypoints_2d_orig[:, :, :-1] + 1) # Estimate camera translation given the model joints and 2D keypoints # by minimizing a weighted least squares loss gt_cam_t = estimate_translation(gt_model_joints, gt_keypoints_2d_orig, focal_length=self.focal_length, img_size=self.options.img_res) opt_cam_t = estimate_translation(opt_joints, gt_keypoints_2d_orig, focal_length=self.focal_length, img_size=self.options.img_res) opt_joint_loss = self.smplify.get_fitting_loss( opt_pose, opt_betas, opt_cam_t, #opt_pose (N,72) (N,10) opt_cam_t: (N,3) 0.5 * self.options.img_res * torch.ones(batch_size, 2, device=self.device), #(N,2) (112, 112) gt_keypoints_2d_orig).mean(dim=-1) # Feed images in the network to predict camera and SMPL parameters pred_rotmat, pred_betas, pred_camera = self.model(images) # if g_smplx == False: #Original pred_output = self.smpl(betas=pred_betas, body_pose=pred_rotmat[:, 1:], global_orient=pred_rotmat[:, 0].unsqueeze(1), pose2rot=False) # else: # pred_output = self.smpl(betas=pred_betas, body_pose=pred_rotmat[:,1:-2], global_orient=pred_rotmat[:,0].unsqueeze(1), pose2rot=False) pred_vertices = pred_output.vertices pred_joints = pred_output.joints # Convert Weak Perspective Camera [s, tx, ty] to camera translation [tx, ty, tz] in 3D given the bounding box size # This camera translation can be used in a full perspective projection pred_cam_t = torch.stack([ pred_camera[:, 1], pred_camera[:, 2], 2 * self.focal_length / (self.options.img_res * pred_camera[:, 0] + 1e-9) ], dim=-1) camera_center = torch.zeros(batch_size, 2, device=self.device) pred_keypoints_2d = perspective_projection( pred_joints, rotation=torch.eye(3, device=self.device).unsqueeze(0).expand( batch_size, -1, -1), translation=pred_cam_t, focal_length=self.focal_length, camera_center=camera_center) # Normalize keypoints to [-1,1] pred_keypoints_2d = pred_keypoints_2d / (self.options.img_res / 2.) #Weak Projection if self.options.bUseWeakProj: pred_keypoints_2d = weakProjection_gpu(pred_joints, pred_camera[:, 0], pred_camera[:, 1:]) #N, 49, 2 bFootOriLoss = False if bFootOriLoss: #Ignore hips and hip centers, foot # LENGTH_THRESHOLD = 0.0089 #1/112.0 #at least it should be 5 pixel #Disable parts gt_keypoints_2d[:, 2 + 25, 2] = 0 gt_keypoints_2d[:, 3 + 25, 2] = 0 gt_keypoints_2d[:, 14 + 25, 2] = 0 #Disable Foots gt_keypoints_2d[:, 5 + 25, 2] = 0 #Left foot gt_keypoints_2d[:, 0 + 25, 2] = 0 #Right foot if self.options.run_smplify: # Convert predicted rotation matrices to axis-angle pred_rotmat_hom = torch.cat([ pred_rotmat.detach().view(-1, 3, 3).detach(), torch.tensor( [0, 0, 1], dtype=torch.float32, device=self.device).view( 1, 3, 1).expand(batch_size * 24, -1, -1) ], dim=-1) pred_pose = rotation_matrix_to_angle_axis( pred_rotmat_hom).contiguous().view(batch_size, -1) # tgm.rotation_matrix_to_angle_axis returns NaN for 0 rotation, so manually hack it pred_pose[torch.isnan(pred_pose)] = 0.0 # Run SMPLify optimization starting from the network prediction new_opt_vertices, new_opt_joints,\ new_opt_pose, new_opt_betas,\ new_opt_cam_t, new_opt_joint_loss = self.smplify( pred_pose.detach(), pred_betas.detach(), pred_cam_t.detach(), 0.5 * self.options.img_res * torch.ones(batch_size, 2, device=self.device), gt_keypoints_2d_orig) new_opt_joint_loss = new_opt_joint_loss.mean(dim=-1) # Will update the dictionary for the examples where the new loss is less than the current one update = (new_opt_joint_loss < opt_joint_loss) # print("new_opt_joint_loss{} vs opt_joint_loss{}".format(new_opt_joint_loss)) if True: #Visualize opt for b in range(batch_size): curImgVis = images[b] #3,224,224 curImgVis = self.de_normalize_img(curImgVis).cpu().numpy() curImgVis = np.transpose(curImgVis, (1, 2, 0)) * 255.0 curImgVis = curImgVis[:, :, [2, 1, 0]] #Denormalize image curImgVis = np.ascontiguousarray(curImgVis, dtype=np.uint8) viewer2D.ImShow(curImgVis, name='rawIm') originalImg = curImgVis.copy() pred_camera_vis = pred_camera.detach().cpu().numpy() opt_vert_vis = opt_vertices[b].detach().cpu().numpy() opt_vert_vis *= pred_camera_vis[b, 0] opt_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already opt_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already opt_vert_vis *= 112 opt_meshes = {'ver': opt_vert_vis, 'f': self.smpl.faces} gt_vert_vis = gt_vertices[b].detach().cpu().numpy() gt_vert_vis *= pred_camera_vis[b, 0] gt_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already gt_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already gt_vert_vis *= 112 gt_meshes = {'ver': gt_vert_vis, 'f': self.smpl.faces} new_opt_output = self.smpl( betas=new_opt_betas, body_pose=new_opt_pose[:, 3:], global_orient=new_opt_pose[:, :3]) new_opt_vertices = new_opt_output.vertices new_opt_joints = new_opt_output.joints new_opt_vert_vis = new_opt_vertices[b].detach().cpu( ).numpy() new_opt_vert_vis *= pred_camera_vis[b, 0] new_opt_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already new_opt_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already new_opt_vert_vis *= 112 new_opt_meshes = { 'ver': new_opt_vert_vis, 'f': self.smpl.faces } glViewer.setMeshData( [new_opt_meshes, gt_meshes, new_opt_meshes], bComputeNormal=True) glViewer.setBackgroundTexture(originalImg) glViewer.setWindowSize(curImgVis.shape[1], curImgVis.shape[0]) glViewer.SetOrthoCamera(True) print(has_smpl[b]) glViewer.show() opt_joint_loss[update] = new_opt_joint_loss[update] opt_vertices[update, :] = new_opt_vertices[update, :] opt_joints[update, :] = new_opt_joints[update, :] opt_pose[update, :] = new_opt_pose[update, :] opt_betas[update, :] = new_opt_betas[update, :] opt_cam_t[update, :] = new_opt_cam_t[update, :] self.fits_dict[(dataset_name, indices.cpu(), rot_angle.cpu(), is_flipped.cpu(), update.cpu())] = (opt_pose.cpu(), opt_betas.cpu()) else: update = torch.zeros(batch_size, device=self.device).byte() # Replace the optimized parameters with the ground truth parameters, if available opt_vertices[has_smpl, :, :] = gt_vertices[has_smpl, :, :] opt_cam_t[has_smpl, :] = gt_cam_t[has_smpl, :] opt_joints[has_smpl, :, :] = gt_model_joints[has_smpl, :, :] opt_pose[has_smpl, :] = gt_pose[has_smpl, :] opt_betas[has_smpl, :] = gt_betas[has_smpl, :] # Assert whether a fit is valid by comparing the joint loss with the threshold valid_fit = (opt_joint_loss < self.options.smplify_threshold).to( self.device) if self.options.ablation_no_pseudoGT: valid_fit[:] = False #Disable all pseudoGT # Add the examples with GT parameters to the list of valid fits valid_fit = valid_fit | has_smpl # if len(valid_fit) > sum(valid_fit): # print(">> Rejected fit: {}/{}".format(len(valid_fit) - sum(valid_fit), len(valid_fit) )) opt_keypoints_2d = perspective_projection( opt_joints, rotation=torch.eye(3, device=self.device).unsqueeze(0).expand( batch_size, -1, -1), translation=opt_cam_t, focal_length=self.focal_length, camera_center=camera_center) opt_keypoints_2d = opt_keypoints_2d / (self.options.img_res / 2.) # Compute loss on SMPL parameters loss_regr_pose, loss_regr_betas = self.smpl_losses( pred_rotmat, pred_betas, opt_pose, opt_betas, valid_fit) # Compute 2D reprojection loss for the keypoints loss_keypoints = self.keypoint_loss(pred_keypoints_2d, gt_keypoints_2d, self.options.openpose_train_weight, self.options.gt_train_weight) # Compute 3D keypoint loss loss_keypoints_3d = self.keypoint_3d_loss(pred_joints, gt_joints, has_pose_3d) # Per-vertex loss for the shape loss_shape = self.shape_loss(pred_vertices, opt_vertices, valid_fit) #Regularization term for shape loss_regr_betas_noReject = torch.mean(pred_betas**2) # Compute total loss # The last component is a loss that forces the network to predict positive depth values if self.options.ablation_loss_2dkeyonly: #2D keypoint only loss = self.options.keypoint_loss_weight * loss_keypoints +\ ((torch.exp(-pred_camera[:,0]*10)) ** 2 ).mean() +\ self.options.beta_loss_weight * loss_regr_betas_noReject #Beta regularization elif self.options.ablation_loss_noSMPLloss: #2D no Pose parameter loss = self.options.keypoint_loss_weight * loss_keypoints +\ self.options.keypoint_loss_weight * loss_keypoints_3d +\ ((torch.exp(-pred_camera[:,0]*10)) ** 2 ).mean() +\ self.options.beta_loss_weight * loss_regr_betas_noReject #Beta regularization else: loss = self.options.shape_loss_weight * loss_shape +\ self.options.keypoint_loss_weight * loss_keypoints +\ self.options.keypoint_loss_weight * loss_keypoints_3d +\ loss_regr_pose + self.options.beta_loss_weight * loss_regr_betas +\ ((torch.exp(-pred_camera[:,0]*10)) ** 2 ).mean() # loss = self.options.keypoint_loss_weight * loss_keypoints #Debug: 2d error only # print("DEBUG: 2donly loss") loss *= 60 # Do backprop self.optimizer.zero_grad() loss.backward() self.optimizer.step() # Pack output arguments for tensorboard logging output = { 'pred_vertices': pred_vertices.detach(), 'opt_vertices': opt_vertices, 'pred_cam_t': pred_cam_t.detach(), 'opt_cam_t': opt_cam_t } losses = { 'loss': loss.detach().item(), 'loss_keypoints': loss_keypoints.detach().item(), 'loss_keypoints_3d': loss_keypoints_3d.detach().item(), 'loss_regr_pose': loss_regr_pose.detach().item(), 'loss_regr_betas': loss_regr_betas.detach().item(), 'loss_shape': loss_shape.detach().item() } if self.options.bDebug_visEFT: #g_debugVisualize: #Debug Visualize input for b in range(batch_size): #denormalizeImg curImgVis = images[b] #3,224,224 curImgVis = self.de_normalize_img(curImgVis).cpu().numpy() curImgVis = np.transpose(curImgVis, (1, 2, 0)) * 255.0 curImgVis = curImgVis[:, :, [2, 1, 0]] #Denormalize image curImgVis = np.ascontiguousarray(curImgVis, dtype=np.uint8) viewer2D.ImShow(curImgVis, name='rawIm') originalImg = curImgVis.copy() # curImgVis = viewer2D.Vis_Skeleton_2D_general(gt_keypoints_2d_orig[b,:,:2].cpu().numpy(), gt_keypoints_2d_orig[b,:,2], bVis= False, image=curImgVis) pred_keypoints_2d_vis = pred_keypoints_2d[ b, :, :2].detach().cpu().numpy() pred_keypoints_2d_vis = 0.5 * self.options.img_res * ( pred_keypoints_2d_vis + 1) #49: (25+24) x 3 curImgVis = viewer2D.Vis_Skeleton_2D_general( pred_keypoints_2d_vis, bVis=False, image=curImgVis) viewer2D.ImShow(curImgVis, scale=2.0, waitTime=1) #Get camera pred_params pred_camera_vis = pred_camera.detach().cpu().numpy() ############### Visualize Mesh ############### pred_vert_vis = pred_vertices[b].detach().cpu().numpy() # meshVertVis = gt_vertices[b].detach().cpu().numpy() # meshVertVis = meshVertVis-pelvis #centering pred_vert_vis *= pred_camera_vis[b, 0] pred_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already pred_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already pred_vert_vis *= 112 pred_meshes = {'ver': pred_vert_vis, 'f': self.smpl.faces} opt_vert_vis = opt_vertices[b].detach().cpu().numpy() opt_vert_vis *= pred_camera_vis[b, 0] opt_vert_vis[:, 0] += pred_camera_vis[ b, 1] #no need +1 (or 112). Rendernig has this offset already opt_vert_vis[:, 1] += pred_camera_vis[ b, 2] #no need +1 (or 112). Rendernig has this offset already opt_vert_vis *= 112 opt_meshes = {'ver': opt_vert_vis, 'f': self.smpl.faces} # glViewer.setMeshData([pred_meshes, opt_meshes], bComputeNormal= True) glViewer.setMeshData([pred_meshes, opt_meshes], bComputeNormal=True) # glViewer.setMeshData([opt_meshes], bComputeNormal= True) ############### Visualize Skeletons ############### #Vis pred-SMPL joint pred_joints_vis = pred_joints[ b, :, :3].detach().cpu().numpy() #[N,49,3] pred_joints_vis = pred_joints_vis.ravel()[:, np.newaxis] #Weak-perspective projection pred_joints_vis *= pred_camera_vis[b, 0] pred_joints_vis[::3] += pred_camera_vis[b, 1] pred_joints_vis[1::3] += pred_camera_vis[b, 2] pred_joints_vis *= 112 #112 == 0.5*224 glViewer.setSkeleton([pred_joints_vis]) # #GT joint gt_jointsVis = gt_joints[b, :, :3].cpu().numpy() #[N,49,3] # gt_pelvis = (gt_smpljointsVis[ 25+2,:] + gt_smpljointsVis[ 25+3,:]) / 2 # gt_smpljointsVis = gt_smpljointsVis- gt_pelvis gt_jointsVis = gt_jointsVis.ravel()[:, np.newaxis] gt_jointsVis *= pred_camera_vis[b, 0] gt_jointsVis[::3] += pred_camera_vis[b, 1] gt_jointsVis[1::3] += pred_camera_vis[b, 2] gt_jointsVis *= 112 glViewer.addSkeleton([gt_jointsVis], jointType='spin') # #Vis SMPL's Skeleton # gt_smpljointsVis = gt_model_joints[b,:,:3].cpu().numpy() #[N,49,3] # # gt_pelvis = (gt_smpljointsVis[ 25+2,:] + gt_smpljointsVis[ 25+3,:]) / 2 # # gt_smpljointsVis = gt_smpljointsVis- gt_pelvis # gt_smpljointsVis = gt_smpljointsVis.ravel()[:,np.newaxis] # gt_smpljointsVis*=pred_camera_vis[b,0] # gt_smpljointsVis[::3] += pred_camera_vis[b,1] # gt_smpljointsVis[1::3] += pred_camera_vis[b,2] # gt_smpljointsVis*=112 # glViewer.addSkeleton( [gt_smpljointsVis]) # #Vis GT joint (not model (SMPL) joint!!) # if has_pose_3d[b]: # gt_jointsVis = gt_model_joints[b,:,:3].cpu().numpy() #[N,49,3] # # gt_jointsVis = gt_joints[b,:,:3].cpu().numpy() #[N,49,3] # # gt_pelvis = (gt_jointsVis[ 25+2,:] + gt_jointsVis[ 25+3,:]) / 2 # # gt_jointsVis = gt_jointsVis- gt_pelvis # gt_jointsVis = gt_jointsVis.ravel()[:,np.newaxis] # gt_jointsVis*=pred_camera_vis[b,0] # gt_jointsVis[::3] += pred_camera_vis[b,1] # gt_jointsVis[1::3] += pred_camera_vis[b,2] # gt_jointsVis*=112 # glViewer.addSkeleton( [gt_jointsVis]) # # glViewer.show() glViewer.setBackgroundTexture(originalImg) glViewer.setWindowSize(curImgVis.shape[1], curImgVis.shape[0]) glViewer.SetOrthoCamera(True) glViewer.show(0) # continue return output, losses
def exportOursToSpin(out_path): scaleFactor = 1.2 hagglingDBdir = '/home/hjoo/data/pytorch_motionSynth/motionsynth_data/data/processed_panoptic' haggling_files = os.listdir(hagglingDBdir + '/panopticDB_pkl_hagglingProcessed') # structs we need imgnames_, scales_, centers_, parts_, openposes_ = [], [], [], [], [] #additional 3D poses_, shapes_, skel3D_, has_smpl_ = [], [], [], [] rhand_2d_list, rhand_3d_list, lhand_2d_list, lhand_3d_list = [], [], [], [] subject_id_list = [] for mocapName in haggling_files: seqname = mocapName[:18] #'170221_haggling_b1' if "170221_haggling_b" not in seqname and "170228_haggling_b" not in seqname: #testing set continue print(f"processing{mocapName}") groupid = mocapName[24:-4] # 3 print(f"{mocapName}, {seqname}, {groupid}") # groupname = f"group{groupid}" # mocapName =f'{seqname}_{groupname}.pkl' imgDir = f'/home/hjoo/data/panoptic-toolbox/{seqname}/hdImgs/00_00' calibdata = f'/home/hjoo/data/panoptic-toolbox/{seqname}/calibration_{seqname}.json' cam = json.load(open(calibdata, "r"))['cameras'] cam = cam[479:] #479 is 00_00 # with open('/run/media/hjoo/disk/data/panoptic_mtc/a4_release/annotation.pkl', 'rb') as f: # data = pickle.load(f) # with open('/run/media/hjoo/disk/data/panoptic_mtc/a4_release/camera_data.pkl', 'rb') as f: # cam = pickle.load(f) bodydatapath = f'{hagglingDBdir}/panopticDB_pkl_hagglingProcessed/' + mocapName bodydata = pickle.load(open(bodydatapath, "rb"), encoding='latin1') handdatapath = f'{hagglingDBdir}/panopticDB_hand_pkl_hagglingProcessed/' + mocapName handdata = pickle.load(open(handdatapath, "rb"), encoding='latin1') # if True: # for subj in bodydata['subjects']: # startFrame = subj['startFrame'] # for n in range(subj['joints19'].shape[1]): # imgpath = os.path.join(imgDir,"00_00_%08d.jpg" % (startFrame+n)) # img = cv2.imread(imgpath) # viewer2D.ImShow(img) groupStartFrame = bodydata['startFrame'] subNum = len(bodydata['subjects']) for subid in range(subNum): #[1]: subj_body = bodydata['subjects'][subid]['joints19'] if len(handdata['hand_right']) <= subid: continue if len(handdata['hand_left']) <= subid: continue subj_righthand = handdata['hand_right'][subid]['hand21'] subj_lefthand = handdata['hand_left'][subid]['hand21'] #Validity Score Right subj_righthand_validity = handdata['hand_right'][subid][ 'bValidFrame'] # subj_righthand_score = handdata['hand_right'][subid]['scores'] localStart = groupStartFrame - handdata['hand_right'][subid][ 'startFrame'] #This subj_righthand_validity = subj_righthand_validity[localStart:] # subj_righthand_score = subj_righthand_score[: , localStart:] #Validity Score Left subj_lefthand_validity = handdata['hand_left'][subid][ 'bValidFrame'] # subj_lefthand_score = handdata['hand_left'][subid]['scores'] localStart = groupStartFrame - handdata['hand_left'][subid][ 'startFrame'] #This subj_lefthand_validity = subj_lefthand_validity[localStart:] # subj_lefthand_score = subj_lefthand_score[:, localStart:] print("valid: {}/{}".format( sum(handdata['hand_right'][subid]['bValidFrame']), len(handdata['hand_right'][subid]['bValidFrame']))) print("valid: {}/{}".format( sum(handdata['hand_left'][subid]['bValidFrame']), len(handdata['hand_left'][subid]['bValidFrame']))) startFrame = bodydata['startFrame'] # assert subj_body.shape[1] == subj_righthand.shape[1] frameLeng = subj_body.shape[1] for ii in range(frameLeng): bVis = False frameid = startFrame + ii subject_id = "{}_g{}_s{}_{:08d}".format( seqname, groupid, subid, frameid) #seqname, groupid. subid, frameid # print(subject_id) imgFullPath = os.path.join(imgDir, "00_00_%08d.jpg" % (frameid)) body_landmark = np.array(subj_body[:, ii]).reshape( -1, 3) #19, 3 #SMC19 order # body_landmark = np.array(sample['body']['landmarks']).reshape(-1, 3) #19, 3 #SMC19 order if subj_righthand.shape[1] <= ii: rhand_landmark = np.zeros( (21, 3)) #21, 3 #SMC19 order else: rhand_landmark = np.array(subj_righthand[:, ii]).reshape( -1, 3) #21, 3 #SMC19 order if subj_lefthand.shape[1] <= ii: lhand_landmark = np.zeros( (21, 3)) #21, 3 #SMC19 order else: lhand_landmark = np.array(subj_lefthand[:, ii]).reshape( -1, 3) #21, 3 #SMC19 order calib_data = cam[0] #00 for c in calib_data: calib_data[c] = np.array(calib_data[c]) skeleton_3d_camview = applyExtrinsic(body_landmark, calib_data) #19,3 rhand_3d_camview = applyExtrinsic(rhand_landmark, calib_data) #21,3 lhand_3d_camview = applyExtrinsic(lhand_landmark, calib_data) #21,3 skeleton_2d = project2D(body_landmark, calib_data) #19,2 rhand_2d = project2D(rhand_landmark, calib_data) #19,2 lhand_2d = project2D(lhand_landmark, calib_data) #19,2 imgName = "{}/hdImgs/00_00/00_00_{:08d}.jpg".format( seqname, frameid) # print(imgName) # imgFullPath = os.path.join(imgDir, imgName) # if os.path.exists(imgFullPath) == False: # continue # print(imgName) #Visulaize 3D if False: img = cv2.imread(imgFullPath) img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) viewer2D.ImShow(img, waitTime=1) skeleton_3d_camview = skeleton_3d_camview.ravel()[:, np. newaxis] rhand_3d_camview = rhand_3d_camview.ravel()[:, np.newaxis] lhand_3d_camview = lhand_3d_camview.ravel()[:, np.newaxis] glViewer.setSkeleton([ skeleton_3d_camview, rhand_3d_camview, lhand_3d_camview ]) # glViewer.setSkeleton([skeleton_3d_camview]) glViewer.setBackgroundTexture(img) glViewer.SetOrthoCamera(True) glViewer.show(0) continue min_pt = np.min(skeleton_2d, axis=0) min_pt[0] = max(min_pt[0], 0) min_pt[1] = max(min_pt[1], 0) max_pt = np.max(skeleton_2d, axis=0) max_pt[0] = min(max_pt[0], 1920) max_pt[1] = min(max_pt[1], 1080) # bbox= [ min_pt[0], min_pt[1], max_pt[0], max_pt[1] ] bbox = [ min_pt[0], min_pt[1], max_pt[0] - min_pt[0], max_pt[1] - min_pt[1] ] center = [bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2] scale = scaleFactor * max(bbox[2], bbox[3]) / 200 #Save data # imgnames_.append(os.path.join('train',fileName_saved)) # imgnames_.append(os.path.join('train',fileName_saved)) imgnames_.append(imgName) openposes_.append(np.zeros([25, 3])) #blank centers_.append(center) scales_.append(scale) # has_smpl_.append(1) # poses_.append(sample['pose3DParam']['pose']) #(72,) # shapes_.append(sample['pose3DParam']['shape']) #(10,) #2D keypoints (total26 -> SPIN24) poseidx_spin24 = [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 12, 19, 20, 22, 21, 23 ] poseidx_smc19 = [ 14, 13, 12, 6, 7, 8, 11, 10, 9, 3, 4, 5, 2, 0, 1, 15, 16, 17, 18 ] part = np.zeros([24, 3]) part[poseidx_spin24, :2] = skeleton_2d[ poseidx_smc19] #(52,) totalGT26 type part[poseidx_spin24, 2] = 1 parts_.append(part) #3D joint S = np.zeros([24, 4]) S[poseidx_spin24, :3] = skeleton_3d_camview[ poseidx_smc19, :] * 0.01 #Scaling skeleton 3D (currently cm) -> meter S[poseidx_spin24, 3] = 1 skel3D_.append(S) rhand_2d_list.append(rhand_2d) if len(subj_righthand_validity) <= ii: rhand_validity = 0 else: rhand_validity = subj_righthand_validity[ii] # rhand_score = subj_righthand_score[:,ii].mean() rhand_3d_camview = rhand_3d_camview * 0.01 if rhand_validity == 1: rhand_3d_camview = np.concatenate( [rhand_3d_camview, np.ones((21, 1))], axis=1) else: rhand_3d_camview = np.concatenate( [rhand_3d_camview, np.zeros((21, 1))], axis=1) bVis = True rhand_3d_list.append(rhand_3d_camview) lhand_2d_list.append(lhand_2d) if len(subj_lefthand_validity) <= ii: lhand_validity = 0 else: lhand_validity = subj_lefthand_validity[ii] # lhand_validity = subj_lefthand_validity[ii] lhand_3d_camview = lhand_3d_camview * 0.01 if lhand_validity == 1: lhand_3d_camview = np.concatenate( [lhand_3d_camview, np.ones((21, 1))], axis=1) else: lhand_3d_camview = np.concatenate( [lhand_3d_camview, np.zeros((21, 1))], axis=1) bVis = True lhand_3d_list.append(lhand_3d_camview) subject_id_list.append(subject_id) #Add hand joints # print("right: {} : left: {}".format(rhand_score, lhand_score)) # print("right: {} : left: {}".format(rhand_validity, lhand_validity)) #Debug 2D Visualize if False: #bVis: img = cv2.imread(imgFullPath) # img = cv2.imread( os.path.join( '/run/media/hjoo/disk/data/mpii_human_pose_v1/images',imgnames_[-1]) ) img = viewer2D.Vis_Skeleton_2D_SMC19(skeleton_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(rhand_2d, image=img) img = viewer2D.Vis_Skeleton_2D_Hand(lhand_2d, image=img) img = viewer2D.Vis_Bbox_minmaxPt(img, min_pt, max_pt) viewer2D.ImShow(img, waitTime=0) #Debug 3D Visualize smpl_coco if False: # data3D_coco_vis = np.reshape(data3D_coco, (data3D_coco.shape[0],-1)).transpose() #(Dim, F) # data3D_coco_vis *=0.1 #mm to cm # glViewer.setSkeleton( [ data3D_coco_vis] ,jointType='smplcoco') # glViewer.show() #Debug 3D Visualize, h36m data3D_h36m_vis = np.reshape( data3D_h36m, (data3D_h36m.shape[0], -1)).transpose() #(Dim, F) data3D_h36m_vis *= 0.001 #meter to cm # data3D_smpl24 = np.reshape(data3D_smpl24, (data3D_smpl24.shape[0],-1)).transpose() #(Dim, F) # data3D_smpl24 *=0.1 glViewer.setSkeleton([data3D_h36m_vis], jointType='smplcoco') glViewer.show() # print("Final Img Num: {}, Final Sample Num: {}".format( len(set(imgnames_) , len(imgnames_)) ) ) print("Final Sample Num: {}".format(len(imgnames_))) # store the data struct if not os.path.isdir(out_path): os.makedirs(out_path) # out_file = os.path.join(out_path, '1031-mpii3D_train_44257_all.npz') out_file = os.path.join(out_path, f'panoptic_{mocapName[:-4]}') print(f"Save to {out_file}") np.savez( out_file, imgname=imgnames_, center=centers_, scale=scales_, part=parts_, openpose=openposes_, S=skel3D_, rhand_3d=rhand_3d_list, rhand_2d=rhand_2d_list, lhand_3d=lhand_3d_list, lhand_2d=lhand_2d_list, subjectid= subject_id_list #To handle sequence data and track the same person in output )