def sample(): """Get samples from a model and visualize them""" actions = data_utils.define_actions( FLAGS.action ) # Load camera parameters SUBJECT_IDS = [1,5,6,7,8,9,11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) # Load 3d data and load (or create) 2d projections train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14 ) if FLAGS.use_sh: train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(actions, FLAGS.data_dir) else: train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data( actions, FLAGS.data_dir, rcams ) print( "done reading and normalizing data." ) device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1} with tf.Session(config=tf.ConfigProto( device_count = device_count )) as sess: # === Create the model === print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.linear_size)) batch_size = 128 model = create_model(sess, actions, batch_size) print("Model loaded") for key2d in test_set_2d.keys(): (subj, b, fname) = key2d print( "Subject: {}, action: {}, fname: {}".format(subj, b, fname) ) # keys should be the same if 3d is in camera coordinates key3d = key2d if FLAGS.camera_frame else (subj, b, '{0}.h5'.format(fname.split('.')[0])) key3d = (subj, b, fname[:-3]) if (fname.endswith('-sh')) and FLAGS.camera_frame else key3d enc_in = test_set_2d[ key2d ] n2d, _ = enc_in.shape dec_out = test_set_3d[ key3d ] n3d, _ = dec_out.shape assert n2d == n3d # Split into about-same-size batches enc_in = np.array_split( enc_in, n2d // batch_size ) dec_out = np.array_split( dec_out, n3d // batch_size ) all_poses_3d = [] for bidx in range( len(enc_in) ): # Dropout probability 0 (keep probability 1) for sampling dp = 1.0 _, _, poses3d = model.step(sess, enc_in[bidx], dec_out[bidx], dp, isTraining=False) # denormalize enc_in[bidx] = data_utils.unNormalizeData( enc_in[bidx], data_mean_2d, data_std_2d, dim_to_ignore_2d ) dec_out[bidx] = data_utils.unNormalizeData( dec_out[bidx], data_mean_3d, data_std_3d, dim_to_ignore_3d ) poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d ) all_poses_3d.append( poses3d ) # Put all the poses together enc_in, dec_out, poses3d = map( np.vstack, [enc_in, dec_out, all_poses_3d] ) # Convert back to world coordinates if FLAGS.camera_frame: N_CAMERAS = 4 N_JOINTS_H36M = 32 # Add global position back dec_out = dec_out + np.tile( test_root_positions[ key3d ], [1,N_JOINTS_H36M] ) # Load the appropriate camera subj, _, sname = key3d cname = sname.split('.')[1] # <-- camera name scams = {(subj,c+1): rcams[(subj,c+1)] for c in range(N_CAMERAS)} # cams of this subject scam_idx = [scams[(subj,c+1)][-1] for c in range(N_CAMERAS)].index( cname ) # index of camera used the_cam = scams[(subj, scam_idx+1)] # <-- the camera used R, T, f, c, k, p, name = the_cam assert name == cname def cam2world_centered(data_3d_camframe): data_3d_worldframe = cameras.camera_to_world_frame(data_3d_camframe.reshape((-1, 3)), R, T) data_3d_worldframe = data_3d_worldframe.reshape((-1, N_JOINTS_H36M*3)) # subtract root translation return data_3d_worldframe - np.tile( data_3d_worldframe[:,:3], (1,N_JOINTS_H36M) ) # Apply inverse rotation and translation dec_out = cam2world_centered(dec_out) poses3d = cam2world_centered(poses3d) # Grab a random batch to visualize enc_in, dec_out, poses3d = map( np.vstack, [enc_in, dec_out, poses3d] ) idx = np.random.permutation( enc_in.shape[0] ) enc_in, dec_out, poses3d = enc_in[idx, :], dec_out[idx, :], poses3d[idx, :] # Visualize random samples import matplotlib.gridspec as gridspec # 1080p = 1,920 x 1,080 fig = plt.figure( figsize=(19.2, 10.8) ) gs1 = gridspec.GridSpec(5, 9) # 5 rows, 9 columns gs1.update(wspace=-0.00, hspace=0.05) # set the spacing between axes. plt.axis('off') subplot_idx, exidx = 1, 1 nsamples = 15 for i in np.arange( nsamples ): # Plot 2d pose ax1 = plt.subplot(gs1[subplot_idx-1]) p2d = enc_in[exidx,:] viz.show2Dpose( p2d, ax1 ) ax1.invert_yaxis() # Plot 3d gt ax2 = plt.subplot(gs1[subplot_idx], projection='3d') p3d = dec_out[exidx,:] viz.show3Dpose( p3d, ax2 ) # Plot 3d predictions ax3 = plt.subplot(gs1[subplot_idx+1], projection='3d') p3d = poses3d[exidx,:] viz.show3Dpose( p3d, ax3, lcolor="#9b59b6", rcolor="#2ecc71" ) exidx = exidx + 1 subplot_idx = subplot_idx + 3 plt.show()
print(dec_out, 'dec_out') # Visualize random samples import matplotlib.gridspec as gridspec # 1080p = 1,920 x 1,080 fig = plt.figure(figsize=(19.2, 10.8)) #先画出图框大小 gs1 = gridspec.GridSpec(1, 2) # 5 rows, 9 columns gs1.update(wspace=0.05, hspace=0.05) # set the spacing between axes. plt.axis('off') subplot_idx, exidx = 1, 1 nsamples = 1 for i in np.arange(nsamples): # Plot 2d pose ax1 = plt.subplot(gs1[subplot_idx - 1]) p2d = enc_final[exidx - 1, :] viz.show2Dpose(p2d, ax1) ax1.invert_yaxis() # Plot 3d gt ax2 = plt.subplot(gs1[subplot_idx], projection='3d') p3d = dec_final[exidx - 1, :] viz.show3Dpose(p3d, ax2) exidx = exidx + 1 subplot_idx = subplot_idx + 3 plt.show()
def main(_): actions_all = data_utils.define_actions( "All" ) # Load camera parameters SUBJECT_IDS = [1,5,6,7,8,9,11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) # Load 3d data and load (or create) 2d projections train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions_all, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14 ) print("Finished Read 3D Data") # _, _, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(actions_all, FLAGS.data_dir) _, _, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data( actions_all, FLAGS.data_dir, rcams ) print("Finished Read 2D Data") SH_TO_GT_PERM = np.array([SH_NAMES.index( h ) for h in H36M_NAMES if h != '' and h in SH_NAMES]) assert np.all( SH_TO_GT_PERM == np.array([6,2,1,0,3,4,5,7,8,9,13,14,15,12,11,10]) ) test_set = {} manipulation_dir = os.path.dirname(FLAGS.data_dir) manipulation_dir = os.path.dirname(manipulation_dir) manipulation_dir += '/manipulation_video/' manipulation_folders = glob.glob(manipulation_dir + '*') subj = 1 action = 'manipulation-video' for folder in manipulation_folders: seqname = os.path.basename( folder ) with h5py.File(folder + '/' + seqname +'.h5', 'r' ) as h5f: poses = h5f['poses'][:] # Permute the loaded data to make it compatible with H36M poses = poses[:,SH_TO_GT_PERM,:] # Reshape into n x (32*2) matrix poses = np.reshape(poses,[poses.shape[0], -1]) poses_final = np.zeros([poses.shape[0], len(H36M_NAMES)*2]) dim_to_use_x = np.where(np.array([x != '' and x != 'Neck/Nose' for x in H36M_NAMES]))[0] * 2 dim_to_use_y = dim_to_use_x+1 dim_to_use = np.zeros(len(SH_NAMES)*2,dtype=np.int32) dim_to_use[0::2] = dim_to_use_x dim_to_use[1::2] = dim_to_use_y poses_final[:,dim_to_use] = poses # If Uniframe Model Only keep the right 2d pose poses_final = poses_final[1::2,:] print(seqname, poses_final.shape) test_set[ (subj, action, seqname) ] = poses_final print("Finished Read Manipulations Videos") test_set_2d = data_utils.normalize_data( test_set, data_mean_2d, data_std_2d, dim_to_use_2d ) dim_to_use_12_manipulation_joints = np.array([3,4,5, 6,7,8, 9,10,11, 18,19,20, 21,22,23, 24,25,26, 51,52,53, 54,55,56, 57,58,59, 75,76,77, 78,79,80, 81,82,83]) print("Finished Normalize Manipualtion Videos") device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1} with tf.Session(config=tf.ConfigProto( device_count = device_count )) as sess: # === Create the model === print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.linear_size)) batch_size = FLAGS.batch_size #Intial code is 64*2 model = predict_3dpose.create_model(sess, actions_all, batch_size) print("Model loaded") j = 0 for key2d in test_set_2d.keys(): (subj, b, fname) = key2d # if fname != specific_seqname + '.h5': # continue print( "Subject: {}, action: {}, fname: {}".format(subj, b, fname) ) enc_in = test_set_2d[ key2d ] n2d, _ = enc_in.shape # Split into about-same-size batches enc_in = np.array_split( enc_in, n2d // 1 ) all_poses_3d = [] for bidx in range( len(enc_in) ): # Dropout probability 0 (keep probability 1) for sampling dp = 1.0 anything = np.zeros((enc_in[bidx].shape[0], 48)) _, _, poses3d = model.step(sess, enc_in[bidx], anything, dp, isTraining=False) # Denormalize enc_in[bidx] = data_utils.unNormalizeData( enc_in[bidx], data_mean_2d, data_std_2d, dim_to_ignore_2d ) poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d ) all_poses_3d.append( poses3d ) # Put all the poses together enc_in, poses3d = map( np.vstack, [enc_in, all_poses_3d] ) enc_in, poses3d = map( np.vstack, [enc_in, poses3d] ) poses3d_12_manipulation = poses3d[:, dim_to_use_12_manipulation_joints] annotated_images = glob.glob(manipulation_dir + fname + '/info/*.xml') annotated_images = sorted(annotated_images) # 1080p = 1,920 x 1,080 fig = plt.figure(j, figsize=(10, 10)) gs1 = gridspec.GridSpec(3, 3) gs1.update(wspace=-0, hspace=0.1) # set the spacing between axes. plt.axis('off') subplot_idx = 1 nsamples = 3 for i in np.arange( nsamples ): # Plot 2d Detection ax1 = plt.subplot(gs1[subplot_idx-1]) img = mpimg.imread(manipulation_dir + fname + '/skeleton_cropped/' + os.path.basename(annotated_images[i]).split('_')[0] + '.jpg') ax1.imshow(img) # Plot 2d pose ax2 = plt.subplot(gs1[subplot_idx]) p2d = enc_in[i,:] viz.show2Dpose( p2d, ax2 ) ax2.invert_yaxis() # Plot 3d predictions # Compute first the procrustion and print error gt = getJ3dPosFromXML(annotated_images[i]) A = poses3d_12_manipulation[i, :].reshape(gt.shape) _, Z, T, b, c = procrustes.compute_similarity_transform(gt, A, compute_optimal_scale=True) sqerr = np.sqrt(np.sum((gt - (b*A.dot(T)) - c)**2, axis = 1)) print("{0} - {1} - Mean Error (mm) : {2}".format(fname, os.path.basename(annotated_images[i]), np.mean(sqerr))) ax3 = plt.subplot(gs1[subplot_idx+1], projection='3d') temp = poses3d[i,:].reshape((32, 3)) temp = c + temp.dot(T) #Do not scale p3d = temp.reshape((1, 96)) viz.show3Dpose( p3d, ax3, lcolor="#9b59b6", rcolor="#2ecc71" ) ax3.invert_zaxis() ax3.invert_yaxis() subplot_idx = subplot_idx + 3 plt.show() j += 1
os.path.join('./data/jsonAlpha_one/', '{}_keypoints.json').format(str(frame).zfill(12)), 'w') as outfile: json.dump(people, outfile) if __name__ == '__main__': # org_path = r"/home/ubuntu/gaoyu/alphapose/Video3D3_cmu/multi_person/multi2/sep-json/" # save_path = r"./data/jsonAlpha_multi/" org_path = r"/home/ubuntu/gaoyu/alphapose/Video3D3_cmu/sep-json/" save_path = r"./data/jsonAlpha_one/" filelist = strsort(os.listdir(org_path)) print(len(filelist)) for i in range(len(filelist)): frame = filelist[i].split('.')[0] with open(os.path.join(org_path, filelist[i]), encoding='utf8') as fp: json_data = json.load(fp) cmu2human36m(json_data, frame) ''' #save as another json with open(os.path.join('./data/json/','{}_keypoints.json').format(str(2).zfill(12)),'w') as outfile: json.dump(people,outfile) fig = plt.figure(figsize=(9.6, 5.4))#1920:1080 ax = plt.subplot(111) viz.show2Dpose(np.array(human36['pose_keypoints_2d']), ax) plt.show() '''