def read_mpi(data_path, do_transform, H36M_mean2d, H36M_mean3d): #3dhp to H36M M = np.array([10, 8, 11, 12, 13, 14, 15, 16, 4, 5, 6, 1, 2, 3, 0, 7, 9]) N = np.array([9, 8, 13, 14, 15, 12, 11, 10, 3, 4, 5, 2, 1, 0, 6, 7, 16]) # Loads 3dhp data inputs = h5py.File(data_path, 'r') #According to doc, this should be normalized but isn't #Shape (2929, 17, 3) test_set3d = inputs['univ_annot3'] test_set3d = test_set3d[:][:, M, :] # test_set3d = test_set3d[:][:, N, :][:, :-1, :] #Get rid of the first joint, subtract from rest first_joint3d = np.expand_dims(test_set3d[:, 0, :], axis=1) test_set3d = (test_set3d - first_joint3d)[:, 1:, :] #Calculate 3d statistics data_mean3d = np.mean(test_set3d, axis=0) data_std3d = np.std(test_set3d, axis=0) #If transform points, do procrustes transform if do_transform: _, Z, T, b, c = procrustes.compute_similarity_transform( H36M_mean3d.reshape(-1, 3).T, data_mean3d.T, compute_optimal_scale=True) test_set3d = np.transpose( (b * np.transpose(test_set3d, (0, 2, 1)).dot(T)) + c, (0, 2, 1)) data_mean3d = np.mean(test_set3d, axis=0) data_std3d = np.std(test_set3d, axis=0) data_test3d = np.divide(test_set3d - data_mean3d, data_std3d) #Shape (2929, 17, 2) test_set2d = inputs['annot_2d'] # N = np.array([9,8,13,14,15,12,11,10,3,4,5,2,1,0,6,7,16]) # test_set2d = test_set2d[:][:, N, :] #Get rid of the first joint test_set2d = test_set2d[:, 1:, :] / 2.048 #Calculate 2d statistic data_mean2d = np.mean(test_set2d, axis=0) data_std2d = np.std(test_set2d, axis=0) #If transform points, do procrustes transform # if do_transform: # _, Z, T, b, c = procrustes.compute_similarity_transform(H36M_mean2d.reshape(-1, 2), data_mean2d, compute_optimal_scale=True) # test_set2d = (b * test_set2d.dot(T)) + c # data_mean2d = np.mean(test_set2d, axis=0) # data_std2d = np.std(test_set2d, axis=0) data_test2d = np.divide(test_set2d - data_mean2d, data_std2d) return data_test3d, data_mean3d, data_std3d, data_test2d, data_mean2d, data_std2d
def compute_mpjpe_summary(gt_pose, pred_pose): gt_pose = normalize_3d_pose(gt_pose) pred_pose = normalize_3d_pose(pred_pose) pa_pose = pred_pose.copy() try: if opts['procustes']: for i in range(opts['batch_size']): gt = gt_pose[i] out = pred_pose[i] _, Z, T, b, c = procrustes.compute_similarity_transform( gt, out, compute_optimal_scale=True) out = (b * out.dot(T)) + c pa_pose[i, :, :] = out except: #in case of an error, dump the pose batch and exit sio.savemat("poses.mat", {"gt": gt_pose, "pred": pred_pose}) print("error in mpjpe") exit() return np.mean(np.sqrt(np.sum((pa_pose - gt_pose)**2, axis=2)))
def evaluate_batches( sess, model, data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d, data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d, current_step, encoder_inputs, decoder_outputs, current_epoch=0 ): """ Generic method that evaluates performance of a list of batches. May be used to evaluate all actions or a single action. Args sess model data_mean_3d data_std_3d dim_to_use_3d dim_to_ignore_3d data_mean_2d data_std_2d dim_to_use_2d dim_to_ignore_2d current_step encoder_inputs decoder_outputs current_epoch Returns total_err joint_err step_time loss """ n_joints = 17 if not(FLAGS.predict_14) else 14 nbatches = len( encoder_inputs ) # Loop through test examples all_dists, start_time, loss = [], time.time(), 0. log_every_n_batches = 100 for i in range(nbatches): if current_epoch > 0 and (i+1) % log_every_n_batches == 0: print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i+1, nbatches) ) enc_in, dec_out = encoder_inputs[i], decoder_outputs[i] dp = 1.0 # dropout keep probability is always 1 at test time step_loss, loss_summary, poses3d = model.step( sess, enc_in, dec_out, dp, isTraining=False ) loss += step_loss # denormalize enc_in = data_utils.unNormalizeData( enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d ) dec_out = data_utils.unNormalizeData( dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d ) poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d ) # Keep only the relevant dimensions dtu3d = np.hstack( (np.arange(3), dim_to_use_3d) ) if not(FLAGS.predict_14) else dim_to_use_3d dec_out = dec_out[:, dtu3d] poses3d = poses3d[:, dtu3d] assert dec_out.shape[0] == FLAGS.batch_size assert poses3d.shape[0] == FLAGS.batch_size if FLAGS.procrustes: # Apply per-frame procrustes alignment if asked to do so for j in range(FLAGS.batch_size): gt = np.reshape(dec_out[j,:],[-1,3]) out = np.reshape(poses3d[j,:],[-1,3]) _, Z, T, b, c = procrustes.compute_similarity_transform(gt,out) out = out.dot(T)+c poses3d[j,:] = np.reshape(out,[-1,17*3] ) if not(FLAGS.predict_14) else np.reshape(out,[-1,14*3] ) # Compute Euclidean distance error per joint sqerr = (poses3d - dec_out)**2 # Squared error between prediction and expected output dists = np.zeros( (sqerr.shape[0], njoints) ) # Array with L2 error per joint in mm dist_idx = 0 for k in np.arange(0, n_joints*3, 3): # Sum across X,Y, and Z dimenstions to obtain L2 distance dists[:,dist_idx] = np.sqrt( np.sum( sqerr[:, k:k+3], axis=1 )) dist_idx = dist_idx + 1 all_dists.append(dists) assert sqerr.shape[0] == FLAGS.batch_size step_time = (time.time() - start_time) / nbatches loss = loss / nbatches all_dists = np.vstack( all_dists ) # Error per joint and total for all passed batches joint_err = np.mean( all_dists, axis=0 ) total_err = np.mean( all_dists ) return total_err, joint_err, step_time, loss
def main(_): actions_all = data_utils.define_actions("All") actions = data_utils.define_actions("Discussion") # Load camera parameters SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) # Load 3d data and load (or create) 2d projections train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14) train_set_3d = data_utils.remove_first_frame(train_set_3d) test_set_3d = data_utils.remove_first_frame(test_set_3d) train_root_positions = data_utils.remove_first_frame(train_root_positions) test_root_positions = data_utils.remove_first_frame(test_root_positions) print("Finished Read 3D Data") # train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(actions_all, FLAGS.data_dir) # train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.transform_to_2d_biframe_prediction(train_set_2d, # test_set_2d, # data_mean_2d, # data_std_2d, # dim_to_ignore_2d, # dim_to_use_2d) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data( actions_all, FLAGS.data_dir, rcams) train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.transform_to_2d_biframe_prediction( train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d) SH_TO_GT_PERM = np.array( [SH_NAMES.index(h) for h in H36M_NAMES if h != '' and h in SH_NAMES]) assert np.all(SH_TO_GT_PERM == np.array( [6, 2, 1, 0, 3, 4, 5, 7, 8, 9, 13, 14, 15, 12, 11, 10])) test_set = {} manipulation_dir = os.path.dirname(FLAGS.data_dir) manipulation_dir = os.path.dirname(manipulation_dir) manipulation_dir += '/manipulation_video/' manipulation_folders = glob.glob(manipulation_dir + '*') subj = 1 action = 'manipulation-video' for folder in manipulation_folders: seqname = os.path.basename(folder) with h5py.File(folder + '/' + seqname + '.h5', 'r') as h5f: poses = h5f['poses'][:] # Permute the loaded data to make it compatible with H36M poses = poses[:, SH_TO_GT_PERM, :] # Reshape into n x (32*2) matrix poses = np.reshape(poses, [poses.shape[0], -1]) poses_final = np.zeros([poses.shape[0], len(H36M_NAMES) * 2]) dim_to_use_x = np.where( np.array([x != '' and x != 'Neck/Nose' for x in H36M_NAMES]))[0] * 2 dim_to_use_y = dim_to_use_x + 1 dim_to_use = np.zeros(len(SH_NAMES) * 2, dtype=np.int32) dim_to_use[0::2] = dim_to_use_x dim_to_use[1::2] = dim_to_use_y poses_final[:, dim_to_use] = poses print(seqname, poses_final.shape) poses_final[poses_final == 0.] = 0.1 test_set[(subj, action, seqname)] = poses_final test_set = data_utils.uni_frame_to_bi_frame(test_set) test_set_2d = data_utils.normalize_data(test_set, data_mean_2d, data_std_2d, dim_to_use_2d) for key in test_set.keys(): test_set[key] = test_set[key][0::2, :] dim_to_use_12_manipulation_joints = np.array([ 3, 4, 5, 6, 7, 8, 9, 10, 11, 18, 19, 20, 21, 22, 23, 24, 25, 26, 51, 52, 53, 54, 55, 56, 57, 58, 59, 75, 76, 77, 78, 79, 80, 81, 82, 83 ]) print("Finished Normalize Manipualtion Videos") device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1} with tf.Session(config=tf.ConfigProto(device_count=device_count)) as sess: # === Create the model === print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.linear_size)) batch_size = FLAGS.batch_size #Intial code is 64*2 model = predict_3dpose_biframe.create_model(sess, actions_all, batch_size) print("Model loaded") j = 0 for key2d in test_set_2d.keys(): (subj, b, fname) = key2d # if fname != specific_seqname + '.h5': # continue print("Subject: {}, action: {}, fname: {}".format(subj, b, fname)) enc_in = test_set_2d[key2d] n2d, _ = enc_in.shape # Split into about-same-size batches enc_in = np.array_split(enc_in, n2d // 1) all_poses_3d = [] for bidx in range(len(enc_in)): # Dropout probability 0 (keep probability 1) for sampling dp = 1.0 anything = np.zeros((enc_in[bidx].shape[0], 48)) _, _, poses3d = model.step(sess, enc_in[bidx], anything, dp, isTraining=False) # Denormalize enc_in[bidx] = data_utils.unNormalizeData( enc_in[bidx], data_mean_2d, data_std_2d, dim_to_ignore_2d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) all_poses_3d.append(poses3d) # Put all the poses together enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d]) enc_in, poses3d = map(np.vstack, [enc_in, poses3d]) poses3d_12_manipulation = poses3d[:, dim_to_use_12_manipulation_joints] annotated_images = glob.glob(manipulation_dir + fname + '/info/*.xml') annotated_images = sorted(annotated_images) # 1080p = 1,920 x 1,080 fig = plt.figure(j, figsize=(10, 10)) gs1 = gridspec.GridSpec(3, 3) gs1.update(wspace=-0, hspace=0.1) # set the spacing between axes. plt.axis('off') subplot_idx = 1 nsamples = 3 for i in np.arange(nsamples): # Plot 2d Detection ax1 = plt.subplot(gs1[subplot_idx - 1]) img = mpimg.imread( manipulation_dir + fname + '/skeleton_cropped/' + os.path.basename(annotated_images[i]).split('_')[0] + '.jpg') ax1.imshow(img) # Plot 2d pose ax2 = plt.subplot(gs1[subplot_idx]) # p2d = enc_in[i,:] # viz.show2Dpose( p2d, ax2 ) # ax2.invert_yaxis() ax2.imshow(img) # Plot 3d predictions # Compute first the procrustion and print error gt = getJ3dPosFromXML(annotated_images[i]) A = poses3d_12_manipulation[i, :].reshape(gt.shape) _, Z, T, b, c = procrustes.compute_similarity_transform( gt, A, compute_optimal_scale=True) sqerr = np.sqrt(np.sum((gt - (b * A.dot(T)) - c)**2, axis=1)) print("{0} - {1} - Mean Error (mm) : {2}".format( fname, os.path.basename(annotated_images[i]), np.mean(sqerr))) ax3 = plt.subplot(gs1[subplot_idx + 1], projection='3d') temp = poses3d[i, :].reshape((32, 3)) temp = c + temp.dot(T) #Do not scale # p3d = temp.reshape((1, 96)) p3d = poses3d[i, :] viz.show3Dpose(p3d, ax3, lcolor="#9b59b6", rcolor="#2ecc71") ax3.invert_zaxis() ax3.invert_yaxis() subplot_idx = subplot_idx + 3 plt.show() j += 1
def evaluate_batches(sess, model, data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d, current_step, encoder_inputs, decoder_outputs, current_epoch=0): n_joints = 17 nbatches = len(encoder_inputs) all_dists, start_time, loss = [], time.time(), 0. log_every_n_batches = 100 for i in range(nbatches): if current_epoch > 0 and (i + 1) % log_every_n_batches == 0: print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i + 1, nbatches)) enc_in, dec_out = encoder_inputs[i], decoder_outputs[i] dp = 1.0 # dropout keep probability is always 1 at test time step_loss, loss_summary, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) loss += step_loss if (i == 0): dec_out = np.vstack( [dec_out[0, :, :], dec_out[1:, FLAGS.seqlen - 1, :]]) poses3d = np.vstack( [poses3d[0, :, :], poses3d[1:, FLAGS.seqlen - 1, :]]) else: dec_out = np.expand_dims(dec_out[:, FLAGS.seqlen - 1, :], axis=0) poses3d = np.expand_dims(poses3d[:, FLAGS.seqlen - 1, :], axis=0) dec_out = np.reshape(dec_out, [-1, (n_joints - 1) * 3]) poses3d = np.reshape(poses3d, [-1, (n_joints - 1) * 3]) dec_out = data_utils.unNormalizeData(dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d) poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d) dtu3d = np.hstack((np.arange(3), dim_to_use_3d)) dec_out = dec_out[:, dtu3d] poses3d = poses3d[:, dtu3d] if FLAGS.procrustes: # Apply per-frame procrustes alignment for j in range(poses3d.shape[0]): gt = np.reshape(dec_out[j, :], [-1, 3]) out = np.reshape(poses3d[j, :], [-1, 3]) _, Z, T, b, c = procrustes.compute_similarity_transform( gt, out, compute_optimal_scale=True) out = Z poses3d[j, :] = np.reshape(out, [-1, 17 * 3]) # Compute Euclidean distance error per joint sqerr = ( poses3d - dec_out)**2 # Squared error between prediction and expected output dists = np.zeros( (sqerr.shape[0], n_joints)) # Array with L2 error per joint in mm dist_idx = 0 for k in np.arange(0, n_joints * 3, 3): # Sum across X,Y, and Z dimenstions to obtain L2 distance dists[:, dist_idx] = np.sqrt(np.sum(sqerr[:, k:k + 3], axis=1)) dist_idx = dist_idx + 1 all_dists.append(dists) step_time = (time.time() - start_time) / nbatches loss = loss / nbatches all_dists = np.vstack(all_dists) # Error per joint and total for all passed batches joint_err = np.mean(all_dists, axis=0) total_err = np.mean(all_dists) return total_err, joint_err, step_time, loss
def evaluate_batches(sess, model, data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d, data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d, current_step, encoder_inputs, decoder_outputs, current_epoch=0): """ Generic method that evaluates performance of a list of batches. May be used to evaluate all actions or a single action. Args sess model data_mean_3d data_std_3d dim_to_use_3d dim_to_ignore_3d data_mean_2d data_std_2d dim_to_use_2d dim_to_ignore_2d current_step encoder_inputs decoder_outputs current_epoch Returns total_err joint_err step_time loss """ n_joints = 17 if not (FLAGS.predict_14) else 14 nbatches = len(encoder_inputs) # Loop through test examples all_dists, start_time, loss = [], time.time(), 0. log_every_n_batches = 100 all_poses_3d = [] all_enc_in = [] for i in range(nbatches): if current_epoch > 0 and (i + 1) % log_every_n_batches == 0: print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i + 1, nbatches)) enc_in, dec_out = encoder_inputs[i], decoder_outputs[i] # enc_in = data_utils.generage_missing_data(enc_in, FLAGS.miss_num) dp = 1.0 # dropout keep probability is always 1 at test time step_loss, loss_summary, out_all_components_ori = model.step( sess, enc_in, dec_out, dp, isTraining=False) loss += step_loss out_all_components = np.reshape( out_all_components_ori, [-1, model.HUMAN_3D_SIZE + 2, model.num_models]) out_mean = out_all_components[:, :model.HUMAN_3D_SIZE, :] # denormalize enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d) enc_in_ = copy.deepcopy(enc_in) all_enc_in.append(enc_in_) dec_out = data_utils.unNormalizeData(dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d) pose_3d = np.zeros((enc_in.shape[0], 96, out_mean.shape[-1])) for j in range(out_mean.shape[-1]): pose_3d[:, :, j] = data_utils.unNormalizeData(out_mean[:, :, j], data_mean_3d, data_std_3d, dim_to_ignore_3d) pose_3d_ = copy.deepcopy(pose_3d) all_poses_3d.append(pose_3d_) # Keep only the relevant dimensions dtu3d = np.hstack( (np.arange(3), dim_to_use_3d)) if not (FLAGS.predict_14) else dim_to_use_3d dec_out = dec_out[:, dtu3d] pose_3d = pose_3d[:, dtu3d, :] assert dec_out.shape[0] == FLAGS.batch_size assert pose_3d.shape[0] == FLAGS.batch_size if FLAGS.procrustes: # Apply per-frame procrustes alignment if asked to do so for j in range(FLAGS.batch_size): for k in range(model.num_models): gt = np.reshape(dec_out[j, :], [-1, 3]) out = np.reshape(pose_3d[j, :, k], [-1, 3]) _, Z, T, b, c = procrustes.compute_similarity_transform( gt, out, compute_optimal_scale=True) out = (b * out.dot(T)) + c pose_3d[j, :, k] = np.reshape( out, [-1, 17 * 3]) if not (FLAGS.predict_14) else np.reshape( pose_3d[j, :, k], [-1, 14 * 3]) # Compute Euclidean distance error per joint sqerr = (pose_3d - np.expand_dims(dec_out, axis=2) )**2 # Squared error between prediction and expected output dists = np.zeros( (sqerr.shape[0], n_joints, sqerr.shape[2])) # Array with L2 error per joint in mm for m in range(dists.shape[-1]): dist_idx = 0 for k in np.arange(0, n_joints * 3, 3): # Sum across X,Y, and Z dimenstions to obtain L2 distance dists[:, dist_idx, m] = np.sqrt(np.sum(sqerr[:, k:k + 3, m], axis=1)) dist_idx = dist_idx + 1 all_dists.append(dists) assert sqerr.shape[0] == FLAGS.batch_size step_time = (time.time() - start_time) / nbatches loss = loss / nbatches all_dists = np.vstack(all_dists) aver_minerr = np.mean(np.min(np.sum(all_dists, axis=1), axis=1)) / n_joints return aver_minerr, step_time, loss
def evaluate_batches( sess, model, data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d, data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d, current_step, encoder_inputs, decoder_outputs, current_epoch=0 ): """ Generic method that evaluates performance of a list of batches. May be used to evaluate all actions or a single action. Args sess model data_mean_3d data_std_3d dim_to_use_3d dim_to_ignore_3d data_mean_2d data_std_2d dim_to_use_2d dim_to_ignore_2d current_step encoder_inputs decoder_outputs current_epoch Returns total_err joint_err step_time loss """ n_joints = 17 if not(FLAGS.predict_14) else 14 nbatches = len( encoder_inputs ) # Loop through test examples all_dists, start_time, loss = [], time.time(), 0. log_every_n_batches = 100 for i in range(nbatches): if current_epoch > 0 and (i+1) % log_every_n_batches == 0: print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i+1, nbatches) ) enc_in, dec_out = encoder_inputs[i], decoder_outputs[i] dp = 1.0 # dropout keep probability is always 1 at test time step_loss, loss_summary, poses3d = model.step( sess, enc_in, dec_out, dp, isTraining=False ) loss += step_loss # denormalize enc_in = data_utils.unNormalizeData( enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d ) dec_out = data_utils.unNormalizeData( dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d ) poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d ) # Keep only the relevant dimensions dtu3d = np.hstack( (np.arange(3), dim_to_use_3d) ) if not(FLAGS.predict_14) else dim_to_use_3d dec_out = dec_out[:, dtu3d] poses3d = poses3d[:, dtu3d] assert dec_out.shape[0] == FLAGS.batch_size assert poses3d.shape[0] == FLAGS.batch_size if FLAGS.procrustes: # Apply per-frame procrustes alignment if asked to do so for j in range(FLAGS.batch_size): gt = np.reshape(dec_out[j,:],[-1,3]) out = np.reshape(poses3d[j,:],[-1,3]) _, Z, T, b, c = procrustes.compute_similarity_transform(gt,out,compute_optimal_scale=True) out = (b*out.dot(T))+c poses3d[j,:] = np.reshape(out,[-1,17*3] ) if not(FLAGS.predict_14) else np.reshape(out,[-1,14*3] ) # Compute Euclidean distance error per joint sqerr = (poses3d - dec_out)**2 # Squared error between prediction and expected output dists = np.zeros( (sqerr.shape[0], n_joints) ) # Array with L2 error per joint in mm dist_idx = 0 for k in np.arange(0, n_joints*3, 3): # Sum across X,Y, and Z dimenstions to obtain L2 distance dists[:,dist_idx] = np.sqrt( np.sum( sqerr[:, k:k+3], axis=1 )) dist_idx = dist_idx + 1 all_dists.append(dists) assert sqerr.shape[0] == FLAGS.batch_size step_time = (time.time() - start_time) / nbatches loss = loss / nbatches all_dists = np.vstack( all_dists ) # Error per joint and total for all passed batches joint_err = np.mean( all_dists, axis=0 ) total_err = np.mean( all_dists ) return total_err, joint_err, step_time, loss
def evaluate_batches( sess, model, data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d, data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d, current_step, encoder_inputs, decoder_outputs, current_epoch=0, birdNames = False ): """ Generic method that evaluates performance of a list of batches. May be used to evaluate all actions or a single action. Args sess: tensorflow session model: tensorflow model to run evaluation with data_mean_3d: the mean of the training data in 3d data_std_3d: the standard deviation of the training data in 3d dim_to_use_3d: out of all the 96 dimensions that represent a 3d body in h36m, compute results for this subset dim_to_ignore_3d: complelment of the above data_mean_2d: mean of the training data in 2d data_std_2d: standard deviation of the training data in 2d dim_to_use_2d: out of the 64 dimensions that represent a body in 2d in h35m, use this subset dim_to_ignore_2d: complement of the above current_step: training iteration step encoder_inputs: input for the network decoder_outputs: expected output for the network current_epoch: current training epoch Returns total_err: average mm error over all joints joint_err: average mm error per joint step_time: time it took to evaluate one batch loss: validation loss of the network """ if birdNames == False: n_joints = 17 if not(FLAGS.predict_14) else 14 else: n_joints = 7 nbatches = len( encoder_inputs ) # Loop through test examples all_dists, start_time, loss = [], time.time(), 0. log_every_n_batches = 100 for i in range(nbatches): if current_epoch > 0 and (i+1) % log_every_n_batches == 0: print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i+1, nbatches) ) enc_in, dec_out = encoder_inputs[i], decoder_outputs[i] dp = 1.0 # dropout keep probability is always 1 at test time step_loss, loss_summary, poses3d = model.step( sess, enc_in, dec_out, dp, isTraining=False ) loss += step_loss # denormalize enc_in = data_utils.unNormalizeData( enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d ) dec_out = data_utils.unNormalizeData( dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d ) poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d ) # Keep only the relevant dimensions dtu3d = np.hstack( (np.arange(3), dim_to_use_3d) ) if not(FLAGS.predict_14) else dim_to_use_3d dec_out = dec_out[:, dtu3d] poses3d = poses3d[:, dtu3d] assert dec_out.shape[0] == FLAGS.batch_size assert poses3d.shape[0] == FLAGS.batch_size if FLAGS.procrustes: # Apply per-frame procrustes alignment if asked to do so for j in range(FLAGS.batch_size): gt = np.reshape(dec_out[j,:],[-1,3]) out = np.reshape(poses3d[j,:],[-1,3]) _, Z, T, b, c = procrustes.compute_similarity_transform(gt,out,compute_optimal_scale=True) out = (b*out.dot(T))+c poses3d[j,:] = np.reshape(out,[-1,17*3] ) if not(FLAGS.predict_14) else np.reshape(out,[-1,14*3] ) # Compute Euclidean distance error per joint sqerr = (poses3d - dec_out)**2 # Squared error between prediction and expected output dists = np.zeros( (sqerr.shape[0], n_joints) ) # Array with L2 error per joint in mm dist_idx = 0 for k in np.arange(0, n_joints*3, 3): # Sum across X,Y, and Z dimenstions to obtain L2 distance dists[:,dist_idx] = np.sqrt( np.sum( sqerr[:, k:k+3], axis=1 )) dist_idx = dist_idx + 1 all_dists.append(dists) assert sqerr.shape[0] == FLAGS.batch_size step_time = (time.time() - start_time) / nbatches loss = loss / nbatches all_dists = np.vstack( all_dists ) # Error per joint and total for all passed batches joint_err = np.mean( all_dists, axis=0 ) total_err = np.mean( all_dists ) return total_err, joint_err, step_time, loss
def mpi(): actions = data_utils.define_actions(FLAGS.action) number_of_actions = len(actions) # Load camera parameters SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11] rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS) # Load 3d data and load (or create) 2d projections train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data( actions, FLAGS.data_dir + 'train_images.txt', FLAGS.data_dir + 'valid_images.txt', FLAGS.data_dir + 'train.h5', FLAGS.data_dir + 'valid.h5', FLAGS.camera_frame, rcams, FLAGS.predict_14) # Read stacked hourglass 2D predictions if use_sh, otherwise use groundtruth 2D projections if FLAGS.use_sh: train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions( actions, FLAGS.data_dir + 'train_images.txt', FLAGS.data_dir + 'valid_images.txt', FLAGS.data_dir + 'train.h5', FLAGS.data_dir + 'valid.h5') else: train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data( actions, FLAGS.data_dir + 'train_images.txt', FLAGS.data_dir + 'valid_images.txt', FLAGS.data_dir + 'train.h5', FLAGS.data_dir + 'valid.h5', rcams) print("done reading and normalizing data.") # Load MPI data for testing: mpi_test3d, mpi_mean3d, mpi_std3d, mpi_test2d, mpi_mean2d, mpi_std2d = data_utils.read_mpi( '/mnt/lustre/xingyifei/test_3dhp/annotTest.h5', True, data_mean_2d, data_mean_3d) # Avoid using the GPU if requested device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1} with tf.Session(config=tf.ConfigProto(device_count=device_count, allow_soft_placement=True)) as sess: # === Create the model === print("Creating %d bi-layers of %d units." % (FLAGS.num_layers, FLAGS.linear_size)) model = create_model(sess, actions, FLAGS.batch_size) model.train_writer.add_graph(sess.graph) print("Model created") #=== This is the training loop === step_time, loss, val_loss = 0.0, 0.0, 0.0 current_step = 0 if FLAGS.load <= 0 else FLAGS.load + 1 previous_losses = [] step_time, loss = 0, 0 current_epoch = 0 log_every_n_batches = 100 # === Testing after this epoch === isTraining = False n_joints = 16 if not (FLAGS.predict_14) else 14 #Process inputs to batches n = len(mpi_test3d) n_extra = n % model.batch_size if n_extra > 0: # Otherwise examples are already a multiple of batch size encoder_inputs = mpi_test2d[:-n_extra, :, :].reshape(-1, 32) decoder_outputs = mpi_test3d[:-n_extra, :, :].reshape(-1, 48) n_batches = n // model.batch_size encoder_inputs = np.split(encoder_inputs, n_batches) decoder_outputs = np.split(decoder_outputs, n_batches) nbatches = len(encoder_inputs) # Loop through test examples all_dists, start_time, loss = [], time.time(), 0. log_every_n_batches = 100 for i in range(nbatches): if current_epoch > 0 and (i + 1) % log_every_n_batches == 0: print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i + 1, nbatches)) enc_in, dec_out = encoder_inputs[i], decoder_outputs[i] dp = 1.0 # dropout keep probability is always 1 at test time step_loss, loss_summary, poses3d = model.step(sess, enc_in, dec_out, dp, isTraining=False) loss += step_loss # denormalize enc_in = (enc_in.reshape(-1, 16, 2) * mpi_std2d + mpi_mean2d).reshape(-1, 32) dec_out = (dec_out.reshape(-1, 16, 3) * mpi_std3d + mpi_mean3d).reshape(-1, 48) poses3d = (poses3d.reshape(-1, 16, 3) * mpi_std3d + mpi_mean3d).reshape(-1, 48) assert dec_out.shape[0] == FLAGS.batch_size assert poses3d.shape[0] == FLAGS.batch_size if FLAGS.procrustes: # Apply per-frame procrustes alignment if asked to do so for j in range(FLAGS.batch_size): gt = np.reshape(dec_out[j, :], [-1, 3]) out = np.reshape(poses3d[j, :], [-1, 3]) _, Z, T, b, c = procrustes.compute_similarity_transform( gt, out, compute_optimal_scale=True) out = (b * out.dot(T)) + c poses3d[j, :] = np.reshape(out, [-1, 16 * 3]) if not ( FLAGS.predict_14) else np.reshape(out, [-1, 14 * 3]) # Compute Euclidean distance error per joint sqerr = ( poses3d - dec_out )**2 # Squared error between prediction and expected output dists = np.zeros((sqerr.shape[0], n_joints)) # Array with L2 error per joint in mm dist_idx = 0 for k in np.arange(0, n_joints * 3, 3): # Sum across X,Y, and Z dimenstions to obtain L2 distance dists[:, dist_idx] = np.sqrt(np.sum(sqerr[:, k:k + 3], axis=1)) dist_idx = dist_idx + 1 all_dists.append(dists) assert sqerr.shape[0] == FLAGS.batch_size step_time = (time.time() - start_time) / nbatches loss = loss / nbatches all_dists = np.vstack(all_dists) PCK_150 = all_dists < 150 from pdb import set_trace as st st() #AUC Metric non_zero = np.count_nonzero(PCK_150, axis=1) * 1. zeros = np.ones(non_zero.shape) * len(PCK_150[0]) - non_zero TP = non_zero / len(PCK_150[0]) FP = zeros / len(PCK_150[0]) AUC = (1 + TP - FP) / 2. AUC = np.mean(AUC) #PCK@150 PCK = np.mean(np.mean(PCK_150, axis=1)) # Error per joint and total for all passed batches joint_err = np.mean(all_dists, axis=0) total_err = np.mean(all_dists) print("=============================\n" "Step-time (ms): %.4f\n" "Val loss avg: %.4f\n" "Val error avg (mm): %.2f\n" "=============================" % (1000 * step_time, loss, total_err)) print("PCK ERROR: " + str(PCK)) print("AUC ERROR: " + str(AUC)) for i in range(n_joints): # 6 spaces, right-aligned, 5 decimal places print("Error in joint {0:02d} (mm): {1:>5.2f}".format( i + 1, joint_err[i])) print("=============================")