def read_mpi(data_path, do_transform, H36M_mean2d, H36M_mean3d):
#3dhp to H36M
M = np.array([10, 8, 11, 12, 13, 14, 15, 16, 4, 5, 6, 1, 2, 3, 0, 7, 9])
N = np.array([9, 8, 13, 14, 15, 12, 11, 10, 3, 4, 5, 2, 1, 0, 6, 7, 16])
# Loads 3dhp data
inputs = h5py.File(data_path, 'r')
#According to doc, this should be normalized but isn't
#Shape (2929, 17, 3)
test_set3d = inputs['univ_annot3']
test_set3d = test_set3d[:][:, M, :]
# test_set3d = test_set3d[:][:, N, :][:, :-1, :]
#Get rid of the first joint, subtract from rest
first_joint3d = np.expand_dims(test_set3d[:, 0, :], axis=1)
test_set3d = (test_set3d - first_joint3d)[:, 1:, :]
#Calculate 3d statistics
data_mean3d = np.mean(test_set3d, axis=0)
data_std3d = np.std(test_set3d, axis=0)
#If transform points, do procrustes transform
if do_transform:
_, Z, T, b, c = procrustes.compute_similarity_transform(
H36M_mean3d.reshape(-1, 3).T,
data_mean3d.T,
compute_optimal_scale=True)
test_set3d = np.transpose(
(b * np.transpose(test_set3d, (0, 2, 1)).dot(T)) + c, (0, 2, 1))
data_mean3d = np.mean(test_set3d, axis=0)
data_std3d = np.std(test_set3d, axis=0)
data_test3d = np.divide(test_set3d - data_mean3d, data_std3d)
#Shape (2929, 17, 2)
test_set2d = inputs['annot_2d']
# N = np.array([9,8,13,14,15,12,11,10,3,4,5,2,1,0,6,7,16])
# test_set2d = test_set2d[:][:, N, :]
#Get rid of the first joint
test_set2d = test_set2d[:, 1:, :] / 2.048
#Calculate 2d statistic
data_mean2d = np.mean(test_set2d, axis=0)
data_std2d = np.std(test_set2d, axis=0)
#If transform points, do procrustes transform
# if do_transform:
# _, Z, T, b, c = procrustes.compute_similarity_transform(H36M_mean2d.reshape(-1, 2), data_mean2d, compute_optimal_scale=True)
# test_set2d = (b * test_set2d.dot(T)) + c
# data_mean2d = np.mean(test_set2d, axis=0)
# data_std2d = np.std(test_set2d, axis=0)
data_test2d = np.divide(test_set2d - data_mean2d, data_std2d)
return data_test3d, data_mean3d, data_std3d, data_test2d, data_mean2d, data_std2d
def compute_mpjpe_summary(gt_pose, pred_pose):
gt_pose = normalize_3d_pose(gt_pose)
pred_pose = normalize_3d_pose(pred_pose)
pa_pose = pred_pose.copy()
try:
if opts['procustes']:
for i in range(opts['batch_size']):
gt = gt_pose[i]
out = pred_pose[i]
_, Z, T, b, c = procrustes.compute_similarity_transform(
gt, out, compute_optimal_scale=True)
out = (b * out.dot(T)) + c
pa_pose[i, :, :] = out
except:
#in case of an error, dump the pose batch and exit
sio.savemat("poses.mat", {"gt": gt_pose, "pred": pred_pose})
print("error in mpjpe")
exit()
return np.mean(np.sqrt(np.sum((pa_pose - gt_pose)**2, axis=2)))
def evaluate_batches( sess, model,
data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d,
data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d,
current_step, encoder_inputs, decoder_outputs, current_epoch=0 ):
"""
Generic method that evaluates performance of a list of batches.
May be used to evaluate all actions or a single action.
Args
sess
model
data_mean_3d
data_std_3d
dim_to_use_3d
dim_to_ignore_3d
data_mean_2d
data_std_2d
dim_to_use_2d
dim_to_ignore_2d
current_step
encoder_inputs
decoder_outputs
current_epoch
Returns
total_err
joint_err
step_time
loss
"""
n_joints = 17 if not(FLAGS.predict_14) else 14
nbatches = len( encoder_inputs )
# Loop through test examples
all_dists, start_time, loss = [], time.time(), 0.
log_every_n_batches = 100
for i in range(nbatches):
if current_epoch > 0 and (i+1) % log_every_n_batches == 0:
print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i+1, nbatches) )
enc_in, dec_out = encoder_inputs[i], decoder_outputs[i]
dp = 1.0 # dropout keep probability is always 1 at test time
step_loss, loss_summary, poses3d = model.step( sess, enc_in, dec_out, dp, isTraining=False )
loss += step_loss
# denormalize
enc_in = data_utils.unNormalizeData( enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d )
dec_out = data_utils.unNormalizeData( dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d )
poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d )
# Keep only the relevant dimensions
dtu3d = np.hstack( (np.arange(3), dim_to_use_3d) ) if not(FLAGS.predict_14) else dim_to_use_3d
dec_out = dec_out[:, dtu3d]
poses3d = poses3d[:, dtu3d]
assert dec_out.shape[0] == FLAGS.batch_size
assert poses3d.shape[0] == FLAGS.batch_size
if FLAGS.procrustes:
# Apply per-frame procrustes alignment if asked to do so
for j in range(FLAGS.batch_size):
gt = np.reshape(dec_out[j,:],[-1,3])
out = np.reshape(poses3d[j,:],[-1,3])
_, Z, T, b, c = procrustes.compute_similarity_transform(gt,out)
out = out.dot(T)+c
poses3d[j,:] = np.reshape(out,[-1,17*3] ) if not(FLAGS.predict_14) else np.reshape(out,[-1,14*3] )
# Compute Euclidean distance error per joint
sqerr = (poses3d - dec_out)**2 # Squared error between prediction and expected output
dists = np.zeros( (sqerr.shape[0], njoints) ) # Array with L2 error per joint in mm
dist_idx = 0
for k in np.arange(0, n_joints*3, 3):
# Sum across X,Y, and Z dimenstions to obtain L2 distance
dists[:,dist_idx] = np.sqrt( np.sum( sqerr[:, k:k+3], axis=1 ))
dist_idx = dist_idx + 1
all_dists.append(dists)
assert sqerr.shape[0] == FLAGS.batch_size
step_time = (time.time() - start_time) / nbatches
loss = loss / nbatches
all_dists = np.vstack( all_dists )
# Error per joint and total for all passed batches
joint_err = np.mean( all_dists, axis=0 )
total_err = np.mean( all_dists )
return total_err, joint_err, step_time, loss
def main(_):
actions_all = data_utils.define_actions("All")
actions = data_utils.define_actions("Discussion")
# Load camera parameters
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
# Load 3d data and load (or create) 2d projections
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, FLAGS.predict_14)
train_set_3d = data_utils.remove_first_frame(train_set_3d)
test_set_3d = data_utils.remove_first_frame(test_set_3d)
train_root_positions = data_utils.remove_first_frame(train_root_positions)
test_root_positions = data_utils.remove_first_frame(test_root_positions)
print("Finished Read 3D Data")
# train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(actions_all, FLAGS.data_dir)
# train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.transform_to_2d_biframe_prediction(train_set_2d,
# test_set_2d,
# data_mean_2d,
# data_std_2d,
# dim_to_ignore_2d,
# dim_to_use_2d)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data(
actions_all, FLAGS.data_dir, rcams)
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.transform_to_2d_biframe_prediction(
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d,
dim_to_use_2d)
SH_TO_GT_PERM = np.array(
[SH_NAMES.index(h) for h in H36M_NAMES if h != '' and h in SH_NAMES])
assert np.all(SH_TO_GT_PERM == np.array(
[6, 2, 1, 0, 3, 4, 5, 7, 8, 9, 13, 14, 15, 12, 11, 10]))
test_set = {}
manipulation_dir = os.path.dirname(FLAGS.data_dir)
manipulation_dir = os.path.dirname(manipulation_dir)
manipulation_dir += '/manipulation_video/'
manipulation_folders = glob.glob(manipulation_dir + '*')
subj = 1
action = 'manipulation-video'
for folder in manipulation_folders:
seqname = os.path.basename(folder)
with h5py.File(folder + '/' + seqname + '.h5', 'r') as h5f:
poses = h5f['poses'][:]
# Permute the loaded data to make it compatible with H36M
poses = poses[:, SH_TO_GT_PERM, :]
# Reshape into n x (32*2) matrix
poses = np.reshape(poses, [poses.shape[0], -1])
poses_final = np.zeros([poses.shape[0], len(H36M_NAMES) * 2])
dim_to_use_x = np.where(
np.array([x != '' and x != 'Neck/Nose'
for x in H36M_NAMES]))[0] * 2
dim_to_use_y = dim_to_use_x + 1
dim_to_use = np.zeros(len(SH_NAMES) * 2, dtype=np.int32)
dim_to_use[0::2] = dim_to_use_x
dim_to_use[1::2] = dim_to_use_y
poses_final[:, dim_to_use] = poses
print(seqname, poses_final.shape)
poses_final[poses_final == 0.] = 0.1
test_set[(subj, action, seqname)] = poses_final
test_set = data_utils.uni_frame_to_bi_frame(test_set)
test_set_2d = data_utils.normalize_data(test_set, data_mean_2d,
data_std_2d, dim_to_use_2d)
for key in test_set.keys():
test_set[key] = test_set[key][0::2, :]
dim_to_use_12_manipulation_joints = np.array([
3, 4, 5, 6, 7, 8, 9, 10, 11, 18, 19, 20, 21, 22, 23, 24, 25, 26, 51,
52, 53, 54, 55, 56, 57, 58, 59, 75, 76, 77, 78, 79, 80, 81, 82, 83
])
print("Finished Normalize Manipualtion Videos")
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.linear_size))
batch_size = FLAGS.batch_size #Intial code is 64*2
model = predict_3dpose_biframe.create_model(sess, actions_all,
batch_size)
print("Model loaded")
j = 0
for key2d in test_set_2d.keys():
(subj, b, fname) = key2d
# if fname != specific_seqname + '.h5':
# continue
print("Subject: {}, action: {}, fname: {}".format(subj, b, fname))
enc_in = test_set_2d[key2d]
n2d, _ = enc_in.shape
# Split into about-same-size batches
enc_in = np.array_split(enc_in, n2d // 1)
all_poses_3d = []
for bidx in range(len(enc_in)):
# Dropout probability 0 (keep probability 1) for sampling
dp = 1.0
anything = np.zeros((enc_in[bidx].shape[0], 48))
_, _, poses3d = model.step(sess,
enc_in[bidx],
anything,
dp,
isTraining=False)
# Denormalize
enc_in[bidx] = data_utils.unNormalizeData(
enc_in[bidx], data_mean_2d, data_std_2d, dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d,
dim_to_ignore_3d)
all_poses_3d.append(poses3d)
# Put all the poses together
enc_in, poses3d = map(np.vstack, [enc_in, all_poses_3d])
enc_in, poses3d = map(np.vstack, [enc_in, poses3d])
poses3d_12_manipulation = poses3d[:,
dim_to_use_12_manipulation_joints]
annotated_images = glob.glob(manipulation_dir + fname +
'/info/*.xml')
annotated_images = sorted(annotated_images)
# 1080p = 1,920 x 1,080
fig = plt.figure(j, figsize=(10, 10))
gs1 = gridspec.GridSpec(3, 3)
gs1.update(wspace=-0, hspace=0.1) # set the spacing between axes.
plt.axis('off')
subplot_idx = 1
nsamples = 3
for i in np.arange(nsamples):
# Plot 2d Detection
ax1 = plt.subplot(gs1[subplot_idx - 1])
img = mpimg.imread(
manipulation_dir + fname + '/skeleton_cropped/' +
os.path.basename(annotated_images[i]).split('_')[0] +
'.jpg')
ax1.imshow(img)
# Plot 2d pose
ax2 = plt.subplot(gs1[subplot_idx])
# p2d = enc_in[i,:]
# viz.show2Dpose( p2d, ax2 )
# ax2.invert_yaxis()
ax2.imshow(img)
# Plot 3d predictions
# Compute first the procrustion and print error
gt = getJ3dPosFromXML(annotated_images[i])
A = poses3d_12_manipulation[i, :].reshape(gt.shape)
_, Z, T, b, c = procrustes.compute_similarity_transform(
gt, A, compute_optimal_scale=True)
sqerr = np.sqrt(np.sum((gt - (b * A.dot(T)) - c)**2, axis=1))
print("{0} - {1} - Mean Error (mm) : {2}".format(
fname, os.path.basename(annotated_images[i]),
np.mean(sqerr)))
ax3 = plt.subplot(gs1[subplot_idx + 1], projection='3d')
temp = poses3d[i, :].reshape((32, 3))
temp = c + temp.dot(T) #Do not scale
# p3d = temp.reshape((1, 96))
p3d = poses3d[i, :]
viz.show3Dpose(p3d, ax3, lcolor="#9b59b6", rcolor="#2ecc71")
ax3.invert_zaxis()
ax3.invert_yaxis()
subplot_idx = subplot_idx + 3
plt.show()
j += 1
def evaluate_batches(sess,
model,
data_mean_3d,
data_std_3d,
dim_to_use_3d,
dim_to_ignore_3d,
current_step,
encoder_inputs,
decoder_outputs,
current_epoch=0):
n_joints = 17
nbatches = len(encoder_inputs)
all_dists, start_time, loss = [], time.time(), 0.
log_every_n_batches = 100
for i in range(nbatches):
if current_epoch > 0 and (i + 1) % log_every_n_batches == 0:
print("Working on test epoch {0}, batch {1} / {2}".format(
current_epoch, i + 1, nbatches))
enc_in, dec_out = encoder_inputs[i], decoder_outputs[i]
dp = 1.0 # dropout keep probability is always 1 at test time
step_loss, loss_summary, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
loss += step_loss
if (i == 0):
dec_out = np.vstack(
[dec_out[0, :, :], dec_out[1:, FLAGS.seqlen - 1, :]])
poses3d = np.vstack(
[poses3d[0, :, :], poses3d[1:, FLAGS.seqlen - 1, :]])
else:
dec_out = np.expand_dims(dec_out[:, FLAGS.seqlen - 1, :], axis=0)
poses3d = np.expand_dims(poses3d[:, FLAGS.seqlen - 1, :], axis=0)
dec_out = np.reshape(dec_out, [-1, (n_joints - 1) * 3])
poses3d = np.reshape(poses3d, [-1, (n_joints - 1) * 3])
dec_out = data_utils.unNormalizeData(dec_out, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
dtu3d = np.hstack((np.arange(3), dim_to_use_3d))
dec_out = dec_out[:, dtu3d]
poses3d = poses3d[:, dtu3d]
if FLAGS.procrustes:
# Apply per-frame procrustes alignment
for j in range(poses3d.shape[0]):
gt = np.reshape(dec_out[j, :], [-1, 3])
out = np.reshape(poses3d[j, :], [-1, 3])
_, Z, T, b, c = procrustes.compute_similarity_transform(
gt, out, compute_optimal_scale=True)
out = Z
poses3d[j, :] = np.reshape(out, [-1, 17 * 3])
# Compute Euclidean distance error per joint
sqerr = (
poses3d -
dec_out)**2 # Squared error between prediction and expected output
dists = np.zeros(
(sqerr.shape[0], n_joints)) # Array with L2 error per joint in mm
dist_idx = 0
for k in np.arange(0, n_joints * 3, 3):
# Sum across X,Y, and Z dimenstions to obtain L2 distance
dists[:, dist_idx] = np.sqrt(np.sum(sqerr[:, k:k + 3], axis=1))
dist_idx = dist_idx + 1
all_dists.append(dists)
step_time = (time.time() - start_time) / nbatches
loss = loss / nbatches
all_dists = np.vstack(all_dists)
# Error per joint and total for all passed batches
joint_err = np.mean(all_dists, axis=0)
total_err = np.mean(all_dists)
return total_err, joint_err, step_time, loss
def evaluate_batches(sess,
model,
data_mean_3d,
data_std_3d,
dim_to_use_3d,
dim_to_ignore_3d,
data_mean_2d,
data_std_2d,
dim_to_use_2d,
dim_to_ignore_2d,
current_step,
encoder_inputs,
decoder_outputs,
current_epoch=0):
"""
Generic method that evaluates performance of a list of batches.
May be used to evaluate all actions or a single action.
Args
sess
model
data_mean_3d
data_std_3d
dim_to_use_3d
dim_to_ignore_3d
data_mean_2d
data_std_2d
dim_to_use_2d
dim_to_ignore_2d
current_step
encoder_inputs
decoder_outputs
current_epoch
Returns
total_err
joint_err
step_time
loss
"""
n_joints = 17 if not (FLAGS.predict_14) else 14
nbatches = len(encoder_inputs)
# Loop through test examples
all_dists, start_time, loss = [], time.time(), 0.
log_every_n_batches = 100
all_poses_3d = []
all_enc_in = []
for i in range(nbatches):
if current_epoch > 0 and (i + 1) % log_every_n_batches == 0:
print("Working on test epoch {0}, batch {1} / {2}".format(
current_epoch, i + 1, nbatches))
enc_in, dec_out = encoder_inputs[i], decoder_outputs[i]
# enc_in = data_utils.generage_missing_data(enc_in, FLAGS.miss_num)
dp = 1.0 # dropout keep probability is always 1 at test time
step_loss, loss_summary, out_all_components_ori = model.step(
sess, enc_in, dec_out, dp, isTraining=False)
loss += step_loss
out_all_components = np.reshape(
out_all_components_ori,
[-1, model.HUMAN_3D_SIZE + 2, model.num_models])
out_mean = out_all_components[:, :model.HUMAN_3D_SIZE, :]
# denormalize
enc_in = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d,
dim_to_ignore_2d)
enc_in_ = copy.deepcopy(enc_in)
all_enc_in.append(enc_in_)
dec_out = data_utils.unNormalizeData(dec_out, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
pose_3d = np.zeros((enc_in.shape[0], 96, out_mean.shape[-1]))
for j in range(out_mean.shape[-1]):
pose_3d[:, :,
j] = data_utils.unNormalizeData(out_mean[:, :, j],
data_mean_3d, data_std_3d,
dim_to_ignore_3d)
pose_3d_ = copy.deepcopy(pose_3d)
all_poses_3d.append(pose_3d_)
# Keep only the relevant dimensions
dtu3d = np.hstack(
(np.arange(3),
dim_to_use_3d)) if not (FLAGS.predict_14) else dim_to_use_3d
dec_out = dec_out[:, dtu3d]
pose_3d = pose_3d[:, dtu3d, :]
assert dec_out.shape[0] == FLAGS.batch_size
assert pose_3d.shape[0] == FLAGS.batch_size
if FLAGS.procrustes:
# Apply per-frame procrustes alignment if asked to do so
for j in range(FLAGS.batch_size):
for k in range(model.num_models):
gt = np.reshape(dec_out[j, :], [-1, 3])
out = np.reshape(pose_3d[j, :, k], [-1, 3])
_, Z, T, b, c = procrustes.compute_similarity_transform(
gt, out, compute_optimal_scale=True)
out = (b * out.dot(T)) + c
pose_3d[j, :, k] = np.reshape(
out, [-1, 17 *
3]) if not (FLAGS.predict_14) else np.reshape(
pose_3d[j, :, k], [-1, 14 * 3])
# Compute Euclidean distance error per joint
sqerr = (pose_3d - np.expand_dims(dec_out, axis=2)
)**2 # Squared error between prediction and expected output
dists = np.zeros(
(sqerr.shape[0], n_joints,
sqerr.shape[2])) # Array with L2 error per joint in mm
for m in range(dists.shape[-1]):
dist_idx = 0
for k in np.arange(0, n_joints * 3, 3):
# Sum across X,Y, and Z dimenstions to obtain L2 distance
dists[:, dist_idx,
m] = np.sqrt(np.sum(sqerr[:, k:k + 3, m], axis=1))
dist_idx = dist_idx + 1
all_dists.append(dists)
assert sqerr.shape[0] == FLAGS.batch_size
step_time = (time.time() - start_time) / nbatches
loss = loss / nbatches
all_dists = np.vstack(all_dists)
aver_minerr = np.mean(np.min(np.sum(all_dists, axis=1), axis=1)) / n_joints
return aver_minerr, step_time, loss
def evaluate_batches( sess, model,
data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d,
data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d,
current_step, encoder_inputs, decoder_outputs, current_epoch=0 ):
"""
Generic method that evaluates performance of a list of batches.
May be used to evaluate all actions or a single action.
Args
sess
model
data_mean_3d
data_std_3d
dim_to_use_3d
dim_to_ignore_3d
data_mean_2d
data_std_2d
dim_to_use_2d
dim_to_ignore_2d
current_step
encoder_inputs
decoder_outputs
current_epoch
Returns
total_err
joint_err
step_time
loss
"""
n_joints = 17 if not(FLAGS.predict_14) else 14
nbatches = len( encoder_inputs )
# Loop through test examples
all_dists, start_time, loss = [], time.time(), 0.
log_every_n_batches = 100
for i in range(nbatches):
if current_epoch > 0 and (i+1) % log_every_n_batches == 0:
print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i+1, nbatches) )
enc_in, dec_out = encoder_inputs[i], decoder_outputs[i]
dp = 1.0 # dropout keep probability is always 1 at test time
step_loss, loss_summary, poses3d = model.step( sess, enc_in, dec_out, dp, isTraining=False )
loss += step_loss
# denormalize
enc_in = data_utils.unNormalizeData( enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d )
dec_out = data_utils.unNormalizeData( dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d )
poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d )
# Keep only the relevant dimensions
dtu3d = np.hstack( (np.arange(3), dim_to_use_3d) ) if not(FLAGS.predict_14) else dim_to_use_3d
dec_out = dec_out[:, dtu3d]
poses3d = poses3d[:, dtu3d]
assert dec_out.shape[0] == FLAGS.batch_size
assert poses3d.shape[0] == FLAGS.batch_size
if FLAGS.procrustes:
# Apply per-frame procrustes alignment if asked to do so
for j in range(FLAGS.batch_size):
gt = np.reshape(dec_out[j,:],[-1,3])
out = np.reshape(poses3d[j,:],[-1,3])
_, Z, T, b, c = procrustes.compute_similarity_transform(gt,out,compute_optimal_scale=True)
out = (b*out.dot(T))+c
poses3d[j,:] = np.reshape(out,[-1,17*3] ) if not(FLAGS.predict_14) else np.reshape(out,[-1,14*3] )
# Compute Euclidean distance error per joint
sqerr = (poses3d - dec_out)**2 # Squared error between prediction and expected output
dists = np.zeros( (sqerr.shape[0], n_joints) ) # Array with L2 error per joint in mm
dist_idx = 0
for k in np.arange(0, n_joints*3, 3):
# Sum across X,Y, and Z dimenstions to obtain L2 distance
dists[:,dist_idx] = np.sqrt( np.sum( sqerr[:, k:k+3], axis=1 ))
dist_idx = dist_idx + 1
all_dists.append(dists)
assert sqerr.shape[0] == FLAGS.batch_size
step_time = (time.time() - start_time) / nbatches
loss = loss / nbatches
all_dists = np.vstack( all_dists )
# Error per joint and total for all passed batches
joint_err = np.mean( all_dists, axis=0 )
total_err = np.mean( all_dists )
return total_err, joint_err, step_time, loss
def evaluate_batches( sess, model,
data_mean_3d, data_std_3d, dim_to_use_3d, dim_to_ignore_3d,
data_mean_2d, data_std_2d, dim_to_use_2d, dim_to_ignore_2d,
current_step, encoder_inputs, decoder_outputs, current_epoch=0, birdNames = False ):
"""
Generic method that evaluates performance of a list of batches.
May be used to evaluate all actions or a single action.
Args
sess: tensorflow session
model: tensorflow model to run evaluation with
data_mean_3d: the mean of the training data in 3d
data_std_3d: the standard deviation of the training data in 3d
dim_to_use_3d: out of all the 96 dimensions that represent a 3d body in h36m, compute results for this subset
dim_to_ignore_3d: complelment of the above
data_mean_2d: mean of the training data in 2d
data_std_2d: standard deviation of the training data in 2d
dim_to_use_2d: out of the 64 dimensions that represent a body in 2d in h35m, use this subset
dim_to_ignore_2d: complement of the above
current_step: training iteration step
encoder_inputs: input for the network
decoder_outputs: expected output for the network
current_epoch: current training epoch
Returns
total_err: average mm error over all joints
joint_err: average mm error per joint
step_time: time it took to evaluate one batch
loss: validation loss of the network
"""
if birdNames == False:
n_joints = 17 if not(FLAGS.predict_14) else 14
else:
n_joints = 7
nbatches = len( encoder_inputs )
# Loop through test examples
all_dists, start_time, loss = [], time.time(), 0.
log_every_n_batches = 100
for i in range(nbatches):
if current_epoch > 0 and (i+1) % log_every_n_batches == 0:
print("Working on test epoch {0}, batch {1} / {2}".format( current_epoch, i+1, nbatches) )
enc_in, dec_out = encoder_inputs[i], decoder_outputs[i]
dp = 1.0 # dropout keep probability is always 1 at test time
step_loss, loss_summary, poses3d = model.step( sess, enc_in, dec_out, dp, isTraining=False )
loss += step_loss
# denormalize
enc_in = data_utils.unNormalizeData( enc_in, data_mean_2d, data_std_2d, dim_to_ignore_2d )
dec_out = data_utils.unNormalizeData( dec_out, data_mean_3d, data_std_3d, dim_to_ignore_3d )
poses3d = data_utils.unNormalizeData( poses3d, data_mean_3d, data_std_3d, dim_to_ignore_3d )
# Keep only the relevant dimensions
dtu3d = np.hstack( (np.arange(3), dim_to_use_3d) ) if not(FLAGS.predict_14) else dim_to_use_3d
dec_out = dec_out[:, dtu3d]
poses3d = poses3d[:, dtu3d]
assert dec_out.shape[0] == FLAGS.batch_size
assert poses3d.shape[0] == FLAGS.batch_size
if FLAGS.procrustes:
# Apply per-frame procrustes alignment if asked to do so
for j in range(FLAGS.batch_size):
gt = np.reshape(dec_out[j,:],[-1,3])
out = np.reshape(poses3d[j,:],[-1,3])
_, Z, T, b, c = procrustes.compute_similarity_transform(gt,out,compute_optimal_scale=True)
out = (b*out.dot(T))+c
poses3d[j,:] = np.reshape(out,[-1,17*3] ) if not(FLAGS.predict_14) else np.reshape(out,[-1,14*3] )
# Compute Euclidean distance error per joint
sqerr = (poses3d - dec_out)**2 # Squared error between prediction and expected output
dists = np.zeros( (sqerr.shape[0], n_joints) ) # Array with L2 error per joint in mm
dist_idx = 0
for k in np.arange(0, n_joints*3, 3):
# Sum across X,Y, and Z dimenstions to obtain L2 distance
dists[:,dist_idx] = np.sqrt( np.sum( sqerr[:, k:k+3], axis=1 ))
dist_idx = dist_idx + 1
all_dists.append(dists)
assert sqerr.shape[0] == FLAGS.batch_size
step_time = (time.time() - start_time) / nbatches
loss = loss / nbatches
all_dists = np.vstack( all_dists )
# Error per joint and total for all passed batches
joint_err = np.mean( all_dists, axis=0 )
total_err = np.mean( all_dists )
return total_err, joint_err, step_time, loss
def mpi():
actions = data_utils.define_actions(FLAGS.action)
number_of_actions = len(actions)
# Load camera parameters
SUBJECT_IDS = [1, 5, 6, 7, 8, 9, 11]
rcams = cameras.load_cameras(FLAGS.cameras_path, SUBJECT_IDS)
# Load 3d data and load (or create) 2d projections
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions = data_utils.read_3d_data(
actions, FLAGS.data_dir + 'train_images.txt',
FLAGS.data_dir + 'valid_images.txt', FLAGS.data_dir + 'train.h5',
FLAGS.data_dir + 'valid.h5', FLAGS.camera_frame, rcams,
FLAGS.predict_14)
# Read stacked hourglass 2D predictions if use_sh, otherwise use groundtruth 2D projections
if FLAGS.use_sh:
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir + 'train_images.txt',
FLAGS.data_dir + 'valid_images.txt', FLAGS.data_dir + 'train.h5',
FLAGS.data_dir + 'valid.h5')
else:
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data(
actions, FLAGS.data_dir + 'train_images.txt',
FLAGS.data_dir + 'valid_images.txt', FLAGS.data_dir + 'train.h5',
FLAGS.data_dir + 'valid.h5', rcams)
print("done reading and normalizing data.")
# Load MPI data for testing:
mpi_test3d, mpi_mean3d, mpi_std3d, mpi_test2d, mpi_mean2d, mpi_std2d = data_utils.read_mpi(
'/mnt/lustre/xingyifei/test_3dhp/annotTest.h5', True, data_mean_2d,
data_mean_3d)
# Avoid using the GPU if requested
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(device_count=device_count,
allow_soft_placement=True)) as sess:
# === Create the model ===
print("Creating %d bi-layers of %d units." %
(FLAGS.num_layers, FLAGS.linear_size))
model = create_model(sess, actions, FLAGS.batch_size)
model.train_writer.add_graph(sess.graph)
print("Model created")
#=== This is the training loop ===
step_time, loss, val_loss = 0.0, 0.0, 0.0
current_step = 0 if FLAGS.load <= 0 else FLAGS.load + 1
previous_losses = []
step_time, loss = 0, 0
current_epoch = 0
log_every_n_batches = 100
# === Testing after this epoch ===
isTraining = False
n_joints = 16 if not (FLAGS.predict_14) else 14
#Process inputs to batches
n = len(mpi_test3d)
n_extra = n % model.batch_size
if n_extra > 0: # Otherwise examples are already a multiple of batch size
encoder_inputs = mpi_test2d[:-n_extra, :, :].reshape(-1, 32)
decoder_outputs = mpi_test3d[:-n_extra, :, :].reshape(-1, 48)
n_batches = n // model.batch_size
encoder_inputs = np.split(encoder_inputs, n_batches)
decoder_outputs = np.split(decoder_outputs, n_batches)
nbatches = len(encoder_inputs)
# Loop through test examples
all_dists, start_time, loss = [], time.time(), 0.
log_every_n_batches = 100
for i in range(nbatches):
if current_epoch > 0 and (i + 1) % log_every_n_batches == 0:
print("Working on test epoch {0}, batch {1} / {2}".format(
current_epoch, i + 1, nbatches))
enc_in, dec_out = encoder_inputs[i], decoder_outputs[i]
dp = 1.0 # dropout keep probability is always 1 at test time
step_loss, loss_summary, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
loss += step_loss
# denormalize
enc_in = (enc_in.reshape(-1, 16, 2) * mpi_std2d +
mpi_mean2d).reshape(-1, 32)
dec_out = (dec_out.reshape(-1, 16, 3) * mpi_std3d +
mpi_mean3d).reshape(-1, 48)
poses3d = (poses3d.reshape(-1, 16, 3) * mpi_std3d +
mpi_mean3d).reshape(-1, 48)
assert dec_out.shape[0] == FLAGS.batch_size
assert poses3d.shape[0] == FLAGS.batch_size
if FLAGS.procrustes:
# Apply per-frame procrustes alignment if asked to do so
for j in range(FLAGS.batch_size):
gt = np.reshape(dec_out[j, :], [-1, 3])
out = np.reshape(poses3d[j, :], [-1, 3])
_, Z, T, b, c = procrustes.compute_similarity_transform(
gt, out, compute_optimal_scale=True)
out = (b * out.dot(T)) + c
poses3d[j, :] = np.reshape(out, [-1, 16 * 3]) if not (
FLAGS.predict_14) else np.reshape(out, [-1, 14 * 3])
# Compute Euclidean distance error per joint
sqerr = (
poses3d - dec_out
)**2 # Squared error between prediction and expected output
dists = np.zeros((sqerr.shape[0],
n_joints)) # Array with L2 error per joint in mm
dist_idx = 0
for k in np.arange(0, n_joints * 3, 3):
# Sum across X,Y, and Z dimenstions to obtain L2 distance
dists[:, dist_idx] = np.sqrt(np.sum(sqerr[:, k:k + 3], axis=1))
dist_idx = dist_idx + 1
all_dists.append(dists)
assert sqerr.shape[0] == FLAGS.batch_size
step_time = (time.time() - start_time) / nbatches
loss = loss / nbatches
all_dists = np.vstack(all_dists)
PCK_150 = all_dists < 150
from pdb import set_trace as st
st()
#AUC Metric
non_zero = np.count_nonzero(PCK_150, axis=1) * 1.
zeros = np.ones(non_zero.shape) * len(PCK_150[0]) - non_zero
TP = non_zero / len(PCK_150[0])
FP = zeros / len(PCK_150[0])
AUC = (1 + TP - FP) / 2.
AUC = np.mean(AUC)
#PCK@150
PCK = np.mean(np.mean(PCK_150, axis=1))
# Error per joint and total for all passed batches
joint_err = np.mean(all_dists, axis=0)
total_err = np.mean(all_dists)
print("=============================\n"
"Step-time (ms): %.4f\n"
"Val loss avg: %.4f\n"
"Val error avg (mm): %.2f\n"
"=============================" %
(1000 * step_time, loss, total_err))
print("PCK ERROR: " + str(PCK))
print("AUC ERROR: " + str(AUC))
for i in range(n_joints):
# 6 spaces, right-aligned, 5 decimal places
print("Error in joint {0:02d} (mm): {1:>5.2f}".format(
i + 1, joint_err[i]))
print("=============================")
