def read_data(fname):
hf = h5py.File(fname, 'r')
action = 'discussion'
gt_sequences = np.zeros((8, 100, 99))
pred_sequences = np.zeros((8, 100, 99))
euler_gt_sequences = np.zeros((8, 100, 99))
euler_pred_sequences = np.zeros((8, 100, 99))
error_hf = hf.get('mean_' + action + '_error/')
errors = np.array(error_hf)
for i in range(8):
gt_fname = 'expmap/gt/' + action + '_' + str(i)
n1 = np.array(hf.get(gt_fname))
gt_sequences[i, :, :] = n1
pred_fname = 'expmap/preds/' + action + '_' + str(i)
n2 = np.array(hf.get(pred_fname))
pred_sequences[i, :, :] = n2
# converting back to euler angles
for j in np.arange(gt_sequences.shape[1]):
for k in np.arange(3, 97, 3):
euler_gt_sequences[i, j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(gt_sequences[i, j, k:k + 3]))
euler_pred_sequences[i, j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(pred_sequences[i, j, k:k + 3]))
euler_gt_sequences[i, :, 0:6] = 0
euler_pred_sequences[i, :, 0:6] = 0
return euler_gt_sequences, euler_pred_sequences, errors
def get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
to_euler=True):
srnn_gts_euler = {}
# print ("entering here")
for action in actions:
srnn_gt_euler = []
_, _, srnn_expmap = model.get_batch_srnn(test_set,
action,
noise_rate=FLAGS.n_r)
# expmap -> rotmat -> euler
for i in np.arange(srnn_expmap.shape[0]):
denormed = data_utils.unNormalizeData(srnn_expmap[i, :, :],
data_mean, data_std,
dim_to_ignore, actions)
if to_euler:
for j in np.arange(denormed.shape[0]):
# print (denormed.shape)
for k in np.arange(3, 97, 3):
denormed[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(denormed[j, k:k + 3]))
srnn_gt_euler.append(denormed)
# Put back in the dictionary, every action will have 8 sequences of euler space
srnn_gts_euler[action] = srnn_gt_euler
# print (np.array(srnn_gts_euler[action]).shape)
return srnn_gts_euler
def revert_coordinate_space(channels, R0, T0):
"""
Bring a series of poses to a canonical form so they are facing the camera when they start.
Adapted from
https://github.com/asheshjain399/RNNexp/blob/7fc5a53292dc0f232867beb66c3a9ef845d705cb/structural_rnn/CRFProblems/H3.6m/dataParser/Utils/revertCoordinateSpace.m
Args
channels: n-by-99 matrix of poses
R0: 3x3 rotation for the first frame
T0: 1x3 position for the first frame
Returns
channels_rec: The passed poses, but the first has T0 and R0, and the
rest of the sequence is modified accordingly.
"""
n, d = channels.shape
channels_rec = copy.copy(channels)
R_prev = R0
T_prev = T0
rootRotInd = np.arange(3, 6)
# Loop through the passed posses
for ii in range(n):
R_diff = data_utils.expmap2rotmat(channels[ii, rootRotInd])
R = R_diff.dot(R_prev)
channels_rec[ii, rootRotInd] = data_utils.rotmat2expmap(R)
T = T_prev + (
(R_prev.T).dot(np.reshape(channels[ii, :3], [3, 1]))).reshape(-1)
channels_rec[ii, :3] = T
T_prev = T
R_prev = R
return channels_rec
def visualization_callback(data):
rospy.loginfo('%s: %s' % (data.header.seq, data.header.stamp))
skeleto = data.skeleto
angles = np.array(skeleto)
assert len(angles) == 99
# Structure that indicates parents for each joint
njoints = 32
stamp = data.header.stamp
rate = rospy.Rate(1000)
br = tf.TransformBroadcaster()
while rospy.get_rostime() < stamp:
rate.sleep()
for i in range(njoints):
tran = offset[i]
rot = np.eye(3)
if (i > 0):
rm = np.eye(4)
rot = data_utils.expmap2rotmat(angles[expmapInd[i]])
rm[0:3, 0:3] = np.linalg.inv(rot)
quat = tf.transformations.quaternion_from_matrix(rm)
br.sendTransform(tran / 1000, quat, stamp,
'%sj%s' % (g_tf_prefix, i),
'%sj%s' % (g_tf_prefix, parent[i]))
else:
quat = tf.transformations.quaternion_from_euler(np.pi / 2, 0, 0)
br.sendTransform(tran / 1000, quat, stamp,
'%sj%s' % (g_tf_prefix, i), 'skeleto')
def denormalize_and_convert_to_euler(data, data_mean, data_std, dim_to_ignore,
actions, one_hot):
"""
Denormalizes data and converts to Euler angles
(all losses are computed on Euler angles).
Args
data: dictionary with human poses.
data_mean: d-long vector with the mean of the training data.
data_std: d-long vector with the standard deviation of the training data.
dim_to_ignore: dimensions to ignore because the std is too small or for other reasons.
actions: list of strings with the actions in the data dictionary.
one_hot: whether the data comes with one-hot encoding.
Returns
all_denormed: a list with nbatch entries. Each entry is an n-by-d matrix
that corresponds to a denormalized sequence in Euler angles
"""
all_denormed = []
# expmap -> rotmat -> euler
for i in np.arange(data.shape[0]):
denormed = data_utils.unNormalizeData(data[i, :, :], data_mean,
data_std, dim_to_ignore, actions,
one_hot)
for j in np.arange(denormed.shape[0]):
for k in np.arange(3, 97, 3):
denormed[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(denormed[j, k:k + 3]))
all_denormed.append(denormed)
return all_denormed
def read_data(gt_sequences, pred_sequences):
euler_gt_sequences = np.zeros((100, 99))
euler_pred_sequences = np.zeros((100, 99))
# converting back to euler angles
for j in np.arange(gt_sequences.shape[1]):
for k in np.arange(3, 97, 3):
euler_gt_sequences[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(gt_sequences[j, k:k + 3]))
euler_pred_sequences[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(pred_sequences[j, k:k + 3]))
euler_gt_sequences[:, 0:6] = 0
euler_pred_sequences[:, 0:6] = 0
return euler_gt_sequences, euler_pred_sequences
def get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
one_hot,
from_exp=True,
to_euler=True):
"""
Get the ground truths for srnn's sequences, and convert to Euler angles.
(the error is always computed in Euler angles).
Args
actions: a list of actions to get ground truths for.
model: training model we are using (we only use the "get_batch" method).
test_set: dictionary with normalized training data.
data_mean: d-long vector with the mean of the training data.
data_std: d-long vector with the standard deviation of the training data.
dim_to_ignore: dimensions that we are not using to train/predict.
one_hot: whether the data comes with one-hot encoding indicating action.
to_euler: whether to convert the angles to Euler format or keep thm in exponential map
Returns
srnn_gts_euler: a dictionary where the keys are actions, and the values
are the ground_truth, denormalized expected outputs of srnns's seeds.
"""
srnn_gts = {}
for action in actions:
srnn_gt = []
_, _, srnn = model.get_batch_srnn(test_set, action, False)
for i in np.arange(srnn.shape[0]):
denormed = data_utils.unNormalizeData(srnn[i, :, :], data_mean,
data_std, dim_to_ignore,
actions, one_hot)
if from_exp and to_euler:
for j in np.arange(denormed.shape[0]):
for k in np.arange(3, 97, 3):
denormed[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(denormed[j, k:k + 3]))
if not from_exp and not to_euler: # from euler to exp
for j in np.arange(denormed.shape[0]):
for k in np.arange(3, 97, 3):
denormed[j, k:k + 3] = data_utils.rotmat2expmap(
data_utils.euler2rotmat(denormed[j, k:k + 3]))
srnn_gt.append(denormed)
# Put back in the dictionary
srnn_gts[action] = srnn_gt
return srnn_gts
def fkl(angles, parent, offset, posInd, expmapInd):
"""
Convert joint angles and bone lenghts into the 3d points of a person.
Based on expmap2xyz.m, available at
https://github.com/asheshjain399/RNNexp/blob/7fc5a53292dc0f232867beb66c3a9ef845d705cb/structural_rnn/CRFProblems/H3.6m/mhmublv/Motion/exp2xyz.m
Args
angles: 99-long vector with 3d position and 3d joint angles in expmap format
parent: 32-long vector with parent-child relationships in the kinematic tree
offset: 96-long vector with bone lenghts
rotInd: 32-long list with indices into angles
expmapInd: 32-long list with indices into expmap angles
Returns
xyz: 32x3 3d points that represent a person in 3d space
"""
assert len(angles) == 117
# Structure that indicates parents for each joint
njoints = 38
xyzStruct = [dict() for x in range(njoints)]
for i in np.arange(njoints):
try:
if not posInd[i]: # If the list is empty
xangle, yangle, zangle = 0, 0, 0
else:
xangle = angles[posInd[i][2] - 1]
yangle = angles[posInd[i][1] - 1]
zangle = angles[posInd[i][0] - 1]
except:
print(i)
r = angles[expmapInd[i]]
thisRotation = data_utils.expmap2rotmat(r)
thisPosition = np.array([xangle, yangle, zangle])
if parent[i] == -1: # Root node
xyzStruct[i]['rotation'] = thisRotation
xyzStruct[i]['xyz'] = np.reshape(offset[i, :],
(1, 3)) + thisPosition
else:
xyzStruct[i]['xyz'] = (offset[i, :] + thisPosition).dot(
xyzStruct[parent[i]]['rotation']) + xyzStruct[parent[i]]['xyz']
xyzStruct[i]['rotation'] = thisRotation.dot(
xyzStruct[parent[i]]['rotation'])
xyz = [xyzStruct[i]['xyz'] for i in range(njoints)]
xyz = np.array(xyz).squeeze()
xyz = xyz[:, [0, 2, 1]]
return np.reshape(xyz, [-1])
def get_srnn_gts_sample(actions, model, test_set, data_mean, data_std, dim_to_ignore, one_hot, to_euler=True):
"""
Get the ground truths for srnn's sequences, and convert to Euler angles. (sampling)
(the error is always computed in Euler angles).
Args
actions: a list of actions to get ground truths for.
model: training model we are using (we only use the "get_batch" method).
test_set: dictionary with normalized training data.
data_mean: d-long vector with the mean of the training data.
data_std: d-long vector with the standard deviation of the training data.
dim_to_ignore: dimensions that we are not using to train/predict.
one_hot: whether the data comes with one-hot encoding indicating action.
to_euler: whether to convert the angles to Euler format or keep thm in exponential map
Returns
srnn_gts_euler: a dictionary where the keys are actions, and the values
are the ground_truth, denormalized expected outputs of srnns's seeds.
"""
srnn_gts_euler = {}
for action in actions:
srnn_gt_euler = []
encoder_input, decoder_input, decoder_output = model.get_batch_srnn(test_set, action)
if FLAGS.omit_one_hot:
srnn_expmap = np.zeros((8, 100+8, 54), dtype=float)
else:
srnn_expmap = np.zeros((8, 100+8, 54+len(actions)), dtype=float)
for i in np.arange(decoder_output.shape[0]):
sequence = np.concatenate((encoder_input[i][-7:],decoder_input[i][0:1], decoder_output[i][:]),axis=0)
srnn_expmap[i, :, :] = sequence[:,:]
# print(srnn_expmap.shape)
# expmap -> rotmat -> euler
for i in np.arange(srnn_expmap.shape[0]):
denormed = data_utils.unNormalizeData(srnn_expmap[i,:,:], data_mean, data_std, dim_to_ignore, actions, one_hot)
if to_euler:
for j in np.arange(denormed.shape[0]):
for k in np.arange(3,97,3):
denormed[j,k:k+3] = data_utils.rotmat2euler( data_utils.expmap2rotmat(denormed[j,k:k+3]))
srnn_gt_euler.append(denormed);
# Put back in the dictionary
srnn_gts_euler[action] = srnn_gt_euler
return srnn_gts_euler
def get_srnn_gts(actions,
test_set,
data_mean,
data_std,
dim_to_ignore,
one_hot,
subject,
subsequence,
to_euler=True):
"""
Get the ground truths for srnn's sequences, and convert to Euler angles.
(the error is always computed in Euler angles).
Args
actions: a list of actions to get ground truths for.
test_set: dictionary with normalized training data.
data_mean: d-long vector with the mean of the training data.
data_std: d-long vector with the standard deviation of the training data.
dim_to_ignore: dimensions that we are not using to train/predict.
one_hot: whether the data comes with one-hot encoding indicating action.
to_euler: whether to convert the angles to Euler format or keep thm in exponential map
Returns
srnn_gts_euler: a dictionary where the keys are actions, and the values
are the ground_truth, denormalized expected outputs of srnns's seeds.
"""
srnn_gts_euler = {}
for action in actions:
srnn_gt_euler = []
# _, _, srnn_expmap = get_batch_srnn( test_set, action )
srnn_expmap = get_batch_srnn(test_set, action, subject, subsequence)
# expmap -> rotmat -> euler
for i in np.arange(srnn_expmap.shape[0]):
denormed = data_utils.unNormalizeData(srnn_expmap[i, :, :],
data_mean, data_std,
dim_to_ignore, actions,
one_hot)
if to_euler:
for j in np.arange(denormed.shape[0]):
for k in np.arange(3, 97, 3):
denormed[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(denormed[j, k:k + 3]))
srnn_gt_euler.append(denormed)
# Put back in the dictionary
srnn_gts_euler[action] = np.array(srnn_gt_euler)
return srnn_gts_euler
def train():
"""Train a seq2seq model on human motion"""
actions = define_actions(
FLAGS.action
) # here is like a list of actions, eg, ['walking'], ['walking, waiting, directions']
train_average_list = []
mean_error_list = []
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir)
number = 97
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=1
) # Allowing GPU memory growth, this means whole GPU
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers,
FLAGS.size)) # what is layer number in original paper?
# print (FLAGS.opt)
model = create_model(sess, actions, optimizer_to_use=FLAGS.opt)
print("Model created")
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore)
#=== This is the training loop ===
step_time, loss, val_loss = 0.0, 0.0, 0.0
current_step = 0 if FLAGS.load <= 0 else FLAGS.load
previous_losses = []
step_time, loss = 0, 0
sampling_rate = FLAGS.sampling_rate
sampling_rate_decay_factor = (
FLAGS.sampling_rate -
FLAGS.ending_sampling_rate) / FLAGS.iterations
for c_step in xrange(FLAGS.iterations):
start_time = time.time()
# === Training step === Every time, you get a new batch and run your model on it to calculate the loss
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
train_set)
_, _, step_loss = model.step_train_t(True,
sess,
encoder_inputs,
decoder_inputs,
decoder_outputs,
sampling_rate=sampling_rate)
sampling_rate_recoder = sampling_rate
sampling_rate = sampling_rate - sampling_rate_decay_factor
step_time += (time.time() - start_time) / FLAGS.test_every
loss += step_loss / FLAGS.test_every
current_step += 1
# === step decay ===
if current_step % FLAGS.learning_rate_step == 0:
sess.run(model.learning_rate_decay_op)
print("the learning rate becomes to: {0}".format(
model.learning_rate.eval()))
# Once in a while, we save checkpoint, print statistics, and run evals. How to get more 160, 320ms's statistics?
if current_step % FLAGS.test_every == 0:
# === Validation with randomly chosen seeds === # only test it on subject 5's same action to get accuracy
forward_only = True # don't learn on these samples
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [
80, 160, 240, 320, 400, 480, 560, 640, 720, 800, 880,
960, 1000
]:
print(" {0:5d} |".format(ms), end="")
print()
# # === Validation with srnn's seeds === # The only difference is how to choose the seed?
action = actions[0]
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
srnn_loss, srnn_poses, srnn_poses_s, srnn_poses_t, _ = model.step_test(
False,
sess,
encoder_inputs,
decoder_inputs,
decoder_outputs,
sampling_rate=0.0)
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore, actions)
# Save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred = srnn_pred_expmap[i]
# Convert from exponential map to Euler angles
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, number, 3):
eulerchannels_pred[j, k:k +
3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j,
k:k +
3]))
gt_i = np.copy(srnn_gts_euler[action][i])
gt_i[:, 0:6] = 0 # the translation is 0?
idx_to_use = np.where(np.std(gt_i, 0) > 1e-4)[0]
euc_error = np.power(
gt_i[:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
# This is simply the mean error over the N_SEQUENCE_TEST examples
mean_mean_errors = np.mean(mean_errors, 0)
# GX: here I think 1000ms for 25 frames, so that you calculate error on one single frame
print("{0: <8}temporal |".format(action), end="")
for ms in [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 24]:
if FLAGS.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]),
end="")
else:
print(" n/a |", end="")
print()
sampled_error = np.mean(mean_mean_errors)
print("temporal output mean error is: ", sampled_error)
print(
"============================\n"
"Global step: %d\n"
"Learning rate: %.10f\n"
"Step-time (ms): %.4f\n"
"Train loss avg: %.4f\n"
"sampling_rate: %.4f\n"
"--------------------------\n"
"val loss: %.4f\n"
"srnn loss: %.4f\n"
"============================" %
(current_step, model.learning_rate.eval(), step_time *
1000, loss, sampling_rate_recoder, step_loss, srnn_loss))
print()
train_average_list.append(loss)
mean_error_list.append(sampled_error)
# Save the model
print("current step is : ", current_step)
if current_step % FLAGS.save_every == 0:
print("Saving the model...")
start_time = time.time()
model.saver.save(sess,
os.path.normpath(
os.path.join(train_dir,
'checkpoint')),
global_step=current_step)
print("done in {0:.2f} ms".format(
(time.time() - start_time) * 1000))
# Reset global time and loss
step_time, loss = 0, 0
sys.stdout.flush()
fig, ax = plt.subplots()
ax.plot(train_average_list, 'r-', label='train loss')
ax.plot(mean_error_list, 'b-', label='mean error')
legend = ax.legend(loc=0)
plt.grid(True)
plt.title('sequence in ' + str(FLAGS.seq_length_in) + ', out ' +
str(FLAGS.seq_length_out))
plot_name = FLAGS.action + "_alpha_" + str(FLAGS.alpha) + "_beta_" + str(
FLAGS.beta) + "_gamma_" + str(FLAGS.gamma) + ".png"
plt.savefig(plot_name)
print('over')
def compute_test_error(self, action, pred_pose, srnn_gts_expmap,
srnn_gts_euler, one_hot, samples_fname):
"""
Compute the test error
"""
pred_pose = np.squeeze(pred_pose)
predict_expmap = data_utils.revert_output_format(
pred_pose, self.data_mean, self.data_std, self.dim_to_ignore,
self.actions, one_hot)
mean_errors = np.zeros(
(len(predict_expmap), predict_expmap[0].shape[0]))
for i in np.arange(8):
eulerchannels_pred = np.copy(predict_expmap[i])
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(0, 97, 3):
idx = [k, k + 1, k + 2]
eulerchannels_pred[j, idx] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(predict_expmap[i][j, idx]))
eulerchannels_pred[:, 0:6] = 0
srnn_gts_euler[action][i][:,
0:6] = 0 ## Fixed by ZHEN, we also need
# Pick only the dimensions with sufficient standard deviation. Others are ignored.
# the below code is wrong!!!
# After a look at the original code, I found that the code below must be fixed!!! (ZHEN)
idx_to_use = np.where(
np.std(srnn_gts_euler[action][i], 0) > 1e-4)[0]
#print(idx_to_use)
euc_error = np.power(
srnn_gts_euler[action][i][50:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
mean_mean_errors = np.mean(mean_errors, 0)
print()
print(action)
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 560, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 13, 24]:
if self.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]), end="")
else:
print(" n/a |", end="")
print()
with h5py.File(samples_fname, 'a') as hf:
for i in np.arange(8):
node_name = 'expmap/gt/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=srnn_gts_expmap[action][i])
node_name = 'expmap/preds/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=predict_expmap[i])
node_name = 'mean_{0}_error'.format(action)
hf.create_dataset(node_name, data=mean_mean_errors)
def RvizCallback(data):
rospy.loginfo('%s: %s' % (data.header.seq, data.header.stamp))
g = data.motion
g_motion = g.strip().split(',')
g_model = g_motion[0]
g_action = g_motion[1]
g_path = 'Pre-trained/{0}_{1}/'.format(g_model, g_action)
print "receive the model: {0} ".format(g_model)
print "receive the action: {0} ".format(g_action)
g_gts, g_preds = readFile(g_path)
# Broadcast tf
parent, offset, rotInd, expmapInd = _some_variables()
# numpy implementation
with h5py.File('Motion/{0}_{1}.h5'.format(g_model, g_action), 'r') as h5f:
# expmap_gt = h5f['expmap/gt/walking_0'][:]
# expmap_pred = h5f['expmap/preds/walking_0'][:]
expmap_gt = h5f['expmap/gts/{0}_0'.format(g_action)][:]
expmap_pred = h5f['expmap/preds/{0}_0'.format(g_action)][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[
0] # 100x99
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
angles = expmap_gt[i, :]
assert len(angles) == 99
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
angles = expmap_pred[i, :]
assert len(angles) == 99
# Structure that indicates parents for each joint
njoints = 32
stamp = data.header.stamp
br = tf.TransformBroadcaster()
while rospy.get_rostime() < stamp:
rate.sleep()
for i in range(njoints):
tran = offset[i]
rot = np.eye(3)
if (i > 0):
rm = np.eye(4)
rot = expmap2rotmat(angles[expmapInd[i]])
rm[0:3, 0:3] = np.linalg.inv(rot)
quat = tf.transformations.quaternion_from_matrix(rm)
br.sendTransform(tran / 1000, quat, stamp, '%s_%s' % (g_model, i),
'%s_%s' % (g_model, parent[i]))
else:
quat = tf.transformations.quaternion_from_euler(np.pi / 2, 0, 0)
br.sendTransform(tran / 1000, quat, stamp, '%s_%s' % (g_model, i),
'motion')
def sample():
"""Sample predictions for srnn's seeds"""
if FLAGS.load <= 0:
raise (ValueError, "Must give an iteration to read parameters from")
actions = define_actions(FLAGS.action)
# Use the CPU if asked to
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.size))
sampling = True
model = create_model(sess, actions, sampling)
print("Model created")
# Load all the data
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir,
not FLAGS.omit_one_hot, FLAGS.train_on_euler)
# === Read and denormalize the gt with srnn's seeds, as we'll need them many times for evaluation in Euler Angles ===
srnn_gts_expmap = get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
not FLAGS.omit_one_hot,
from_exp=not FLAGS.train_on_euler,
to_euler=False)
srnn_gts_euler = get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
not FLAGS.omit_one_hot,
from_exp=not FLAGS.train_on_euler)
# Clean and create a new h5 file of samples
SAMPLES_FNAME = 'samples.h5'
try:
os.remove(SAMPLES_FNAME)
except OSError:
pass
# Predict and save for each action
for action in actions:
# Make prediction with srnn' seeds
action_prefix, action_postfix_input, action_postfix_output, action_poses = model.get_batch_srnn(
test_set, action, FLAGS.velocity)
forward_only = True
srnn_seeds = True
srnn_mse_loss_fw, _, srnn_poses = model.step(
sess, action_prefix[:, :, 6:], action_postfix_input[:, :, 6:],
action_postfix_output[:, :, 6:], action_poses[:, :, 6:],
forward_only, srnn_seeds)
srnn_poses = np.concatenate((np.transpose(
action_postfix_output[:, :, :6], [1, 0, 2]), srnn_poses),
axis=-1)
# denorm
srnn_pred = data_utils.revert_output_format(
srnn_poses, action_poses[:, FLAGS.seq_length_in - 1, :],
data_mean, data_std, dim_to_ignore, actions,
not FLAGS.omit_one_hot, FLAGS.velocity)
# change to expmap
if FLAGS.train_on_euler:
for i in np.arange(8):
for j in np.arange(srnn_pred[i].shape[0]):
for k in np.arange(3, 97, 3):
srnn_pred[i][j,
k:k + 3] = data_utils.rotmat2expmap(
data_utils.euler2rotmat(
srnn_pred[i][j, k:k + 3]))
# Save the samples
with h5py.File(SAMPLES_FNAME, 'a') as hf:
for i in np.arange(8):
# Save conditioning ground truth
node_name = 'expmap/gt/{1}_{0}'.format(i, action)
hf.create_dataset(node_name,
data=srnn_gts_expmap[action][i])
# Save prediction
node_name = 'expmap/preds/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=srnn_pred[i])
# Compute and save the errors here
mean_errors = np.zeros((len(srnn_pred), srnn_pred[0].shape[0]))
for i in np.arange(8):
eulerchannels_pred = srnn_pred[i]
# change back to euler
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 97, 3):
eulerchannels_pred[j,
k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k +
3]))
eulerchannels_pred[:, 0:6] = 0
# Pick only the dimensions with sufficient standard deviation. Others are ignored.
idx_to_use = np.where(np.std(eulerchannels_pred, 0) > 1e-4)[0]
euc_error = np.power(
srnn_gts_euler[action][i][:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
mean_mean_errors = np.mean(mean_errors, 0)
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 13, 24]:
if FLAGS.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]), end="")
else:
print(" n/a |", end="")
print()
with h5py.File(SAMPLES_FNAME, 'a') as hf:
node_name = 'mean_{0}_error'.format(action)
hf.create_dataset(node_name, data=mean_mean_errors)
return
def angles_error(srnn_poses_s, srnn_poses_t, srnn_poses_p, srnn_poses_m,
data_mean, data_std, dim_to_ignore, actions, number,
srnn_gts_euler, srnn_gts_expmap):
action = actions[0]
srnn_pred_expmap_s = data_utils.revert_output_format(
srnn_poses_s, data_mean, data_std, dim_to_ignore, actions)
srnn_pred_expmap_t = data_utils.revert_output_format(
srnn_poses_t, data_mean, data_std, dim_to_ignore, actions)
srnn_pred_expmap_p = data_utils.revert_output_format(
srnn_poses_p, data_mean, data_std, dim_to_ignore, actions)
srnn_pred_expmap_m = data_utils.revert_output_format(
srnn_poses_m, data_mean, data_std, dim_to_ignore, actions)
mean_errors_s = np.zeros(
(len(srnn_pred_expmap_s), srnn_pred_expmap_s[0].shape[0]))
mean_errors_t = np.zeros(
(len(srnn_pred_expmap_t), srnn_pred_expmap_t[0].shape[0]))
mean_errors_p = np.zeros(
(len(srnn_pred_expmap_p), srnn_pred_expmap_p[0].shape[0]))
mean_errors_m = np.zeros(
(len(srnn_pred_expmap_m), srnn_pred_expmap_m[0].shape[0]))
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred_s = srnn_pred_expmap_s[i]
eulerchannels_pred_t = srnn_pred_expmap_t[i]
eulerchannels_pred_p = srnn_pred_expmap_p[i]
eulerchannels_pred_m = srnn_pred_expmap_m[i]
# Convert from exponential map to Euler angles
for j in np.arange(eulerchannels_pred_s.shape[0]):
for k in np.arange(3, number, 3):
eulerchannels_pred_s[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(eulerchannels_pred_s[j, k:k + 3]))
eulerchannels_pred_t[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(eulerchannels_pred_t[j, k:k + 3]))
eulerchannels_pred_p[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(eulerchannels_pred_p[j, k:k + 3]))
eulerchannels_pred_m[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(eulerchannels_pred_m[j, k:k + 3]))
gt_i = np.copy(srnn_gts_euler[action][i])
gt_i[:, 0:6] = 0 # the translation is 0?
idx_to_use = np.where(np.std(gt_i, 0) > 1e-4)[0]
euc_error_s = np.power(
gt_i[:, idx_to_use] - eulerchannels_pred_s[:, idx_to_use], 2)
euc_error_t = np.power(
gt_i[:, idx_to_use] - eulerchannels_pred_t[:, idx_to_use], 2)
euc_error_p = np.power(
gt_i[:, idx_to_use] - eulerchannels_pred_p[:, idx_to_use], 2)
euc_error_m = np.power(
gt_i[:, idx_to_use] - eulerchannels_pred_m[:, idx_to_use], 2)
euc_error_s = np.sum(euc_error_s, 1)
euc_error_t = np.sum(euc_error_t, 1)
euc_error_p = np.sum(euc_error_p, 1)
euc_error_m = np.sum(euc_error_m, 1)
euc_error_s = np.sqrt(euc_error_s)
euc_error_t = np.sqrt(euc_error_t)
euc_error_p = np.sqrt(euc_error_p)
euc_error_m = np.sqrt(euc_error_m)
mean_errors_s[i, :] = euc_error_s
mean_errors_t[i, :] = euc_error_t
mean_errors_p[i, :] = euc_error_p
mean_errors_m[i, :] = euc_error_m
mean_mean_errors_s = np.mean(mean_errors_s, 0)
mean_mean_errors_t = np.mean(mean_errors_t, 0)
mean_mean_errors_p = np.mean(mean_errors_p, 0)
mean_mean_errors_m = np.mean(mean_errors_m, 0)
return mean_mean_errors_s, mean_mean_errors_t, mean_mean_errors_p, mean_mean_errors_m
def train():
"""Train a seq2seq model on human motion"""
actions = define_actions(FLAGS.action)
number_of_actions = len(actions)
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir,
not FLAGS.omit_one_hot)
# Limit TF to take a fraction of the GPU memory
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1)
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.size))
model = create_model(sess, actions)
model.train_writer.add_graph(sess.graph)
print("Model created")
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore,
not FLAGS.omit_one_hot)
#=== This is the training loop ===
step_time, loss, val_loss = 0.0, 0.0, 0.0
current_step = 0 if FLAGS.load <= 0 else FLAGS.load + 1
previous_losses = []
step_time, loss = 0, 0
for _ in xrange(FLAGS.iterations):
start_time = time.time()
# === Training step ===
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
train_set, not FLAGS.omit_one_hot)
_, step_loss, loss_summary, lr_summary = model.step(
sess, encoder_inputs, decoder_inputs, decoder_outputs, False)
model.train_writer.add_summary(loss_summary, current_step)
model.train_writer.add_summary(lr_summary, current_step)
if current_step % 10 == 0:
print("step {0:04d}; step_loss: {1:.4f}".format(
current_step, step_loss))
step_time += (time.time() - start_time) / FLAGS.test_every
loss += step_loss / FLAGS.test_every
current_step += 1
# === step decay ===
if current_step % FLAGS.learning_rate_step == 0:
sess.run(model.learning_rate_decay_op)
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.test_every == 0:
# === Validation with randomly chosen seeds ===
forward_only = True
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
test_set, not FLAGS.omit_one_hot)
step_loss, loss_summary = model.step(sess, encoder_inputs,
decoder_inputs,
decoder_outputs,
forward_only)
val_loss = step_loss # Loss book-keeping
model.test_writer.add_summary(loss_summary, current_step)
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 560, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
# === Validation with srnn's seeds ===
for action in actions:
# Evaluate the model on the test batches
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
srnn_loss, srnn_poses, _ = model.step(
sess, encoder_inputs, decoder_inputs, decoder_outputs,
True, True)
# Denormalize the output
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore,
actions, not FLAGS.omit_one_hot)
# Save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
# Training is done in exponential map, but the error is reported in
# Euler angles, as in previous work.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-247769197
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred = srnn_pred_expmap[i]
# Convert from exponential map to Euler angles
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 97, 3):
eulerchannels_pred[
j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k + 3]))
# The global translation (first 3 entries) and global rotation
# (next 3 entries) are also not considered in the error, so the_key
# are set to zero.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-249404882
gt_i = np.copy(srnn_gts_euler[action][i])
gt_i[:, 0:6] = 0
# Now compute the l2 error. The following is numpy port of the error
# function provided by Ashesh Jain (in matlab), available at
# https://github.com/asheshjain399/RNNexp/blob/srnn/structural_rnn/CRFProblems/H3.6m/dataParser/Utils/motionGenerationError.m#L40-L54
idx_to_use = np.where(np.std(gt_i, 0) > 1e-4)[0]
euc_error = np.power(
gt_i[:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
# This is simply the mean error over the N_SEQUENCE_TEST examples
mean_mean_errors = np.mean(mean_errors, 0)
# Pretty print of the results for 80, 160, 320, 400, 560 and 1000 ms
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 13, 24]:
if FLAGS.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]),
end="")
else:
print(" n/a |", end="")
print()
# Ugly massive if-then to log the error to tensorboard :shrug:
if action == "walking":
summaries = sess.run(
[
model.walking_err80_summary,
model.walking_err160_summary,
model.walking_err320_summary,
model.walking_err400_summary,
model.walking_err560_summary,
model.walking_err1000_summary
], {
model.walking_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.walking_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.walking_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.walking_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.walking_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.walking_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "eating":
summaries = sess.run(
[
model.eating_err80_summary,
model.eating_err160_summary,
model.eating_err320_summary,
model.eating_err400_summary,
model.eating_err560_summary,
model.eating_err1000_summary
], {
model.eating_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.eating_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.eating_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.eating_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.eating_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.eating_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "smoking":
summaries = sess.run(
[
model.smoking_err80_summary,
model.smoking_err160_summary,
model.smoking_err320_summary,
model.smoking_err400_summary,
model.smoking_err560_summary,
model.smoking_err1000_summary
], {
model.smoking_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.smoking_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.smoking_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.smoking_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.smoking_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.smoking_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "discussion":
summaries = sess.run(
[
model.discussion_err80_summary,
model.discussion_err160_summary,
model.discussion_err320_summary,
model.discussion_err400_summary,
model.discussion_err560_summary,
model.discussion_err1000_summary
], {
model.discussion_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.discussion_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.discussion_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.discussion_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.discussion_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.discussion_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "directions":
summaries = sess.run(
[
model.directions_err80_summary,
model.directions_err160_summary,
model.directions_err320_summary,
model.directions_err400_summary,
model.directions_err560_summary,
model.directions_err1000_summary
], {
model.directions_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.directions_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.directions_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.directions_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.directions_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.directions_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "greeting":
summaries = sess.run(
[
model.greeting_err80_summary,
model.greeting_err160_summary,
model.greeting_err320_summary,
model.greeting_err400_summary,
model.greeting_err560_summary,
model.greeting_err1000_summary
], {
model.greeting_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.greeting_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.greeting_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.greeting_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.greeting_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.greeting_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "phoning":
summaries = sess.run(
[
model.phoning_err80_summary,
model.phoning_err160_summary,
model.phoning_err320_summary,
model.phoning_err400_summary,
model.phoning_err560_summary,
model.phoning_err1000_summary
], {
model.phoning_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.phoning_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.phoning_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.phoning_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.phoning_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.phoning_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "posing":
summaries = sess.run(
[
model.posing_err80_summary,
model.posing_err160_summary,
model.posing_err320_summary,
model.posing_err400_summary,
model.posing_err560_summary,
model.posing_err1000_summary
], {
model.posing_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.posing_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.posing_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.posing_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.posing_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.posing_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "purchases":
summaries = sess.run(
[
model.purchases_err80_summary,
model.purchases_err160_summary,
model.purchases_err320_summary,
model.purchases_err400_summary,
model.purchases_err560_summary,
model.purchases_err1000_summary
], {
model.purchases_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.purchases_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.purchases_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.purchases_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.purchases_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.purchases_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "sitting":
summaries = sess.run(
[
model.sitting_err80_summary,
model.sitting_err160_summary,
model.sitting_err320_summary,
model.sitting_err400_summary,
model.sitting_err560_summary,
model.sitting_err1000_summary
], {
model.sitting_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.sitting_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.sitting_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.sitting_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.sitting_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.sitting_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "sittingdown":
summaries = sess.run(
[
model.sittingdown_err80_summary,
model.sittingdown_err160_summary,
model.sittingdown_err320_summary,
model.sittingdown_err400_summary,
model.sittingdown_err560_summary,
model.sittingdown_err1000_summary
], {
model.sittingdown_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.sittingdown_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.sittingdown_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.sittingdown_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.sittingdown_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.sittingdown_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "takingphoto":
summaries = sess.run(
[
model.takingphoto_err80_summary,
model.takingphoto_err160_summary,
model.takingphoto_err320_summary,
model.takingphoto_err400_summary,
model.takingphoto_err560_summary,
model.takingphoto_err1000_summary
], {
model.takingphoto_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.takingphoto_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.takingphoto_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.takingphoto_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.takingphoto_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.takingphoto_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "waiting":
summaries = sess.run(
[
model.waiting_err80_summary,
model.waiting_err160_summary,
model.waiting_err320_summary,
model.waiting_err400_summary,
model.waiting_err560_summary,
model.waiting_err1000_summary
], {
model.waiting_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.waiting_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.waiting_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.waiting_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.waiting_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.waiting_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "walkingdog":
summaries = sess.run(
[
model.walkingdog_err80_summary,
model.walkingdog_err160_summary,
model.walkingdog_err320_summary,
model.walkingdog_err400_summary,
model.walkingdog_err560_summary,
model.walkingdog_err1000_summary
], {
model.walkingdog_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.walkingdog_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.walkingdog_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.walkingdog_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.walkingdog_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.walkingdog_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
elif action == "walkingtogether":
summaries = sess.run(
[
model.walkingtogether_err80_summary,
model.walkingtogether_err160_summary,
model.walkingtogether_err320_summary,
model.walkingtogether_err400_summary,
model.walkingtogether_err560_summary,
model.walkingtogether_err1000_summary
], {
model.walkingtogether_err80:
mean_mean_errors[1]
if FLAGS.seq_length_out >= 2 else None,
model.walkingtogether_err160:
mean_mean_errors[3]
if FLAGS.seq_length_out >= 4 else None,
model.walkingtogether_err320:
mean_mean_errors[7]
if FLAGS.seq_length_out >= 8 else None,
model.walkingtogether_err400:
mean_mean_errors[9]
if FLAGS.seq_length_out >= 10 else None,
model.walkingtogether_err560:
mean_mean_errors[13]
if FLAGS.seq_length_out >= 14 else None,
model.walkingtogether_err1000:
mean_mean_errors[24]
if FLAGS.seq_length_out >= 25 else None
})
for i in np.arange(len(summaries)):
model.test_writer.add_summary(summaries[i],
current_step)
print()
print("============================\n"
"Global step: %d\n"
"Learning rate: %.4f\n"
"Step-time (ms): %.4f\n"
"Train loss avg: %.4f\n"
"--------------------------\n"
"Val loss: %.4f\n"
"srnn loss: %.4f\n"
"============================" %
(model.global_step.eval(), model.learning_rate.eval(),
step_time * 1000, loss, val_loss, srnn_loss))
print()
previous_losses.append(loss)
# Save the model
if current_step % FLAGS.save_every == 0:
print("Saving the model...")
start_time = time.time()
model.saver.save(sess,
os.path.normpath(
os.path.join(train_dir,
'checkpoint')),
global_step=current_step)
print("done in {0:.2f} ms".format(
(time.time() - start_time) * 1000))
# Reset global time and loss
step_time, loss = 0, 0
sys.stdout.flush()
def train():
"""Train a seq2seq model on human motion"""
actions = define_actions(args.action)
number_of_actions = len(actions)
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, args.seq_length_in, args.seq_length_out, args.data_dir,
not args.omit_one_hot)
# Limit TF to take a fraction of the GPU memory
if True:
model = create_model(actions, args.sample)
if not args.use_cpu:
model = model.cuda()
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore,
not args.omit_one_hot)
#=== This is the training loop ===
step_time, loss, val_loss = 0.0, 0.0, 0.0
current_step = 0 if args.load <= 0 else args.load + 1
previous_losses = []
step_time, loss = 0, 0
optimiser = optim.SGD(model.parameters(), lr=args.learning_rate)
#optimiser = optim.Adam(model.parameters(), lr=learning_rate, betas = (0.9, 0.999))
for _ in range(args.iterations):
optimiser.zero_grad()
model.train()
start_time = time.time()
# Actual training
# === Training step ===
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
train_set, not args.omit_one_hot)
encoder_inputs = torch.from_numpy(encoder_inputs).float()
decoder_inputs = torch.from_numpy(decoder_inputs).float()
decoder_outputs = torch.from_numpy(decoder_outputs).float()
if not args.use_cpu:
encoder_inputs = encoder_inputs.cuda()
decoder_inputs = decoder_inputs.cuda()
decoder_outputs = decoder_outputs.cuda()
encoder_inputs = Variable(encoder_inputs)
decoder_inputs = Variable(decoder_inputs)
decoder_outputs = Variable(decoder_outputs)
preds = model(encoder_inputs, decoder_inputs)
step_loss = (preds - decoder_outputs)**2
step_loss = step_loss.mean()
# Actual backpropagation
step_loss.backward()
optimiser.step()
step_loss = step_loss.cpu().data.numpy()
if current_step % 10 == 0:
print("step {0:04d}; step_loss: {1:.4f}".format(
current_step, step_loss))
step_time += (time.time() - start_time) / args.test_every
loss += step_loss / args.test_every
current_step += 1
# === step decay ===
if current_step % args.learning_rate_step == 0:
args.learning_rate = args.learning_rate * args.learning_rate_decay_factor
optimiser = optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(0.9, 0.999))
print("Decay learning rate. New value at " +
str(args.learning_rate))
#cuda.empty_cache()
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % args.test_every == 0:
model.eval()
# === Validation with randomly chosen seeds ===
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
test_set, not args.omit_one_hot)
encoder_inputs = torch.from_numpy(encoder_inputs).float()
decoder_inputs = torch.from_numpy(decoder_inputs).float()
decoder_outputs = torch.from_numpy(decoder_outputs).float()
if not args.use_cpu:
encoder_inputs = encoder_inputs.cuda()
decoder_inputs = decoder_inputs.cuda()
decoder_outputs = decoder_outputs.cuda()
encoder_inputs = Variable(encoder_inputs)
decoder_inputs = Variable(decoder_inputs)
decoder_outputs = Variable(decoder_outputs)
preds = model(encoder_inputs, decoder_inputs)
step_loss = (preds - decoder_outputs)**2
step_loss = step_loss.mean()
val_loss = step_loss # Loss book-keeping
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 560, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
# === Validation with srnn's seeds ===
for action in actions:
# Evaluate the model on the test batches
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
#### Evaluate model on action
encoder_inputs = torch.from_numpy(encoder_inputs).float()
decoder_inputs = torch.from_numpy(decoder_inputs).float()
decoder_outputs = torch.from_numpy(decoder_outputs).float()
if not args.use_cpu:
encoder_inputs = encoder_inputs.cuda()
decoder_inputs = decoder_inputs.cuda()
decoder_outputs = decoder_outputs.cuda()
encoder_inputs = Variable(encoder_inputs)
decoder_inputs = Variable(decoder_inputs)
decoder_outputs = Variable(decoder_outputs)
srnn_poses = model(encoder_inputs, decoder_inputs)
srnn_loss = (srnn_poses - decoder_outputs)**2
srnn_loss.cpu().data.numpy()
srnn_loss = srnn_loss.mean()
srnn_poses = srnn_poses.cpu().data.numpy()
srnn_poses = srnn_poses.transpose([1, 0, 2])
srnn_loss = srnn_loss.cpu().data.numpy()
# Denormalize the output
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore,
actions, not args.omit_one_hot)
# Save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
# Training is done in exponential map, but the error is reported in
# Euler angles, as in previous work.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-247769197
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred = srnn_pred_expmap[i]
# Convert from exponential map to Euler angles
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 97, 3):
eulerchannels_pred[
j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k + 3]))
# The global translation (first 3 entries) and global rotation
# (next 3 entries) are also not considered in the error, so the_key
# are set to zero.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-249404882
gt_i = np.copy(srnn_gts_euler[action][i])
gt_i[:, 0:6] = 0
# Now compute the l2 error. The following is numpy port of the error
# function provided by Ashesh Jain (in matlab), available at
# https://github.com/asheshjain399/RNNexp/blob/srnn/structural_rnn/CRFProblems/H3.6m/dataParser/Utils/motionGenerationError.m#L40-L54
idx_to_use = np.where(np.std(gt_i, 0) > 1e-4)[0]
euc_error = np.power(
gt_i[:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
# This is simply the mean error over the N_SEQUENCE_TEST examples
mean_mean_errors = np.mean(mean_errors, 0)
# Pretty print of the results for 80, 160, 320, 400, 560 and 1000 ms
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 13, 24]:
if args.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]),
end="")
else:
print(" n/a |", end="")
print()
print()
print("============================\n"
"Global step: %d\n"
"Learning rate: %.4f\n"
"Step-time (ms): %.4f\n"
"Train loss avg: %.4f\n"
"--------------------------\n"
"Val loss: %.4f\n"
"srnn loss: %.4f\n"
"============================" %
(current_step, args.learning_rate, step_time * 1000,
loss, val_loss, srnn_loss))
torch.save(model, train_dir + '/model_' + str(current_step))
print()
previous_losses.append(loss)
# Reset global time and loss
step_time, loss = 0, 0
sys.stdout.flush()
def evaluate():
"""Evaluations for srnn's seeds"""
actions = define_actions(FLAGS.action)
seq_length_out = 25
# Load all the data
parent, offset, rotInd, expmapInd = forward_kinematics._some_variables()
with h5py.File('samples.h5', 'r') as h5f:
# Predict and save for each action
for action in actions:
xyz_gt = []
xyz_pred = []
# Compute and save the errors here
mean_errors = []
for i in np.arange(8):
srnn_pred = h5f['expmap/preds/' + action + '_' +
str(i)][:seq_length_out, :]
srnn_gts = h5f['expmap/gt/' + action + '_' +
str(i)][:seq_length_out, :]
rotation_pred = forward_kinematics.revert_coordinate_space(
srnn_pred, np.eye(3), np.zeros(3))
rotation_gt = forward_kinematics.revert_coordinate_space(
srnn_gts, np.eye(3), np.zeros(3))
xyz_gt_tmp, xyz_pred_tmp = np.zeros(
(seq_length_out, 96)), np.zeros((seq_length_out, 96))
# Compute 3d points for each frame
for j in range(seq_length_out):
xyz_gt_tmp[j, :] = forward_kinematics.fkl(
rotation_gt[j, :], parent, offset, rotInd, expmapInd)
for j in range(seq_length_out):
xyz_pred_tmp[j, :] = forward_kinematics.fkl(
rotation_pred[j, :], parent, offset, rotInd, expmapInd)
xyz_gt.append(xyz_gt_tmp)
xyz_pred.append(xyz_pred_tmp)
# change back to euler to evaluate MAE
for j in np.arange(srnn_pred.shape[0]):
for k in np.arange(3, 97, 3):
srnn_gts[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(srnn_gts[j, k:k + 3]))
srnn_pred[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(srnn_pred[j, k:k + 3]))
srnn_pred[:, 0:6] = 0
# Pick only the dimensions with sufficient standard deviation. Others are ignored.
idx_to_use = np.where(np.std(srnn_pred, 0) > 1e-4)[0]
euc_error = np.power(
srnn_gts[:, idx_to_use] - srnn_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors.append(euc_error)
# MAE
mean_errors = np.array(mean_errors)
mean_mean_errors = np.mean(mean_errors, 0)
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 560, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 13, 24]:
if seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]), end="")
else:
print(" n/a |", end="")
print()
# MPJPE & PCK
xyz_gt = np.array(xyz_gt)
xyz_pred = np.array(xyz_pred)
# Evaluation using pck score
pck = np.zeros(seq_length_out)
for k in range(seq_length_out):
l2_norm, euclidean, pck[k], pck_allthreshold = distance_3D(
xyz_pred[:, k, :], xyz_gt[:, k, :])
print(
"Predict next %d frames, l2_norm: %f, MPJPE: %f, PCK: %f" %
(k + 1, l2_norm, euclidean, pck[k]))
if k == 24: # 1000ms # 9:400ms
print("All threshold:")
print(pck_allthreshold)
return
def sample():
"""Sample predictions for srnn's seeds"""
actions = define_actions(args.action)
if True:
# === Create the model ===
print("Creating %d layers of %d units." % (args.num_layers, args.size))
sampling = True
model = create_model(actions, sampling)
if not args.use_cpu:
model = model.cuda()
print("Model created")
# Load all the data
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, args.seq_length_in, args.seq_length_out, args.data_dir,
not args.omit_one_hot)
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_expmap = get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
not args.omit_one_hot,
to_euler=False)
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore,
not args.omit_one_hot)
# Clean and create a new h5 file of samples
SAMPLES_FNAME = 'samples.h5'
try:
os.remove(SAMPLES_FNAME)
except OSError:
pass
# Predict and save for each action
for action in actions:
# Make prediction with srnn' seeds
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
encoder_inputs = torch.from_numpy(encoder_inputs).float()
decoder_inputs = torch.from_numpy(decoder_inputs).float()
decoder_outputs = torch.from_numpy(decoder_outputs).float()
if not args.use_cpu:
encoder_inputs = encoder_inputs.cuda()
decoder_inputs = decoder_inputs.cuda()
decoder_outputs = decoder_outputs.cuda()
encoder_inputs = Variable(encoder_inputs)
decoder_inputs = Variable(decoder_inputs)
decoder_outputs = Variable(decoder_outputs)
srnn_poses = model(encoder_inputs, decoder_inputs)
srnn_loss = (srnn_poses - decoder_outputs)**2
srnn_loss.cpu().data.numpy()
srnn_loss = srnn_loss.mean()
srnn_poses = srnn_poses.cpu().data.numpy()
srnn_poses = srnn_poses.transpose([1, 0, 2])
srnn_loss = srnn_loss.cpu().data.numpy()
# denormalizes too
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore, actions,
not args.omit_one_hot)
# Save the samples
with h5py.File(SAMPLES_FNAME, 'a') as hf:
for i in np.arange(8):
# Save conditioning ground truth
node_name = 'expmap/gt/{1}_{0}'.format(i, action)
hf.create_dataset(node_name,
data=srnn_gts_expmap[action][i])
# Save prediction
node_name = 'expmap/preds/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=srnn_pred_expmap[i])
# Compute and save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
for i in np.arange(8):
eulerchannels_pred = srnn_pred_expmap[i]
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 97, 3):
eulerchannels_pred[j,
k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k +
3]))
eulerchannels_pred[:, 0:6] = 0
# Pick only the dimensions with sufficient standard deviation. Others are ignored.
idx_to_use = np.where(np.std(eulerchannels_pred, 0) > 1e-4)[0]
euc_error = np.power(
srnn_gts_euler[action][i][:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
mean_mean_errors = np.mean(mean_errors, 0)
print(action)
print(','.join(map(str, mean_mean_errors.tolist())))
with h5py.File(SAMPLES_FNAME, 'a') as hf:
node_name = 'mean_{0}_error'.format(action)
hf.create_dataset(node_name, data=mean_mean_errors)
return
def sample():
actions = define_actions(
FLAGS.action
) # here is like a list of actions, eg, ['walking'], ['walking, waiting, directions']
_, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir)
number = 97
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=1
) # Allowing GPU memory growth, this means whole GPU
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
device_count=device_count)) as sess:
model = create_model(sess, actions, optimizer_to_use=FLAGS.opt)
print("Model created")
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore)
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [
80, 160, 240, 320, 400, 480, 560, 640, 720, 800, 880, 960, 1000
]:
print(" {0:5d} |".format(ms), end="")
print()
# # === Validation with srnn's seeds === # The only difference is how to choose the seed?
action = actions[0]
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
srnn_loss, srnn_poses, srnn_poses_s, srnn_poses_t, _ = model.step_test(
False,
sess,
encoder_inputs,
decoder_inputs,
decoder_outputs,
sampling_rate=0.0)
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore, actions)
# Save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred = srnn_pred_expmap[i]
# Convert from exponential map to Euler angles
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, number, 3):
eulerchannels_pred[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(eulerchannels_pred[j,
k:k + 3]))
gt_i = np.copy(srnn_gts_euler[action][i])
gt_i[:, 0:6] = 0 # the translation is 0?
idx_to_use = np.where(np.std(gt_i, 0) > 1e-4)[0]
euc_error = np.power(
gt_i[:, idx_to_use] - eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
# This is simply the mean error over the N_SEQUENCE_TEST examples
mean_mean_errors = np.mean(mean_errors, 0)
# GX: here I think 1000ms for 25 frames, so that you calculate error on one single frame
print("{0: <8}temporal |".format(action), end="")
for ms in [1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 24]:
if FLAGS.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]), end="")
else:
print(" n/a |", end="")
print()
sampled_error = np.mean(mean_mean_errors)
print("temporal output mean error is: ", sampled_error)
print()
def sample():
"""Sample predictions for srnn's seeds"""
if FLAGS.load <= 0:
raise (ValueError, "Must give an iteration to read parameters from")
actions = define_actions(FLAGS.action)
# Use the CPU if asked to
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.size))
sampling = True
model = create_model(sess, actions, sampling)
print("Model created")
# Load all the data
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir,
not FLAGS.omit_one_hot)
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_expmap = get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
not FLAGS.omit_one_hot,
to_euler=False)
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore,
not FLAGS.omit_one_hot)
# Clean and create a new h5 file of samples
SAMPLES_FNAME = 'samples.h5'
try:
os.remove(SAMPLES_FNAME)
except OSError:
pass
# Predict and save for each action
for action in actions:
# Make prediction with srnn' seeds
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
forward_only = True
srnn_seeds = True
srnn_loss, srnn_poses, _ = model.step(sess, encoder_inputs,
decoder_inputs,
decoder_outputs,
forward_only, srnn_seeds)
# denormalizes too
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore, actions,
not FLAGS.omit_one_hot)
# Save the conditioning seeds
# Save the samples
with h5py.File(SAMPLES_FNAME, 'a') as hf:
for i in np.arange(8):
# Save conditioning ground truth
node_name = 'expmap/gt/{1}_{0}'.format(i, action)
hf.create_dataset(node_name,
data=srnn_gts_expmap[action][i])
# Save prediction
node_name = 'expmap/preds/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=srnn_pred_expmap[i])
# Compute and save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
for i in np.arange(8):
eulerchannels_pred = srnn_pred_expmap[i]
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 97, 3):
eulerchannels_pred[j,
k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k +
3]))
eulerchannels_pred[:, 0:6] = 0
# Pick only the dimensions with sufficient standard deviation. Others are ignored.
idx_to_use = np.where(np.std(eulerchannels_pred, 0) > 1e-4)[0]
euc_error = np.power(
srnn_gts_euler[action][i][:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
mean_mean_errors = np.mean(mean_errors, 0)
print(action)
print(','.join(map(str, mean_mean_errors.tolist())))
with h5py.File(SAMPLES_FNAME, 'a') as hf:
node_name = 'mean_{0}_error'.format(action)
hf.create_dataset(node_name, data=mean_mean_errors)
return
def train():
actions = define_actions(FLAGS.action)
number_of_actions = len(actions)
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir, not FLAGS.omit_one_hot )
print("Creating %d layers of %d units." % (FLAGS.num_layers, FLAGS.size))
model = create_model(actions,sampling=False)
model.to(device)
print("Model created")
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_euler = get_srnn_gts( actions, model, test_set, data_mean,
data_std, dim_to_ignore, not FLAGS.omit_one_hot )
#=== This is the training loop ===
step_time, loss, val_loss = 0.0, 0.0, 0.0
current_step = 0 if FLAGS.load <= 0 else FLAGS.load + 1
previous_losses = []
step_time, loss = 0, 0
lr = FLAGS.learning_rate
optimizer = torch.optim.SGD(model.parameters(), lr=lr)
for _ in xrange(FLAGS.iterations):
start_time = time.time()
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(train_set, not FLAGS.omit_one_hot )
model.train()
output, _ = model(transform(encoder_inputs), transform(decoder_inputs))
optimizer.zero_grad()
step_loss = model.loss(output[:,:,:model.HUMAN_SIZE], transform(decoder_outputs)[:,:,:model.HUMAN_SIZE])
step_loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(),FLAGS.max_gradient_norm)
optimizer.step()
if current_step % FLAGS.show_every == 0:
print("step {0:04d}; step_loss: {1:.4f}".format(current_step, step_loss ))
step_time += (time.time() - start_time) / FLAGS.test_every
loss += step_loss / FLAGS.test_every
current_step += 1
## step decay ##
if current_step % FLAGS.learning_rate_step == 0:
lr *= FLAGS.learning_rate_decay_factor
for g in optimizer.param_groups:
g['lr'] = lr
## Validation step ##
if current_step % FLAGS.test_every == 0:
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(test_set, not FLAGS.omit_one_hot )
##TODO: a forward pass
model.eval()
output, _ = model(transform(encoder_inputs), transform(decoder_inputs))
step_loss = model.loss(output[:,:,:model.HUMAN_SIZE],transform(decoder_outputs)[:,:,:model.HUMAN_SIZE])
val_loss = step_loss
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 560, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
# === Validation with srnn's seeds ===
for action in actions:
# Evaluate the model on the test batches
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(test_set, action)
srnn_poses, _ = model(transform(encoder_inputs), transform(decoder_inputs))
srnn_loss = model.loss(srnn_poses[:,:,:model.HUMAN_SIZE],transform(decoder_outputs)[:,:,:model.HUMAN_SIZE])
# Denormalize the output
srnn_pred_expmap = data_utils.revert_output_format(srnn_poses.cpu().detach().numpy(),
data_mean, data_std, dim_to_ignore, actions, not FLAGS.omit_one_hot )
# Save the errors here
mean_errors = np.zeros( (len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]) )
# Training is done in exponential map, but the error is reported in
# Euler angles, as in previous work.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-247769197
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred = srnn_pred_expmap[i]
# Convert from exponential map to Euler angles
for j in np.arange( eulerchannels_pred.shape[0] ):
for k in np.arange(3,97,3):
eulerchannels_pred[j,k:k+3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat( eulerchannels_pred[j,k:k+3] ))
# The global translation (first 3 entries) and global rotation
# (next 3 entries) are also not considered in the error, so the_key
# are set to zero.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-249404882
gt_i=np.copy(srnn_gts_euler[action][i])
gt_i[:,0:6] = 0
# Now compute the l2 error. The following is numpy port of the error
# function provided by Ashesh Jain (in matlab), available at
# https://github.com/asheshjain399/RNNexp/blob/srnn/structural_rnn/CRFProblems/H3.6m/dataParser/Utils/motionGenerationError.m#L40-L54
idx_to_use = np.where( np.std( gt_i, 0 ) > 1e-4 )[0]
euc_error = np.power( gt_i[:,idx_to_use] - eulerchannels_pred[:,idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt( euc_error )
mean_errors[i,:] = euc_error
# This is simply the mean error over the N_SEQUENCE_TEST examples
mean_mean_errors = np.mean( mean_errors, 0 )
# Pretty print of the results for 80, 160, 320, 400, 560 and 1000 ms
print("{0: <16} |".format(action), end="")
for ms in [1,3,7,9,13,24]:
if FLAGS.seq_length_out >= ms+1:
print(" {0:.3f} |".format( mean_mean_errors[ms] ), end="")
else:
print(" n/a |", end="")
print()
print()
print("============================\n"
"Global step: %d\n"
"Learning rate: %.4f\n"
"Step-time (ms): %.4f\n"
"Train loss avg: %.4f\n"
"--------------------------\n"
"Val loss: %.4f\n"
"srnn loss: %.4f\n"
"============================" % (current_step,
lr, step_time*1000, loss,
val_loss, srnn_loss))
print()
previous_losses.append(loss)
# Save the model
if current_step % FLAGS.save_every == 0:
print( "Saving the model..." ); start_time = time.time()
torch.save(model.state_dict(), os.path.normpath(os.path.join(train_dir, 'checkpoint-{0}.pt'.format(current_step))))
print( "done in {0:.2f} ms".format( (time.time() - start_time)*1000) )
# Reset global time and loss
step_time, loss = 0, 0
sys.stdout.flush()
