def denormalize_and_convert_to_euler( data, data_mean, data_std, dim_to_ignore, actions, one_hot ):
"""
Denormalizes data and converts to Euler angles
(all losses are computed on Euler angles).
Args
data: dictionary with human poses.
data_mean: d-long vector with the mean of the training data.
data_std: d-long vector with the standard deviation of the training data.
dim_to_ignore: dimensions to ignore because the std is too small or for other reasons.
actions: list of strings with the actions in the data dictionary.
one_hot: whether the data comes with one-hot encoding.
Returns
all_denormed: a list with nbatch entries. Each entry is an n-by-d matrix
that corresponds to a denormalized sequence in Euler angles
"""
all_denormed = []
# expmap -> rotmat -> euler
for i in np.arange( data.shape[0] ):
denormed = data_utils.unNormalizeData(data[i,:,:], data_mean, data_std, dim_to_ignore, actions, one_hot )
for j in np.arange( denormed.shape[0] ):
for k in np.arange(3,115,3):
denormed[j,k:k+3] = data_utils.rotmat2euler( data_utils.expmap2rotmat( denormed[j,k:k+3] ))
all_denormed.append( denormed )
return all_denormed
def compute_test_error(self, action, pred_pose, srnn_gts_expmap, srnn_gts_euler, one_hot, samples_fname):
"""
Compute the test error
"""
pred_pose = np.squeeze(pred_pose)
predict_expmap = data_utils_cmu.revert_output_format(pred_pose, self.data_mean, self.data_std,
self.dim_to_ignore,
self.actions, one_hot)
mean_errors = np.zeros((len(predict_expmap), predict_expmap[0].shape[0]))
for i in np.arange(8):
eulerchannels_pred = np.copy(predict_expmap[i])
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(0, 115, 3):
idx = [k, k + 1, k + 2]
eulerchannels_pred[j, idx] = data_utils_cmu.rotmat2euler(
data_utils_cmu.expmap2rotmat(predict_expmap[i][j, idx]))
eulerchannels_pred[:, 0:6] = 0
srnn_gts_euler[action][i][:, 0:6] = 0
idx_to_use = np.where(np.std(srnn_gts_euler[action][i], 0) > 1e-4)[0]
euc_error = np.power(srnn_gts_euler[action][i][50:, idx_to_use] - eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
mean_mean_errors = np.mean(mean_errors, 0)
print()
print(action)
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 560, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 13, 24]:
if self.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]), end="")
else:
print(" n/a |", end="")
print()
with h5py.File(samples_fname, 'a') as hf:
for i in np.arange(8):
node_name = 'expmap/gt/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=srnn_gts_expmap[action][i])
node_name = 'expmap/preds/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=predict_expmap[i])
node_name = 'mean_{0}_error'.format(action)
hf.create_dataset(node_name, data=mean_mean_errors)
def get_srnn_gts(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
one_hot,
to_euler=True):
"""
Get the ground truths for srnn's sequences, and convert to Euler angles.
(the error is always computed in Euler angles).
Args
actions: a list of actions to get ground truths for.
model: training model we are using (we only use the "get_batch" method).
test_set: dictionary with normalized training data.
data_mean: d-long vector with the mean of the training data.
data_std: d-long vector with the standard deviation of the training data.
dim_to_ignore: dimensions that we are not using to train/predict.
one_hot: whether the data comes with one-hot encoding indicating action.
to_euler: whether to convert the angles to Euler format or keep thm in exponential map
Returns
srnn_gts_euler: a dictionary where the keys are actions, and the values
are the ground_truth, denormalized expected outputs of srnns's seeds.
"""
srnn_gts_euler = {}
for action in actions:
srnn_gt_euler = []
_, _, srnn_expmap = model.get_batch_srnn(test_set, action)
# print(srnn_expmap.shape)
# expmap -> rotmat -> euler
for i in np.arange(srnn_expmap.shape[0]):
denormed = data_utils.unNormalizeData(srnn_expmap[i, :, :],
data_mean, data_std,
dim_to_ignore, actions,
one_hot)
if to_euler:
for j in np.arange(denormed.shape[0]):
for k in np.arange(3, 115, 3):
denormed[j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(denormed[j, k:k + 3]))
srnn_gt_euler.append(denormed)
# Put back in the dictionary
srnn_gts_euler[action] = srnn_gt_euler
return srnn_gts_euler
def sample():
"""Sample predictions for srnn's seeds"""
if FLAGS.load <= 0:
raise (ValueError, "Must give an iteration to read parameters from")
actions = define_actions(FLAGS.action)
# Use the CPU if asked to
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.size))
sampling = True
model = create_model(sess, actions, sampling)
print("Model created")
# Load all the data
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir,
not FLAGS.omit_one_hot)
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_expmap = get_srnn_gts_sample(actions,
model,
test_set,
data_mean,
data_std,
dim_to_ignore,
not FLAGS.omit_one_hot,
to_euler=False)
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore,
not FLAGS.omit_one_hot)
# Clean and create a new h5 file of samples
SAMPLES_FNAME = 'samples.h5'
try:
os.remove(SAMPLES_FNAME)
except OSError:
pass
# Predict and save for each action
for action in actions:
# Make prediction with srnn' seeds
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
forward_only = True
srnn_seeds = True
srnn_loss, srnn_poses, _, spatial_weight = model.step(
sess, encoder_inputs, decoder_inputs, decoder_outputs,
forward_only, srnn_seeds)
# denormalizes too
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore, actions,
not FLAGS.omit_one_hot)
# Save the conditioning seeds
# Save the samples
with h5py.File(SAMPLES_FNAME, 'a') as hf:
for i in np.arange(8):
# Save conditioning ground truth
node_name = 'expmap/gt/{1}_{0}'.format(i, action)
hf.create_dataset(node_name,
data=srnn_gts_expmap[action][i])
# Save prediction
node_name = 'expmap/preds/{1}_{0}'.format(i, action)
hf.create_dataset(node_name, data=srnn_pred_expmap[i])
# Compute and save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
for i in np.arange(8):
eulerchannels_pred = srnn_pred_expmap[i]
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 115, 3):
eulerchannels_pred[j,
k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k +
3]))
eulerchannels_pred[:, 0:6] = 0
# Pick only the dimensions with sufficient standard deviation. Others are ignored.
idx_to_use = np.where(np.std(eulerchannels_pred, 0) > 1e-4)[0]
euc_error = np.power(
srnn_gts_euler[action][i][:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
mean_mean_errors = np.mean(mean_errors, 0)
print(action)
print(','.join(map(str, mean_mean_errors.tolist())))
with h5py.File(SAMPLES_FNAME, 'a') as hf:
node_name = 'mean_{0}_error'.format(action)
hf.create_dataset(node_name, data=mean_mean_errors)
return
def train():
"""Train a seq2seq model on human motion"""
actions = define_actions(FLAGS.action)
number_of_actions = len(actions)
train_set, test_set, data_mean, data_std, dim_to_ignore, dim_to_use = read_all_data(
actions, FLAGS.seq_length_in, FLAGS.seq_length_out, FLAGS.data_dir,
not FLAGS.omit_one_hot)
# Limit TF to take a fraction of the GPU memory
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1)
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
device_count=device_count)) as sess:
# === Create the model ===
print("Creating %d layers of %d units." %
(FLAGS.num_layers, FLAGS.size))
model = create_model(sess, actions)
model.train_writer.add_graph(sess.graph)
print("Model created")
# === Read and denormalize the gt with srnn's seeds, as we'll need them
# many times for evaluation in Euler Angles ===
srnn_gts_euler = get_srnn_gts(actions, model, test_set, data_mean,
data_std, dim_to_ignore,
not FLAGS.omit_one_hot)
#=== This is the training loop ===
step_time, loss, val_loss = 0.0, 0.0, 0.0
current_step = 0 if FLAGS.load <= 0 else FLAGS.load + 1
previous_losses = []
step_time, loss = 0, 0
# Init Validation variable for the whole model
minimized_error = 0.0
recorded_step = 0
for _ in xrange(FLAGS.iterations):
start_time = time.time()
# === Training step ===
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
train_set, not FLAGS.omit_one_hot)
_, step_loss, loss_summary, lr_summary, main_loss = model.step(
sess, encoder_inputs, decoder_inputs, decoder_outputs, False)
model.train_writer.add_summary(loss_summary, current_step)
model.train_writer.add_summary(lr_summary, current_step)
if current_step % 10 == 0:
print("step {0:04d}; step_loss: {1:.4f}".format(
current_step, step_loss))
step_time += (time.time() - start_time) / FLAGS.test_every
loss += step_loss / FLAGS.test_every
current_step += 1
# === step decay ===
if current_step % FLAGS.learning_rate_step == 0:
sess.run(model.learning_rate_decay_op)
if current_step % FLAGS.loss_weight_step == 0:
sess.run(model.sub_loss_weight_decay_op)
# Once in a while, we save checkpoint, print statistics, and run evals.
if current_step % FLAGS.test_every == 0:
# === Validation with randomly chosen seeds ===
forward_only = True
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch(
test_set, not FLAGS.omit_one_hot)
step_loss, loss_summary = model.step(sess, encoder_inputs,
decoder_inputs,
decoder_outputs,
forward_only)
val_loss = step_loss # Loss book-keeping
model.test_writer.add_summary(loss_summary, current_step)
print()
print("{0: <16} |".format("milliseconds"), end="")
for ms in [80, 160, 320, 400, 1000]:
print(" {0:5d} |".format(ms), end="")
print()
# Init the variable for accmulated error
accumulated_error = 0.0
mean_mean_errors_avg = {}
for i in range(0, 25):
mean_mean_errors_avg[i] = 0
# === Validation with srnn's seeds ===
for action in actions:
# Evaluate the model on the test batches
encoder_inputs, decoder_inputs, decoder_outputs = model.get_batch_srnn(
test_set, action)
srnn_loss, srnn_poses, _ = model.step(
sess, encoder_inputs, decoder_inputs, decoder_outputs,
True, True)
# Denormalize the output
srnn_pred_expmap = data_utils.revert_output_format(
srnn_poses, data_mean, data_std, dim_to_ignore,
actions, not FLAGS.omit_one_hot)
# Save the errors here
mean_errors = np.zeros(
(len(srnn_pred_expmap), srnn_pred_expmap[0].shape[0]))
# Training is done in exponential map, but the error is reported in
# Euler angles, as in previous work.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-247769197
N_SEQUENCE_TEST = 8
for i in np.arange(N_SEQUENCE_TEST):
eulerchannels_pred = srnn_pred_expmap[i]
# Convert from exponential map to Euler angles
for j in np.arange(eulerchannels_pred.shape[0]):
for k in np.arange(3, 115, 3):
eulerchannels_pred[
j, k:k + 3] = data_utils.rotmat2euler(
data_utils.expmap2rotmat(
eulerchannels_pred[j, k:k + 3]))
# The global translation (first 3 entries) and global rotation
# (next 3 entries) are also not considered in the error, so the_key
# are set to zero.
# See https://github.com/asheshjain399/RNNexp/issues/6#issuecomment-249404882
gt_i = np.copy(srnn_gts_euler[action][i])
gt_i[:, 0:6] = 0
# Now compute the l2 error.
idx_to_use = np.where(np.std(gt_i, 0) > 1e-4)[0]
euc_error = np.power(
gt_i[:, idx_to_use] -
eulerchannels_pred[:, idx_to_use], 2)
euc_error = np.sum(euc_error, 1)
euc_error = np.sqrt(euc_error)
mean_errors[i, :] = euc_error
# This is simply the mean error over the N_SEQUENCE_TEST examples
mean_mean_errors = np.mean(mean_errors, 0)
accumulated_error += np.mean(mean_mean_errors)
for ms in range(0, 25):
mean_mean_errors_avg[ms] += mean_mean_errors[ms]
# Pretty print of the results for 80, 160, 320, 400, 560 and 1000 ms
print("{0: <16} |".format(action), end="")
for ms in [1, 3, 7, 9, 24]:
if FLAGS.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors[ms]),
end="")
# mean_mean_errors_avg[ms] += mean_mean_errors[ms]
else:
print(" n/a |", end="")
print()
for i in range(0, FLAGS.seq_length_out):
mean_mean_errors_avg[
i] = mean_mean_errors_avg[i] / number_of_actions
# Pretty print of the average results for 80, 160, 320, 400, 560 and 1000 ms
print("{0: <16} |".format("average"), end="")
for ms in [1, 3, 7, 9, 24]:
if FLAGS.seq_length_out >= ms + 1:
print(" {0:.3f} |".format(mean_mean_errors_avg[ms]),
end="")
else:
print(" n/a |", end="")
print()
print()
print("============================\n"
"Global step: %d\n"
"Learning rate: %.4f\n"
"Step-time (ms): %.4f\n"
"Train loss avg: %.4f\n"
"--------------------------\n"
"Val loss: %.4f\n"
"srnn loss: %.4f\n"
"============================" %
(model.global_step.eval(), model.learning_rate.eval(),
step_time * 1000, loss, val_loss, srnn_loss))
print()
# Update minimized_error and recorded step
if (accumulated_error < minimized_error) or (minimized_error
== 0.0):
minimized_error = accumulated_error
recorded_step = model.global_step.eval()
print("Updated Error:{0}, step:{1}".format(
minimized_error, recorded_step))
# Save the model
if current_step % FLAGS.save_every == 0:
print("Saving the model...")
start_time = time.time()
model.saver.save(sess,
os.path.normpath(
os.path.join(
train_dir, 'checkpoint')),
global_step=current_step)
print("done in {0:.2f} ms".format(
(time.time() - start_time) * 1000))
else:
print("Error:{0}".format(accumulated_error))
previous_losses.append(loss)
# Reset global time and loss
step_time, loss = 0, 0
sys.stdout.flush()
print()
print("Minimized_error:{0}, on step:{1}".format(
minimized_error, recorded_step))
return minimized_error, recorded_step