def main():
parent = _some_variables()
# numpy implementation
real_person = np.load('sample.npz')['real_person']
real_person = np.reshape(real_person, [real_person.shape[0], -1])
character = np.load('sample.npz')['character']
character = np.reshape(character, [character.shape[0], -1])
nframes_condition = real_person.shape[0] - character.shape[0]
nframes_pred = character.shape[0]
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
if nframes_condition > 0:
for i in range(nframes_condition):
ob.update(fkl_relative(real_person[i], parent))
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
# Plot the prediction
for i in range(nframes_pred):
ob.update(fkl_relative(real_person[nframes_condition + i], parent),
fkl_relative(character[i], parent))
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
action = "phoning"
batch = 5
# numpy implementatio
# S_1.0_(6,6)
with h5py.File('samples.h5', 'r') as h5f:
# with h5py.File( 'samples.h5', 'r' ) as h5f:
expmap_gt = h5f['expmap/gt/' + action + '_' + str(batch)][:]
expmap_pred = h5f['expmap/preds/' + action + '_' + str(batch)][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
print(expmap_gt.shape, expmap_pred.shape)
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob.update(xyz_gt[i, :], weights=[])
plt.show(block=False)
fig.canvas.draw()
plt.axis('off')
# if i % 2== 0:
# plt.savefig(path+"/frame_gt{}.png".format(str(int(i/2))),bbox_inches='tight')
plt.pause(0.01)
# Plot the prediction
for i in range(nframes_pred):
ob.update(xyz_pred[i, :],
weights=[],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
plt.axis('off')
# if i % 2 == 0:
# plt.savefig(path+"/frame_pr{}.png".format(str(int(i/2))),bbox_inches='tight')
plt.pause(0.01)
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
## numpy implementation
#with h5py.File( 'samples.h5', 'r' ) as h5f:
# expmap_gt = h5f['expmap/gt/walking_0'][:]
# expmap_pred = h5f['expmap/preds/walking_0'][:]
# Read data from .txt file
data_dir = "./data/h3.6m/dataset"
test_subject_ids = [5]
actions = [sys.argv[1]]
one_hot = False
test_set, _ = data_utils.load_data(data_dir, test_subject_ids, actions,
one_hot)
subject = 5
subaction = 2
expmap_gt = test_set[(subject, sys.argv[1], subaction, 'even')]
# print( expmap_gt, expmap_gt.shape )
expmap_pred = np.zeros_like(expmap_gt)
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob.update(xyz_gt[i, :])
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
# Plot the prediction
for i in range(nframes_pred):
ob.update(xyz_pred[i, :], lcolor="#9b59b6", rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
def main():
# Load all the data
parser = argparse.ArgumentParser(description='Process some integers.')
# parser.add_argument('--model', type=str, default='srnn')
parser.add_argument('--model',
type=str,
choices=['erd', 'lstm3lr', 'srnn'])
parser.add_argument('--action',
type=str,
choices=['walking', 'smoking', 'eating', 'discussion'])
args = parser.parse_args()
parent, offset, rotInd, expmapInd = _some_variables()
# numpy implementation
with h5py.File('Motion/{0}_{1}.h5'.format(args.model, args.action),
'r') as h5f:
# expmap_gt = h5f['expmap/gt/walking_0'][:]
# expmap_pred = h5f['expmap/preds/walking_0'][:]
expmap_gt = h5f['expmap/gts/{0}_0'.format(args.action)][:]
expmap_pred = h5f['expmap/preds/{0}_0'.format(args.action)][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[
0] # 100x99
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob.update(xyz_gt[i, :])
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
# Plot the prediction
for i in range(nframes_pred):
ob.update(xyz_pred[i, :], lcolor="#9b59b6", rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
# numpy implementation
with h5py.File('samples.h5', 'r') as h5f:
expmap_gt = h5f['expmap/gt/walking_0'][:]
expmap_pred = h5f['expmap/preds/walking_0'][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# debug_here()
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
# === Plot and animate ===
import matplotlib.animation as manimation
FFMpegWriter = manimation.writers['ffmpeg']
metadata = dict(title='Movie Test',
artist='Matplotlib',
comment='Movie support!')
writer = FFMpegWriter(fps=15, metadata=metadata)
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
with writer.saving(fig, 'mocap_pred_anim.mp4', 100):
for i in range(nframes_gt):
ob.update(xyz_gt[i, :])
# plt.show(block=False)
# fig.canvas.draw()
writer.grab_frame()
# plt.pause(0.01)
# Plot the prediction
for i in range(nframes_pred):
ob.update(xyz_pred[i, :], lcolor="#9b59b6", rcolor="#2ecc71")
# plt.show(block=False)
# fig.canvas.draw()
writer.grab_frame()
def run(data_utils):
for action in data_utils.actions:
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
# numpy implementation
with h5py.File('samples.h5', 'r') as h5f:
# boe
expmap_gt = h5f['expmap/gt/' + action + '_0'][:]
expmap_pred = h5f['expmap/preds/' + action + '_0'][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt, expmap_pred)), np.eye(3), np.zeros(3),
data_utils)
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros(
(nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd,
expmapInd, data_utils)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd, data_utils)
# === Plot and animate ===
fig = plt.figure(action)
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob.update(xyz_gt[i, :])
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
# Plot the prediction
for i in range(nframes_pred):
ob.update(xyz_pred[i, :], lcolor="#9b59b6", rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
def main():
xyz_gt = np.load('xyz_gt.npy') # xyz,joint,frame
frame_number = xyz_gt.shape[0]
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
for i in range(frame_number):
ob.update(xyz_gt[i, :])
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
def animate(xyz):
'''
Animate motion in euclidean
Borrowed from
https://github.com/una-dinosauria/human-motion-prediction/blob/master/src/forward_kinematics.py#L156
'''
import viz
import matplotlib.pyplot as plt
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
for i in range(xyz.shape[0]):
ob.update( xyz[i,:] )
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.002)
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
# numpy implementation
# with h5py.File( 'samples.h5', 'r' ) as h5f:
expmap_gt = genfromtxt(sys.argv[1], delimiter=',')
expmap_pred = np.ones(
np.shape(expmap_gt)) #genfromtxt(sys.argv[2], delimiter=',')
print(np.shape(expmap_pred))
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
plt.hold(True)
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob.update(xyz_gt[i, :])
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.00001)
def main():
def read_mat(mat_path):
mat = scipy.io.loadmat(mat_path)
#a=mat['position'].transpose(2,1,0)
a = mat['position']
return a # ndarray type
xyz_info = read_mat(file) # xyz,joint,frame
frame_number = xyz_info.shape[2] # get frame number of current file
xyz_gt = xyz_info.transpose(2, 1, 0)
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
for i in range(frame_number):
ob.update(xyz_gt[i, :])
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.01)
def create_movie():
actions = define_actions("All")
rcams, vcams = cameras.load_cameras('cameras.h5', [1, 5, 6, 7, 8, 9, 11])
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=1)
device_count = {"GPU": 0} if FLAGS.use_cpu else {"GPU": 1}
os.system('mkdir -p ' + FLAGS.output_dir)
train_set_3d, test_set_3d, data_mean_3d, data_std_3d, dim_to_ignore_3d, dim_to_use_3d, train_root_positions, test_root_positions, offsets_train, offsets_test = data_utils.read_3d_data(
actions, FLAGS.data_dir, FLAGS.camera_frame, rcams, vcams)
if (FLAGS.use_sh):
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.read_2d_predictions(
actions, FLAGS.data_dir)
else:
train_set_2d, test_set_2d, data_mean_2d, data_std_2d, dim_to_ignore_2d, dim_to_use_2d = data_utils.create_2d_data(
actions, FLAGS.data_dir, rcams, vcams)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options,
device_count=device_count)) as sess:
# === Create the model ===
isTraining = False
batch_size = 1
nsamples = batch_size
isTraining = False
model = create_model(sess, isTraining, dim_to_use_3d, 1, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
print("Model created")
with h5py.File(FLAGS.data_2d_path, 'r') as h5f:
enc_in = h5f['enc_in'][:]
enc_in = enc_in[:, dim_to_use_2d]
mu = data_mean_2d[dim_to_use_2d]
stddev = data_std_2d[dim_to_use_2d]
enc_in = np.divide((enc_in - mu), stddev)
n2d = enc_in.shape[0]
n_extra = n2d % FLAGS.seqlen
if n_extra > 0:
enc_in = enc_in[:-n_extra, :]
n2d = enc_in.shape[0]
pose_2d_sliding = []
encoder_inputs = []
for i in range(n2d - FLAGS.seqlen + 1):
pose_2d_sliding.append(enc_in[i:i + FLAGS.seqlen, :])
pose_2d_list = np.stack(pose_2d_sliding)
encoder_inputs.append(pose_2d_list)
encoder_inputs = np.vstack(encoder_inputs)
n_splits = n2d - FLAGS.seqlen + 1
encoder_inputs = np.array_split(encoder_inputs, n_splits)
all_poses_3d = []
enc_inputs = []
### MAKE PREDICTIONS ######
for bidx in range(len(encoder_inputs)):
# print("Working on batch {0} / {1}... ".format( bidx+1, len(enc_in)), end="" )
dp = 1.0
enc_in = encoder_inputs[bidx]
dec_out = np.zeros(shape=(1, FLAGS.seqlen, 48))
enc_gt = 0
_, _, poses3d = model.step(sess,
enc_in,
dec_out,
dp,
isTraining=False)
enc_in = np.reshape(enc_in, [-1, 16 * 2])
poses3d = np.reshape(poses3d, [-1, 16 * 3])
if not (bidx == 0):
enc_in = np.expand_dims(enc_in[FLAGS.seqlen - 1, :], axis=0)
poses3d = np.expand_dims(poses3d[FLAGS.seqlen - 1, :], axis=0)
inp = data_utils.unNormalizeData(enc_in, data_mean_2d, data_std_2d,
dim_to_ignore_2d)
poses3d = data_utils.unNormalizeData(poses3d, data_mean_3d,
data_std_3d, dim_to_ignore_3d)
enc_inputs.append(inp)
all_poses_3d.append(poses3d)
enc_in = np.vstack(enc_inputs)
poses3d = np.vstack(all_poses_3d)
## Choose camera_id for reconstruction into world coordinate
### NOTE: FOR ARBITRARY 2D detections selecting any camera of subject 9 and 11 works
the_cam = rcams[(
FLAGS.sub_id,
FLAGS.cam_id)] #54138969# 55011271# 58860488 # 60457274
R, _, _, _, _, _, name = the_cam
print(name)
# # Apply inverse rotation and translation
poses3d = np.reshape(poses3d, [-1, 3])
#### NOTE: ONLY the rotation param matters
X_cam = R.T.dot(poses3d.T)
poses3d = np.reshape(X_cam.T, [-1, 32 * 3])
poses3d = poses3d - np.tile(poses3d[:, :3], [1, 32])
# We should be all set now :)
##### GENERATE THE MOVIE
fig = plt.figure(figsize=(12.8, 7.2))
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 1 + 1, projection='3d')
n2d = enc_in.shape[0]
ob1 = viz.Ax2DPose(ax1)
ob2 = viz.Ax3DPose(ax2, lcolor="#9b59b6", rcolor="#2ecc71")
fnames = sorted(glob.glob(FLAGS.img_dir + "*.jpg"))
#print(fnames[0],fnames[1])
for i in range(n2d):
#t0 = time()
print("Working on figure {0:04d} / {1:05d}... \n".format(
i + 1, n2d),
end='')
p2d = enc_in[i, :]
im = Image.open(fnames[i])
ob1.update(im, p2d)
# Plot 3d gt
p3d = poses3d[i, :]
ob2.update(p3d)
fig.canvas.draw()
img_str = np.fromstring(fig.canvas.tostring_rgb(), np.uint8)
ncols, nrows = fig.canvas.get_width_height()
nparr = np.fromstring(img_str,
dtype=np.uint8).reshape(nrows, ncols, 3)
#img_np = cv2.imdecode(nparr, cv2.CV_LOAD_IMAGE_COLOR)
print(FLAGS.output_dir + '{0:05d}.jpg'.format(i + 1))
cv2.imwrite(FLAGS.output_dir + '{0:05d}.jpg'.format(i + 1),
nparr[:, :, ::-1])
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
# numpy implementation
with h5py.File( args.sample_name, 'r' ) as h5f:
expmap_gt = h5f['expmap/gt/'+args.action_name][:]
expmap_pred = h5f['expmap/preds/'+args.action_name][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space( np.vstack((expmap_gt, expmap_pred)), np.eye(3), np.zeros(3) )
expmap_gt = expmap_all[:nframes_gt,:]
expmap_pred = expmap_all[nframes_gt:,:]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range( nframes_gt ):
xyz_gt[i,:] = fkl( expmap_gt[i,:], parent, offset, rotInd, expmapInd )
for i in range( nframes_pred ):
xyz_pred[i,:] = fkl( expmap_pred[i,:], parent, offset, rotInd, expmapInd )
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
# Plot the conditioning ground truth
# for i in range(nframes_gt):
# ob.update( xyz_gt[i,:] )
# # plt.show(block=False)
# # fig.canvas.draw()
# plt.pause(0.01)
#
# # Plot the prediction
# for i in range(nframes_pred):
# ob.update( xyz_pred[i,:], lcolor="#9b59b6", rcolor="#2ecc71" )
# # plt.show(block=False)
# # fig.canvas.draw()
# plt.pause(0.01)
to_draw = np.append(xyz_gt, xyz_pred,axis=0)
# dirty workround for generation gif
counter = 0
def update(x):
nonlocal counter
if counter < 25:
counter += 1
return ob.update(x)
else:
if counter == 50:
counter = 0
else:
counter += 1
return ob.update(x,lcolor="#9b59b6", rcolor="#2ecc71")
anim = animation.FuncAnimation(fig, update, frames=to_draw, interval=40)
if args.save:
anim.save(args.save_name,writer='imagemagick', fps=25)
else:
plt.show()
def main():
if FLAGS.action=='all':
actions = ["walking", "eating", "smoking", "discussion", "directions",
"greeting", "phoning", "posing", "purchases", "sitting",
"sittingdown", "takingphoto", "waiting", "walkingdog",
"walkingtogether"]
else:
actions=FLAGS.action
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
for seed in np.arange(8):
for action in actions:
# numpy implementation
with h5py.File( PATH + '/samples/'+FLAGS.file, 'r' ) as h5f:
expmap_gt = h5f['expmap/gt/' + action + '_'+str(seed)][:50, :]
# expmap_pred = h5f['expmap/preds/' + action + FLAGS.sub_index][:]
if FLAGS.vis=='preds':
expmap_pred = h5f['expmap/preds/' + action + '_'+str(seed)][:]
else:
expmap_pred = h5f['expmap/gt/' + action + '_'+str(seed)][50:,:]
# expmap_gt = h5f['expmap/gt/' + action][1:50, :]
# expmap_pred = h5f['expmap/gt/' + action][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space( np.vstack((expmap_gt, expmap_pred)), np.eye(3), np.zeros(3) )
expmap_gt = expmap_all[:nframes_gt,:]
expmap_pred = expmap_all[nframes_gt:,:]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range( nframes_gt ):
xyz_gt[i,:] = fkl( expmap_gt[i,:], parent, offset, rotInd, expmapInd )
for i in range( nframes_pred ):
xyz_pred[i,:] = fkl( expmap_pred[i,:], parent, offset, rotInd, expmapInd )
# === Plot and animate ===
fig = plt.figure()
ax = plt.gca(projection='3d')
ob = viz.Ax3DPose(ax)
path=PATH + "/samples/"+FLAGS.file+'gt_'+str(seed) +'/'+action
if not os.path.exists(path):
os.makedirs(path)
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob.update( xyz_gt[i,:] )
plt.show(block=False)
# plt.title(action)
fig.canvas.draw()
# plt.axis('off')
plt.savefig(path+"/frame{}.png".format(i+10),bbox_inches='tight')
plt.pause(0.001)
# Plot the prediction
for i in range(nframes_pred):
ob.update( xyz_pred[i,:], lcolor="#9b59b6", rcolor="#2ecc71" )
plt.show(block=False)
fig.canvas.draw()
# plt.axis('off')
plt.savefig(path+"/frame{}.png".format(i+60),bbox_inches='tight')
plt.pause(0.001)
plt.close()
def generate_video(video_dir, labels_true, labels_pred, xyz_gt, reduced_states,
colors, gen_video):
if gen_video:
if not os.path.exists(video_dir):
os.makedirs(video_dir)
fig = plt.figure(figsize=(8, 6))
gs = gridspec.GridSpec(3, 2, height_ratios=[4, 1, 1])
# body movement
ax1 = plt.subplot(gs[0, 0], projection='3d')
ax1.set_yticklabels([])
ax1.set_xticklabels([])
ax1.set_zticklabels([])
ob = viz.Ax3DPose(ax1)
# predicted labels
ax2 = plt.subplot(gs[1, :])
ax2.set_xlim([0, len(labels_pred)])
ax2.set_ylim([0, 0.2])
ax2.set_ylabel("pred")
ax2.set_xticks([])
ax2.set_yticks([])
p = {}
p[0] = ax2.barh(0, 1, color=colors[labels_pred[0]], height=0.2)
seg = []
prev_label = labels_pred[0]
count = 0
for i in range(len(labels_pred)):
if prev_label == labels_pred[i]:
continue
else:
seg.append(i)
count += 1
p[count] = ax2.barh(0, 1, color=colors[labels_pred[i]], height=0.2)
prev_label = labels_pred[i]
seg.append(len(labels_pred))
# ground truth labels
ax3 = plt.subplot(gs[2, :])
ax3.set_xlim([0, len(labels_true)])
ax3.set_ylim([0, 0.2])
ax3.set_ylabel("GT")
ax3.set_yticks([])
p_ = {}
p_[0] = ax3.barh(0, 1, color=colors[labels_true[0]], height=0.2)
seg_ = []
prev_label_ = labels_true[0]
count_ = 0
for i in range(len(labels_true)):
if prev_label_ == labels_true[i]:
continue
else:
seg_.append(i)
count_ += 1
p_[count_] = ax3.barh(0,
1,
color=colors[labels_true[i]],
height=0.2)
prev_label_ = labels_true[i]
seg_.append(len(labels_true))
ax4 = plt.subplot(gs[0, 1], projection='3d')
ax4.set_xticks([])
ax4.set_yticks([])
ax4.set_zticks([])
start_label = labels_pred[0]
if gen_video:
moviewriter = animation.FFMpegWriter(fps=120)
with moviewriter.saving(fig, video_dir + '.mp4', dpi=100):
# Plot the conditioning ground truth
s = 0
s_ = 0
for i in range(len(labels_true)):
ob.update(xyz_gt[i, :], colors[labels_pred[i]])
if i < seg[s] and s == 0:
p[s].patches[0].set_width(i)
elif i <= seg[s]:
if i == seg[s]:
s += 1
if s == len(seg):
for j in range(s - 1):
if j == 0:
p[j].patches[0].set_width(seg[j])
else:
p[j].patches[0].set_width(seg[j] - seg[j - 1])
p[j + 1].patches[0].set_x(seg[j])
p[j + 1].patches[0].set_width(seg[j + 1] - seg[j])
p[s - 1].patches[0].set_x(seg[s - 2])
p[s - 1].patches[0].set_width(i - seg[s - 2])
else:
if s == 1:
p[0].patches[0].set_width(seg[0])
p[1].patches[0].set_x(p[0].patches[0].get_width())
p[1].patches[0].set_width(i - seg[0])
else:
for j in range(s):
if j == 0:
p[j].patches[0].set_width(seg[j])
else:
p[j].patches[0].set_width(seg[j] -
seg[j - 1])
p[j + 1].patches[0].set_x(seg[j])
p[j + 1].patches[0].set_width(i - seg[j])
if i < seg_[s_] and s_ == 0:
p_[s_].patches[0].set_width(i)
elif i <= seg_[s_]:
if i == seg_[s_]:
s_ += 1
if s_ == len(seg_):
for j in range(s_ - 1):
if j == 0:
p_[j].patches[0].set_width(seg_[j])
else:
p_[j].patches[0].set_width(seg_[j] -
seg_[j - 1])
p_[j + 1].patches[0].set_x(seg_[j])
p_[j + 1].patches[0].set_width(seg_[j + 1] -
seg_[j])
p_[s_ - 1].patches[0].set_x(seg_[s_ - 2])
p_[s_ - 1].patches[0].set_width(i - seg_[s_ - 2])
else:
if s_ == 1:
p_[0].patches[0].set_width(seg_[0])
p_[1].patches[0].set_x(
p_[0].patches[0].get_width())
p_[1].patches[0].set_width(i - seg_[0])
else:
for j in range(s_):
if j == 0:
p_[j].patches[0].set_width(seg_[j])
else:
p_[j].patches[0].set_width(seg_[j] -
seg_[j - 1])
p_[j + 1].patches[0].set_x(seg_[j])
p_[j + 1].patches[0].set_width(i - seg_[j])
ax4.scatter(reduced_states[i, 0],
reduced_states[i, 1],
reduced_states[i, 2],
c=colors[labels_pred[i]],
s=2)
moviewriter.grab_frame()
else:
s = 0
s_ = 0
for i in range(len(labels_true)):
#if i%4==0:
#update 3D body movement
ob.update(xyz_gt[i, :], colors[labels_pred[i]])
#update predicted labels
if i < seg[s] and s == 0:
p[s].patches[0].set_width(i)
elif i <= seg[s]:
if i == seg[s]:
s += 1
if s == len(seg):
for j in range(s - 1):
if j == 0:
p[j].patches[0].set_width(seg[j])
else:
p[j].patches[0].set_width(seg[j] - seg[j - 1])
p[j + 1].patches[0].set_x(seg[j])
p[j + 1].patches[0].set_width(seg[j + 1] - seg[j])
p[s - 1].patches[0].set_x(seg[s - 2])
p[s - 1].patches[0].set_width(i - seg[s - 2])
else:
if s == 1:
p[0].patches[0].set_width(seg[0])
p[1].patches[0].set_x(p[0].patches[0].get_width())
p[1].patches[0].set_width(i - seg[0])
else:
for j in range(s):
if j == 0:
p[j].patches[0].set_width(seg[j])
else:
p[j].patches[0].set_width(seg[j] - seg[j - 1])
p[j + 1].patches[0].set_x(seg[j])
p[j + 1].patches[0].set_width(i - seg[j])
# update ground truth labels
if i < seg_[s_] and s_ == 0:
p_[s_].patches[0].set_width(i)
elif i <= seg_[s_]:
if i == seg_[s_]:
s_ += 1
if s_ == len(seg_):
for j in range(s_ - 1):
if j == 0:
p_[j].patches[0].set_width(seg_[j])
else:
p_[j].patches[0].set_width(seg_[j] - seg_[j - 1])
p_[j + 1].patches[0].set_x(seg_[j])
p_[j + 1].patches[0].set_width(seg_[j + 1] - seg_[j])
p_[s_ - 1].patches[0].set_x(seg_[s_ - 2])
p_[s_ - 1].patches[0].set_width(i - seg_[s_ - 2])
else:
if s_ == 1:
p_[0].patches[0].set_width(seg_[0])
p_[1].patches[0].set_x(p_[0].patches[0].get_width())
p_[1].patches[0].set_width(i - seg_[0])
else:
for j in range(s_):
if j == 0:
p_[j].patches[0].set_width(seg_[j])
else:
p_[j].patches[0].set_width(seg_[j] -
seg_[j - 1])
p_[j + 1].patches[0].set_x(seg_[j])
p_[j + 1].patches[0].set_width(i - seg_[j])
ax4.scatter(reduced_states[i, 0],
reduced_states[i, 1],
reduced_states[i, 2],
c=colors[labels_pred[i]],
s=2)
plt.show(block=False)
#fig.canvas.draw()
plt.pause(0.001)
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
action = 'eating'
test_set_sequence = '5'
# numpy implementation
with h5py.File('samples.h5', 'r') as h5f:
expmap_pred = h5f['expmap/preds/' + action + '_' +
test_set_sequence][:]
expmap_gt = h5f['expmap/gt/' + action + '_' + test_set_sequence][:]
nframes_gt, nframes_pred = expmap_gt.shape[0], expmap_pred.shape[0]
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(np.vstack((expmap_gt, expmap_pred)),
np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, xyz_pred = np.zeros((nframes_gt, 96)), np.zeros((nframes_pred, 96))
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
for i in range(nframes_pred):
xyz_pred[i, :] = fkl(expmap_pred[i, :], parent, offset, rotInd,
expmapInd)
# setting up stuff to save video
FFMpegWriter = animation.writers['ffmpeg']
metadata = dict(title=action + '_' + test_set_sequence,
artist='Matplotlib',
comment='Movie support!')
writer = FFMpegWriter(fps=25,
codec="libx264",
bitrate=-1,
metadata=metadata)
# === Plot and animate ===
fig = plt.figure()
gt_ax = fig.add_subplot(1, 2, 1, projection='3d')
pred_ax = fig.add_subplot(1, 2, 2, projection='3d')
ob_gt = viz.Ax3DPose(gt_ax)
ob_pred = viz.Ax3DPose(pred_ax)
# setting viewing angle
gt_ax.view_init(azim=135)
pred_ax.view_init(azim=45)
with writer.saving(fig, action + "_" + test_set_sequence + ".mp4", 100):
# Plot the conditioning ground truth
for i in range(nframes_gt):
ob_gt.update(xyz_gt[i, :])
#plt.show(block=False)
fig.canvas.draw()
#plt.pause(0.001)
#writer.grab_frame()
# Plot the prediction
#for i in range(nframes_pred):
ob_pred.update(xyz_pred[i, :], lcolor="#9b59b6", rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
plt.pause(0.001)
writer.grab_frame()
def main():
# Load all the data
parent, offset, rotInd, expmapInd = _some_variables()
# short-term models
with h5py.File(
'../final_exp_samples/short-term/jul_unsup_sa/walking_samples.h5',
'r') as h5f5:
jul_unsup_sa_expmap_pred = h5f5['expmap/preds/walking_6'][:]
expmap_gt_5 = h5f5['expmap/gt/walking_6'][:]
with h5py.File(
'../final_exp_samples/short-term/pgru_skip_1/walking_samples_v2.h5',
'r') as h5f6:
pgru_skip_1_expmap_pred = h5f6['expmap/preds/walking_6'][:]
expmap_gt_6 = h5f6['expmap/gt/walking_6'][:]
# load mocap gt and PGRU-d model predictions
with h5py.File(
'../final_exp_samples/long-term/pgru-d/walking_samples_v2.h5',
'r') as h5f1:
pgru_d_expmap_pred = h5f1['expmap/preds/walking_6'][:]
expmap_gt_1 = h5f1['expmap/gt/walking_6'][:]
with h5py.File('../final_exp_samples/long-term/gru-d/walking_samples.h5',
'r') as h5f2:
gru_d_expmap_pred = h5f2['expmap/preds/walking_6'][:]
expmap_gt_2 = h5f2['expmap/gt/walking_6'][:]
with h5py.File('../final_exp_samples/long-term/pgru-a/walking_samples.h5',
'r') as h5f3:
pgru_a_expmap_pred = h5f3['expmap/preds/walking_6'][:]
expmap_gt_3 = h5f3['expmap/gt/walking_6'][:]
with h5py.File(
'../final_exp_samples/long-term/julietta/walking_samples.h5',
'r') as h5f4:
jul_long_expmap_pred = h5f4['expmap/preds/walking_6'][:]
expmap_gt_4 = h5f4['expmap/gt/walking_6'][:]
nframes_gt, nframes_pred = expmap_gt_1.shape[0], pgru_d_expmap_pred.shape[
0]
# computing NPSS metric for all models
#euler_gt_5_seq, euler_jul_unsup_sa_seq = read_data(jul_unsup_sa_expmap_pred, expmap_gt_5)
#euler_gt_6_seq, euler_pgru_skip_1_seq = read_data(pgru_skip_1_expmap_pred, expmap_gt_6)
#euler_gt_1_seq, euler_pgru_d_seq = read_data(pgru_d_expmap_pred, expmap_gt_1)
#euler_gt_2_seq, euler_gru_d_seq = read_data(gru_d_expmap_pred, expmap_gt_2)
#euler_gt_3_seq, euler_pgru_a_seq = read_data(pgru_a_expmap_pred, expmap_gt_3)
#euler_gt_4_seq, euler_jul_long_seq = read_data(jul_long_expmap_pred, expmap_gt_4)
#jul_unsup_sa_emd = compute_metrics(euler_gt_5_seq, euler_jul_unsup_sa_seq)
#pgru_skip_1_emd = compute_metrics(euler_gt_6_seq, euler_pgru_skip_1_seq)
#pgru_d_emd = compute_metrics(euler_gt_1_seq, euler_pgru_d_seq)
#gru_d_emd = compute_metrics(euler_gt_2_seq, euler_gru_d_seq)
#pgru_a_emd = compute_metrics(euler_gt_3_seq, euler_pgru_a_seq)
#jul_long_emd = compute_metrics(euler_gt_4_seq, euler_jul_long_seq)
# Put them together and revert the coordinate space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt_1, pgru_d_expmap_pred)), np.eye(3), np.zeros(3))
expmap_gt = expmap_all[:nframes_gt, :]
pgru_d_expmap_pred = expmap_all[nframes_gt:, :]
# gru-d revert co-ord space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt_2, gru_d_expmap_pred)), np.eye(3), np.zeros(3))
gru_d_expmap_pred = expmap_all[nframes_gt:, :]
# pgru-ac revert co-ord space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt_3, pgru_a_expmap_pred)), np.eye(3), np.zeros(3))
pgru_a_expmap_pred = expmap_all[nframes_gt:, :]
# julietta-long revert co-ord space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt_4, jul_long_expmap_pred)), np.eye(3), np.zeros(3))
jul_long_expmap_pred = expmap_all[nframes_gt:, :]
# jul_unsup_sa revert co-ord space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt_5, jul_unsup_sa_expmap_pred)), np.eye(3),
np.zeros(3))
jul_unsup_sa_expmap_pred = expmap_all[nframes_gt:, :]
# pgru_skip_1 revert co-ord space
expmap_all = revert_coordinate_space(
np.vstack((expmap_gt_6, pgru_skip_1_expmap_pred)), np.eye(3),
np.zeros(3))
pgru_skip_1_expmap_pred = expmap_all[nframes_gt:, :]
# Compute 3d points for each frame
xyz_gt, pgru_d_xyz_pred = np.zeros((nframes_gt, 96)), np.zeros(
(nframes_pred, 96))
gru_d_xyz_pred = np.zeros((nframes_gt, 96))
pgru_a_xyz_pred = np.zeros((nframes_gt, 96))
jul_long_xyz_pred = np.zeros((nframes_gt, 96))
jul_unsup_sa_xyz_pred = np.zeros((nframes_gt, 96))
pgru_skip_1_xyz_pred = np.zeros((nframes_gt, 96))
# ground-truth xyz frames
for i in range(nframes_gt):
xyz_gt[i, :] = fkl(expmap_gt[i, :], parent, offset, rotInd, expmapInd)
# pgru-d xyz frames
for i in range(nframes_pred):
pgru_d_xyz_pred[i, :] = fkl(pgru_d_expmap_pred[i, :], parent, offset,
rotInd, expmapInd)
# gru-d xyz frames
for i in range(nframes_pred):
gru_d_xyz_pred[i, :] = fkl(gru_d_expmap_pred[i, :], parent, offset,
rotInd, expmapInd)
# gru-ac xyz frames
for i in range(nframes_pred):
pgru_a_xyz_pred[i, :] = fkl(pgru_a_expmap_pred[i, :], parent, offset,
rotInd, expmapInd)
# jul-long xyz frames
for i in range(nframes_pred):
jul_long_xyz_pred[i, :] = fkl(jul_long_expmap_pred[i, :], parent,
offset, rotInd, expmapInd)
# jul-unsup-sa xyz frames
for i in range(nframes_pred):
jul_unsup_sa_xyz_pred[i, :] = fkl(jul_unsup_sa_expmap_pred[i, :],
parent, offset, rotInd, expmapInd)
# pgru-skip-1 xyz frames
for i in range(nframes_pred):
pgru_skip_1_xyz_pred[i, :] = fkl(pgru_skip_1_expmap_pred[i, :], parent,
offset, rotInd, expmapInd)
# setting up stuff to save video
FFMpegWriter = animation.writers['ffmpeg']
metadata = dict(title='Walking Sequence 6',
artist='Matplotlib',
comment='Movie support!')
writer = FFMpegWriter(fps=12,
codec="libx264",
bitrate=-1,
metadata=metadata)
# === Plot and animate ===
fig = plt.figure(figsize=(22.0, 11.0))
fig.suptitle("Walking Sequence 6")
fig.subplots_adjust(left=0.05,
bottom=0.05,
right=0.95,
top=0.95,
wspace=None,
hspace=None)
gt_ax = fig.add_subplot(3, 3, 2, projection='3d')
jul_unsup_sa_pred_ax = fig.add_subplot(3, 3, 4, projection='3d')
pgru_skip_1_pred_ax = fig.add_subplot(3, 3, 5, projection='3d')
jul_long_pred_ax = fig.add_subplot(3, 3, 6, projection='3d')
pgru_a_pred_ax = fig.add_subplot(3, 3, 7, projection='3d')
gru_d_pred_ax = fig.add_subplot(3, 3, 8, projection='3d')
pgru_d_pred_ax = fig.add_subplot(3, 3, 9, projection='3d')
# setting viewing angle
gt_ax.view_init(azim=135)
jul_unsup_sa_pred_ax.view_init(azim=45)
pgru_skip_1_pred_ax.view_init(azim=45)
jul_long_pred_ax.view_init(azim=45)
pgru_a_pred_ax.view_init(azim=45)
gru_d_pred_ax.view_init(azim=45)
pgru_d_pred_ax.view_init(azim=45)
font = {
'family': 'serif',
'color': 'black',
'weight': 'normal',
'size': 12,
}
# titles and legends for subplots
gt_ax.set_title("Ground-Truth")
#jul_unsup_sa_emd_str = '$\mathrm{NPSS}=%.3f$'%(jul_unsup_sa_emd)
jul_unsup_sa_pred_ax.set_title("A")
#jul_unsup_sa_pred_ax.text2D(0.35,0.80, jul_unsup_sa_emd_str, fontdict=font, transform=jul_unsup_sa_pred_ax.transAxes)
#pgru_skip_1_emd_str = '$\mathrm{NPSS}=%.3f$'%(pgru_skip_1_emd)
pgru_skip_1_pred_ax.set_title("B")
#pgru_skip_1_pred_ax.text2D(0.35,0.80, pgru_skip_1_emd_str, fontdict=font, transform=pgru_skip_1_pred_ax.transAxes)
#jul_long_emd_str = '$\mathrm{NPSS}=%.3f$'%(jul_long_emd)
jul_long_pred_ax.set_title("C")
#jul_long_pred_ax.text2D(0.35,0.80, jul_long_emd_str, fontdict=font, transform=jul_long_pred_ax.transAxes)
#pgru_a_emd_str = '$\mathrm{NPSS}=%.3f$'%(pgru_a_emd)
pgru_a_pred_ax.set_title("D")
#pgru_a_pred_ax.text2D(0.35,0.80, pgru_a_emd_str, fontdict=font, transform=pgru_a_pred_ax.transAxes)
#gru_d_emd_str = '$\mathrm{NPSS}=%.3f$'%(gru_d_emd)
gru_d_pred_ax.set_title("E")
#gru_d_pred_ax.text2D(0.35, 0.80, gru_d_emd_str, fontdict=font, transform=gru_d_pred_ax.transAxes)
#pgru_d_emd_str = '$\mathrm{NPSS}=%.3f$'%(pgru_d_emd)
pgru_d_pred_ax.set_title("F")
#pgru_d_pred_ax.text2D(0.35, 0.80, pgru_d_emd_str, fontdict=font, transform=pgru_d_pred_ax.transAxes)
ob_gt = viz.Ax3DPose(gt_ax)
jul_unsup_sa_ob_pred = viz.Ax3DPose(jul_unsup_sa_pred_ax)
pgru_skip_1_ob_pred = viz.Ax3DPose(pgru_skip_1_pred_ax)
jul_long_ob_pred = viz.Ax3DPose(jul_long_pred_ax)
pgru_a_ob_pred = viz.Ax3DPose(pgru_a_pred_ax)
gru_d_ob_pred = viz.Ax3DPose(gru_d_pred_ax)
pgru_d_ob_pred = viz.Ax3DPose(pgru_d_pred_ax)
with writer.saving(fig, "walking_seq_6.mp4", 100):
for i in range(nframes_gt):
# Plot the conditioning ground truth
ob_gt.update(xyz_gt[i, :])
fig.canvas.draw()
# Plot the jul-unsup-sa prediction
jul_unsup_sa_ob_pred.update(jul_unsup_sa_xyz_pred[i, :],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
# Plot the pgru-skip-1 prediction
pgru_skip_1_ob_pred.update(pgru_skip_1_xyz_pred[i, :],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
# Plot the jul-long prediction
jul_long_ob_pred.update(jul_long_xyz_pred[i, :],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
# Plot the pgru-ac prediction
pgru_a_ob_pred.update(pgru_a_xyz_pred[i, :],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
# Plot the gru-d prediction
gru_d_ob_pred.update(gru_d_xyz_pred[i, :],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
# Plot the pgru-ac prediction
pgru_d_ob_pred.update(pgru_d_xyz_pred[i, :],
lcolor="#9b59b6",
rcolor="#2ecc71")
plt.show(block=False)
fig.canvas.draw()
writer.grab_frame()
