def bvh_reproj(args):
anim, names, frametime = BVH.load(args.bvh_path)
positions = Animation.positions_global(anim)
print(positions.shape)
camera_data = load_camera(args.json_path)
for cam_name in camera_data.keys():
if args.multi_process:
Process(target=bvh_reproj_for_cam,
args=(args, cam_name, camera_data, positions,
anim)).start()
else:
bvh_reproj_for_cam(args, cam_name, camera_data, positions, anim)
def __init__(self, config, is_train=True):
poses_3d_root, rotations, bones, alphas, contacts, projections = [], [], [], [], [], []
self.frames = []
self.config = config
self.rotation_number = ROTATION_NUMBERS.get(config.arch.rotation_type)
datasets = ['bvh'] #, 'bvh']
if 'h36m' in datasets:
dim_to_use_3d = h36m_utils.dimension_reducer(
3, config.arch.predict_joints)
subjects = h36m_utils.TRAIN_SUBJECTS if is_train else h36m_utils.TEST_SUBJECTS
actions = h36m_utils.define_actions('All')
self.cameras = h36m_utils.load_cameras(config.trainer.data_path)
for subject in subjects:
for action in actions:
for subaction in range(1, 3):
data_file = h5py.File(
'%s/S%s/%s-%s/annot.h5' %
(config.trainer.data_path, subject, action,
subaction), 'r')
data_size = data_file['frame'].size / 4
data_set = np.array(data_file['pose/3d']).reshape(
(-1, 96))[:, dim_to_use_3d]
for i in range(4):
camera_name = data_file['camera'][int(data_size *
i)]
R, T, f, c, k, p, res_w, res_h = self.cameras[(
subject, str(camera_name))]
set_3d = data_set[int(data_size *
i):int(data_size *
(i + 1))].copy()
set_3d_world = h36m_utils.camera_to_world_frame(
set_3d.reshape((-1, 3)), R, T)
# set_3d_world[:, [1, 2]] = set_3d_world[:, [2, 1]]
# set_3d_world[:, [2]] *= -1
# set_3d_world = set_3d_world.reshape((-1, config.arch.predict_joints * 3))
set_3d_root = set_3d_world - np.tile(
set_3d_world[:, :3],
[1, int(set_3d_world.shape[-1] / 3)])
set_bones = self.get_bones(
set_3d_root, config.arch.predict_joints)
set_alphas = np.mean(set_bones, axis=1)
self.frames.append(set_3d_root.shape[0])
poses_3d_root.append(
set_3d_root /
np.expand_dims(set_alphas, axis=-1))
rotations.append(
np.zeros((set_3d_root.shape[0],
int(set_3d_root.shape[1] / 3 *
self.rotation_number))))
bones.append(set_bones /
np.expand_dims(set_alphas, axis=-1))
alphas.append(set_alphas)
contacts.append(
self.get_contact(set_3d_world,
config.arch.predict_joints))
projections.append(
(set_3d_world.copy() /
np.expand_dims(set_alphas, axis=-1)).reshape(
(set_3d_world.shape[0], -1, 3))[:, 0, 2])
if 'bvh' in datasets:
to_keep = [
0, 7, 8, 9, 2, 3, 4, 12, 15, 18, 19, 20, 25, 26, 27
] if config.arch.predict_joints == 15 else [
0, 7, 8, 9, 2, 3, 4, 12, 13, 15, 16, 18, 19, 20, 25, 26, 27
]
parents = [
-1, 0, 1, 2, 0, 4, 5, 0, 7, 7, 9, 10, 7, 12, 13
] if config.arch.predict_joints == 15 else [
-1, 0, 1, 2, 0, 4, 5, 0, 7, 8, 9, 8, 11, 12, 8, 14, 15
]
bvh_files = util.make_dataset(['/mnt/dataset/test_bvh'],
phase='bvh',
data_split=1)
bvh_files = bvh_files[:int(len(bvh_files) *
0.8)] if is_train else bvh_files[
int(len(bvh_files) * 0.8):]
for bvh_file in bvh_files:
original_anim, joint_names, frame_rate = BVH.load(bvh_file)
set_skel_in = original_anim.positions[:, to_keep, :]
set_rotations = original_anim.rotations.qs[:, to_keep, :]
anim = Animation.Animation(
Quaternions(set_rotations), set_skel_in,
original_anim.orients.qs[to_keep, :], set_skel_in,
np.array(parents))
set_3d_world = Animation.positions_global(anim).reshape(
set_rotations.shape[0], -1)
set_3d_world[:, 0:3] = (set_3d_world[:, 3:6] +
set_3d_world[:, 12:15]) / 2
set_3d_root = set_3d_world - np.tile(
set_3d_world[:, :3],
[1, int(set_3d_world.shape[-1] / 3)])
set_bones = self.get_bones(set_3d_root,
config.arch.predict_joints)
set_alphas = np.mean(set_bones, axis=1)
self.frames.append(set_3d_root.shape[0])
poses_3d_root.append(set_3d_root /
np.expand_dims(set_alphas, axis=-1))
rotations.append(
np.zeros((set_3d_root.shape[0],
int(set_3d_root.shape[1] / 3 *
self.rotation_number))))
bones.append(set_bones / np.expand_dims(set_alphas, axis=-1))
alphas.append(set_alphas)
contacts.append(
self.get_contact(set_3d_world, config.arch.predict_joints))
projections.append(
(set_3d_world.copy() /
np.expand_dims(set_alphas, axis=-1)).reshape(
(set_3d_world.shape[0], -1, 3))[:, 0, 2])
self.poses_3d = np.concatenate(poses_3d_root, axis=0)
self.rotations = np.concatenate(rotations, axis=0)
self.bones = np.concatenate(bones, axis=0)
self.alphas = np.concatenate(alphas, axis=0)
self.contacts = np.concatenate(contacts, axis=0)
self.projections = np.concatenate(projections, axis=0)
posed_3d_flip = self.get_flipping(self.poses_3d, 3,
config.arch.predict_joints)
if config.trainer.data_aug_flip and is_train:
self.poses_3d = np.concatenate([self.poses_3d, posed_3d_flip],
axis=0)
self.poses_2d = self.get_projection(self.poses_3d)
self.poses_2d_root = (self.poses_2d -
self.poses_2d[:, 0, None]).reshape(
(self.poses_3d.shape[0], -1))
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
from utils import visualization
fig = plt.figure()
gs = gridspec.GridSpec(1, 2)
for i in range(1):
ax1 = plt.subplot(gs[0], projection='3d')
visualization.show3Dpose(self.poses_3d[i], ax1, radius=5)
ax2 = plt.subplot(gs[1])
visualization.show2Dpose(self.poses_2d_root[i] * 1000 + 500,
ax2,
radius=1000)
fig.savefig('./images/2d_3d/_%d.png' % i)
fig.clear()
self.update_sequence_index()
def main(config, args, output_folder):
resume = args.resume
name_list = [
'Hips', 'RightUpLeg', 'RightLeg', 'RightFoot', 'LeftUpLeg', 'LeftLeg',
'LeftFoot', 'Spine', 'Spine1', 'Neck', 'Head', 'LeftArm',
'LeftForeArm', 'LeftHand', 'RightArm', 'RightForeArm', 'RightHand'
]
model = getattr(models, config.arch.type)(config)
checkpoint = torch.load(resume)
state_dict = checkpoint['state_dict']
model.load_state_dict(state_dict)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
if args.input == 'h36m':
test_data_loader = h36m_loader(config, is_training=False)
test_parameters = [
torch.from_numpy(np.array(item)).float().to(device)
for item in test_data_loader.dataset.get_parameters()
]
error_list = {}
errors = []
sampling_export = np.random.choice(test_data_loader.n_samples - 1,
size=4,
replace=False)
for video_idx, datas in enumerate(test_data_loader):
video_name = datas[-1][0]
datas = [item.float().to(device) for item in datas[:-1]]
poses_2d, poses_3d, bones, contacts, alphas, proj_facters = datas
with torch.no_grad():
pre_bones, pre_rotations, pre_rotations_full, pre_pose_3d, pre_c, pre_proj = model.forward_fk(
poses_2d, test_parameters)
error = metric.mean_points_error(poses_3d,
pre_pose_3d) * torch.mean(
alphas[0]).data.cpu().numpy()
errors.append(error)
action_name = video_name.split('_')[1].split(' ')[0]
if action_name in error_list.keys():
error_list[action_name].append(error)
else:
error_list[action_name] = [error]
if video_idx in sampling_export:
if config.arch.translation:
R, T, f, c, k, p, res_w, res_h = test_data_loader.dataset.cameras[
(int(video_name.split('_')[0].replace('S', '')),
int(video_name.split('_')[-1]))]
pose_2d_film = (poses_2d[0, :, :2].cpu().numpy() -
c[:, 0]) / f[:, 0]
translations = np.ones(shape=(pose_2d_film.shape[0], 3))
translations[:, :2] = pose_2d_film
translation = (translations * np.repeat(
pre_proj[0].cpu().numpy(), 3, axis=-1).reshape(
(-1, 3))) * 5
else:
translation = np.zeros((poses_2d.shape[1], 3))
rotations = pre_rotations_full[0].cpu().numpy()
length = (pre_bones * test_parameters[3].unsqueeze(0) +
test_parameters[2].repeat(bones.shape[0], 1,
1))[0].cpu().numpy()
BVH.save('%s/%s.bvh' % (output_folder, video_name),
Animation.load_from_network(translation,
rotations,
length,
third_dimension=1),
names=name_list)
error_file = '%s/errors.txt' % output_folder
with open(error_file, 'w') as f:
f.writelines('=====Action===== ==mm==\n')
total = []
for key in error_list.keys():
mean_error = np.mean(np.array(error_list[key]))
total.append(mean_error)
print('%16s %.2f' % (key, mean_error))
f.writelines('%16s %.2f \n' % (key, mean_error))
print('%16s %.2f' % ('Average', np.mean(np.array(errors))))
f.writelines('%16s %.2f \n' %
('Average', np.mean(np.array(errors))))
f.close()
else:
parameters = [
torch.from_numpy(np.array(item)).float().to(device) for item in
h36m_loader(config, is_training=True).dataset.get_parameters()
]
def export(pose_folder):
video_name = pose_folder.split('/')[-1]
files = util.make_dataset([pose_folder],
phase='json',
data_split=1,
sort=True,
sort_index=0)
IMAGE_WIDTH = 1080 # Should be changed refer to your test data
pose_batch = []
confidence_batch = []
for pose_file_name in files:
with open(pose_file_name, 'r') as f:
h36m_locations, h36m_confidence = h36m_utils.convert_openpose(
json.load(f))
pose_batch.append(h36m_locations)
confidence_batch.append(h36m_confidence)
poses_2d = np.concatenate(pose_batch, axis=0) / IMAGE_WIDTH
confidences = np.concatenate(confidence_batch, axis=0)
poses_2d_root = (
poses_2d -
np.tile(poses_2d[:, :2], [1, int(poses_2d.shape[-1] / 2)]))
if config.arch.confidence:
poses_2d_root_c = np.zeros(
(poses_2d_root.shape[0],
int(poses_2d_root.shape[-1] / 2 * 3)))
for joint_index in range(int(poses_2d_root.shape[-1] / 2)):
poses_2d_root_c[:, 3 *
joint_index] = poses_2d_root[:, 2 *
joint_index].copy(
)
poses_2d_root_c[:, 3 * joint_index +
1] = poses_2d_root[:, 2 * joint_index +
1].copy()
poses_2d_root_c[:, 3 * joint_index + 2] = np.array(
confidences)[:, joint_index].copy()
poses_2d = poses_2d_root_c
poses_2d = np.divide((poses_2d - parameters[0].cpu().numpy()),
parameters[1].cpu().numpy())
poses_2d = torch.from_numpy(
np.array(poses_2d)).unsqueeze(0).float().to(device)
with torch.no_grad():
pre_bones, pre_rotations, pre_rotations_full, pre_pose_3d, pre_c, pre_proj = model.forward_fk(
poses_2d, parameters)
if config.arch.translation:
pose_2d_film = (poses_2d[0, :, :2].cpu().numpy() - 0.5)
translations = np.ones(shape=(pose_2d_film.shape[0], 3))
translations[:, :2] = pose_2d_film
translation = (translations * np.repeat(
pre_proj[0].cpu().numpy(), 3, axis=-1).reshape(
(-1, 3))) * 3
translation[:] -= translation[[0]]
else:
translation = np.zeros((poses_2d.shape[1], 3))
rotations = pre_rotations_full[0].cpu().numpy()
length = (pre_bones * parameters[3].unsqueeze(0) +
parameters[2].repeat(pre_bones.shape[0], 1,
1))[0].cpu().numpy()
BVH.save('%s/%s.bvh' % (output_folder, video_name),
Animation.load_from_network(translation,
rotations,
length,
third_dimension=1),
names=name_list)
print('The bvh file of %s has been saved!' % video_name)
export(args.input)
def export(pose_folder):
video_name = pose_folder.split('/')[-1]
files = util.make_dataset([pose_folder],
phase='json',
data_split=1,
sort=True,
sort_index=0)
IMAGE_WIDTH = 1080 # Should be changed refer to your test data
pose_batch = []
confidence_batch = []
for pose_file_name in files:
with open(pose_file_name, 'r') as f:
h36m_locations, h36m_confidence = h36m_utils.convert_openpose(
json.load(f))
pose_batch.append(h36m_locations)
confidence_batch.append(h36m_confidence)
poses_2d = np.concatenate(pose_batch, axis=0) / IMAGE_WIDTH
confidences = np.concatenate(confidence_batch, axis=0)
poses_2d_root = (
poses_2d -
np.tile(poses_2d[:, :2], [1, int(poses_2d.shape[-1] / 2)]))
if config.arch.confidence:
poses_2d_root_c = np.zeros(
(poses_2d_root.shape[0],
int(poses_2d_root.shape[-1] / 2 * 3)))
for joint_index in range(int(poses_2d_root.shape[-1] / 2)):
poses_2d_root_c[:, 3 *
joint_index] = poses_2d_root[:, 2 *
joint_index].copy(
)
poses_2d_root_c[:, 3 * joint_index +
1] = poses_2d_root[:, 2 * joint_index +
1].copy()
poses_2d_root_c[:, 3 * joint_index + 2] = np.array(
confidences)[:, joint_index].copy()
poses_2d = poses_2d_root_c
poses_2d = np.divide((poses_2d - parameters[0].cpu().numpy()),
parameters[1].cpu().numpy())
poses_2d = torch.from_numpy(
np.array(poses_2d)).unsqueeze(0).float().to(device)
with torch.no_grad():
pre_bones, pre_rotations, pre_rotations_full, pre_pose_3d, pre_c, pre_proj = model.forward_fk(
poses_2d, parameters)
if config.arch.translation:
pose_2d_film = (poses_2d[0, :, :2].cpu().numpy() - 0.5)
translations = np.ones(shape=(pose_2d_film.shape[0], 3))
translations[:, :2] = pose_2d_film
translation = (translations * np.repeat(
pre_proj[0].cpu().numpy(), 3, axis=-1).reshape(
(-1, 3))) * 3
translation[:] -= translation[[0]]
else:
translation = np.zeros((poses_2d.shape[1], 3))
rotations = pre_rotations_full[0].cpu().numpy()
length = (pre_bones * parameters[3].unsqueeze(0) +
parameters[2].repeat(pre_bones.shape[0], 1,
1))[0].cpu().numpy()
BVH.save('%s/%s.bvh' % (output_folder, video_name),
Animation.load_from_network(translation,
rotations,
length,
third_dimension=1),
names=name_list)
print('The bvh file of %s has been saved!' % video_name)
def remove_fs(anim,
foot,
output_path,
fid_l=(4, 5),
fid_r=(9, 10),
interp_length=5,
force_on_floor=True):
(anim, names, ftime), glb = nrot2anim(anim)
T = len(glb)
fid = list(fid_l) + list(fid_r)
fid_l, fid_r = np.array(fid_l), np.array(fid_r)
foot_heights = np.minimum(glb[:, fid_l, 1],
glb[:, fid_r, 1]).min(axis=1) # [T, 2] -> [T]
# print(np.min(foot_heights))
floor_height = softmin(foot_heights, softness=0.5, axis=0)
# print(floor_height)
glb[:, :, 1] -= floor_height
anim.positions[:, 0, 1] -= floor_height
for i, fidx in enumerate(fid):
fixed = foot[i] # [T]
"""
for t in range(T):
glb[t, fidx][1] = max(glb[t, fidx][1], 0.25)
"""
s = 0
while s < T:
while s < T and fixed[s] == 0:
s += 1
if s >= T:
break
t = s
avg = glb[t, fidx].copy()
while t + 1 < T and fixed[t + 1] == 1:
t += 1
avg += glb[t, fidx].copy()
avg /= (t - s + 1)
if force_on_floor:
avg[1] = 0.0
for j in range(s, t + 1):
glb[j, fidx] = avg.copy()
# print(fixed[s - 1:t + 2])
s = t + 1
for s in range(T):
if fixed[s] == 1:
continue
l, r = None, None
consl, consr = False, False
for k in range(interp_length):
if s - k - 1 < 0:
break
if fixed[s - k - 1]:
l = s - k - 1
consl = True
break
for k in range(interp_length):
if s + k + 1 >= T:
break
if fixed[s + k + 1]:
r = s + k + 1
consr = True
break
if not consl and not consr:
continue
if consl and consr:
litp = lerp(alpha(1.0 * (s - l + 1) / (interp_length + 1)),
glb[s, fidx], glb[l, fidx])
ritp = lerp(alpha(1.0 * (r - s + 1) / (interp_length + 1)),
glb[s, fidx], glb[r, fidx])
itp = lerp(alpha(1.0 * (s - l + 1) / (r - l + 1)), ritp, litp)
glb[s, fidx] = itp.copy()
continue
if consl:
litp = lerp(alpha(1.0 * (s - l + 1) / (interp_length + 1)),
glb[s, fidx], glb[l, fidx])
glb[s, fidx] = litp.copy()
continue
if consr:
ritp = lerp(alpha(1.0 * (r - s + 1) / (interp_length + 1)),
glb[s, fidx], glb[r, fidx])
glb[s, fidx] = ritp.copy()
targetmap = {}
for j in range(glb.shape[1]):
targetmap[j] = glb[:, j]
ik = JacobianInverseKinematics(anim,
targetmap,
iterations=10,
damping=4.0,
silent=False)
ik()
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
BVH.save(output_path, anim, names, ftime)
def save_bvh_from_network_output(nrot, output_path):
anim = AnimationData.from_network_output(nrot)
bvh, names, ftime = anim.get_BVH()
if not os.path.exists(os.path.dirname(output_path)):
os.makedirs(os.path.dirname(output_path))
BVH.save(output_path, bvh, names, ftime)
import sys
sys.path.append('./')
import numpy as np
import utils.BVH as BVH
from utils.Quaternions import Quaternions
from utils import util
rotations_bvh = []
bvh_files = util.make_dataset(['/mnt/dataset/cmubvh'], phase='bvh', data_split=1, sort_index=0)
for file in bvh_files:
original_anim, _, frametime = BVH.load(file, rotate=True)
sampling = 3
to_keep = [0, 7, 8, 2, 3, 12, 13, 15, 18, 19, 25, 26]
real_rotations = original_anim.rotations.qs[1:, to_keep, :]
rotations_bvh.append(real_rotations[np.arange(0, real_rotations.shape[0] // sampling) * sampling].astype('float32'))
np.savez_compressed('./data/data_cmu.npz', rotations=rotations_bvh)