def get_pair(self, idxA):
maskA = np.zeros(shape=self.out_frame, dtype=self.dtype)
maskB = np.zeros(shape=self.out_frame, dtype=self.dtype)
scale = self.bvh_scale
aug_q = Quaternions.id(1)
if self.scale_augment:
t = random.gauss(1, self.scale_std)
t = max(t, 0.1)
scale *= t
if self.rot_augment:
rot = random.random()
rot = (rot - 0.5) * 2 * np.pi
aug_q = Quaternions.from_angle_axis(rot, np.array([0, 1, 0])) * aug_q
quatA, localA, rtjA, skelA = self.get_normalized_data(idxA, scale, aug_q)
random_slice = get_random_slice(localA, self.out_frame)
maskA[:np.min([self.out_frame, localA.shape[0]])] = 1.0
localA = repeat_last_frame(localA[random_slice], self.out_frame)
rtjA = repeat_last_frame(rtjA[random_slice], self.out_frame)
skelA = repeat_last_frame(skelA[random_slice], self.out_frame)
quatA = repeat_last_frame(quatA[random_slice], self.out_frame)
# get real skeleton and motion
skelA_name, _, _ = self.train[idxA]
idxB = np.random.randint(self.__len__())
while skelA_name == self.train[idxB][0]:
idxB = np.random.randint(self.__len__())
quatB, localB, rtjB, skelB = self.get_normalized_data(idxB, scale, aug_q)
random_slice = get_random_slice(localB, self.out_frame)
maskB[:np.min([self.out_frame, localB.shape[0]])] = 1.0
localB = repeat_last_frame(localB[random_slice], self.out_frame)
rtjB = repeat_last_frame(rtjB[random_slice], self.out_frame)
skelB = repeat_last_frame(skelB[random_slice], self.out_frame)
quatB = repeat_last_frame(quatB[random_slice], self.out_frame)
localA = localA.astype(self.dtype)
rtjA = rtjA.astype(self.dtype)
skelA = skelA.astype(self.dtype)
quatA = quatA.astype(self.dtype)
localB = localB.astype(self.dtype)
rtjB = rtjB.astype(self.dtype)
skelB = skelB.astype(self.dtype)
quatB = quatB.astype(self.dtype)
return (localA, rtjA, skelA, quatA, maskA,
localB, rtjB, skelB, quatB, maskB)
def rotations_global(anim):
"""
Global Animation Rotations
This relies on joint ordering
being incremental. That means a joint
J1 must not be a ancestor of J0 if
J0 appears before J1 in the joint
ordering.
Parameters
----------
anim : Animation
Input animation
Returns
-------
points : (F, J) Quaternions
global rotations for every frame F
and joint J
"""
joints = np.arange(anim.shape[1])
parents = np.arange(anim.shape[1])
locals = anim.rotations
globals = Quaternions.id(anim.shape)
globals[:, 0] = locals[:, 0]
for i in range(1, anim.shape[1]):
globals[:, i] = globals[:, anim.parents[i]] * locals[:, i]
return globals
def orients_global(anim):
joints = np.arange(anim.shape[1])
parents = np.arange(anim.shape[1])
locals = anim.orients
globals = Quaternions.id(anim.shape[1])
globals[:, 0] = locals[:, 0]
for i in range(1, anim.shape[1]):
globals[:, i] = globals[:, anim.parents[i]] * locals[:, i]
return globals
def load(filename, start=None, end=None, order=None, world=False):
"""
Reads a BVH file and constructs an animation
Parameters
----------
filename: str
File to be opened
start : int
Optional Starting Frame
end : int
Optional Ending Frame
order : str
Optional Specifier for joint order.
Given as string E.G 'xyz', 'zxy'
world : bool
If set to true euler angles are applied
together in world space rather than local
space
Returns
-------
(animation, joint_names, frametime)
Tuple of loaded animation and joint names
"""
f = open(filename, "r")
i = 0
active = -1
end_site = False
names = []
orients = Quaternions.id(0)
offsets = np.array([]).reshape((0, 3))
parents = np.array([], dtype=int)
for line in f:
if "HIERARCHY" in line: continue
if "MOTION" in line: continue
""" Modified line read to handle mixamo data """
# rmatch = re.match(r"ROOT (\w+)", line)
rmatch = re.match(r"ROOT (\w+:?\w+)", line)
if rmatch:
names.append(rmatch.group(1))
offsets = np.append(offsets, np.array([[0, 0, 0]]), axis=0)
orients.qs = np.append(orients.qs,
np.array([[1, 0, 0, 0]]),
axis=0)
parents = np.append(parents, active)
active = (len(parents) - 1)
continue
if "{" in line:
continue
if "}" in line:
if end_site:
end_site = False
else:
active = parents[active]
continue
offmatch = re.match(
r"\s*OFFSET\s+([\-\d\.e]+)\s+([\-\d\.e]+)\s+([\-\d\.e]+)", line)
if offmatch:
if not end_site:
offsets[active] = np.array(
[list(map(float, offmatch.groups()))])
continue
chanmatch = re.match(r"\s*CHANNELS\s+(\d+)", line)
if chanmatch:
channels = int(chanmatch.group(1))
if order is None:
channelis = 0 if channels == 3 else 3
channelie = 3 if channels == 3 else 6
parts = line.split()[2 + channelis:2 + channelie]
if any([p not in channelmap for p in parts]):
continue
order = "".join([channelmap[p] for p in parts])
continue
""" Modified line read to handle mixamo data """
# jmatch = re.match("\s*JOINT\s+(\w+)", line)
jmatch = re.match("\s*JOINT\s+(\w+:?\w+)", line)
if jmatch:
names.append(jmatch.group(1))
offsets = np.append(offsets, np.array([[0, 0, 0]]), axis=0)
orients.qs = np.append(orients.qs,
np.array([[1, 0, 0, 0]]),
axis=0)
parents = np.append(parents, active)
active = (len(parents) - 1)
continue
if "End Site" in line:
end_site = True
continue
fmatch = re.match("\s*Frames:\s+(\d+)", line)
if fmatch:
if start and end:
fnum = (end - start) - 1
else:
fnum = int(fmatch.group(1))
jnum = len(parents)
positions = offsets[np.newaxis].repeat(fnum, axis=0)
rotations = np.zeros((fnum, len(orients), 3))
continue
fmatch = re.match("\s*Frame Time:\s+([\d\.]+)", line)
if fmatch:
frametime = float(fmatch.group(1))
continue
if (start and end) and (i < start or i >= end - 1):
i += 1
continue
dmatch = line.strip().split(' ')
if dmatch:
data_block = np.array(list(map(float, dmatch)))
N = len(parents)
fi = i - start if start else i
if channels == 3:
positions[fi, 0:1] = data_block[0:3]
rotations[fi, :] = data_block[3:].reshape(N, 3)
elif channels == 6:
data_block = data_block.reshape(N, 6)
positions[fi, :] = data_block[:, 0:3]
rotations[fi, :] = data_block[:, 3:6]
elif channels == 9:
positions[fi, 0] = data_block[0:3]
data_block = data_block[3:].reshape(N - 1, 9)
rotations[fi, 1:] = data_block[:, 3:6]
positions[fi, 1:] += data_block[:, 0:3] * data_block[:, 6:9]
else:
raise Exception("Too many channels! %i" % channels)
i += 1
f.close()
eulers = np.radians(rotations)
rotations = Quaternions.from_euler(eulers, order=order, world=world)
# print("BVH.load: {} done. ({} order)".format(filename, order))
return Animation(rotations, positions, orients, offsets, parents,
eulers), names, frametime
def rotations_parents_global(anim):
rotations = rotations_global(anim)
rotations = rotations[:, anim.parents]
rotations[:, 0] = Quaternions.id(len(anim))
return rotations
def load_from_maya(root, start, end):
"""
Load Animation Object from Maya Joint Skeleton
Parameters
----------
root : PyNode
Root Joint of Maya Skeleton
start, end : int, int
Start and End frame index of Maya Animation
Returns
-------
animation : Animation
Loaded animation from maya
names : [str]
Joint names from maya
"""
import pymel.core as pm
original_time = pm.currentTime(q=True)
pm.currentTime(start)
""" Build Structure """
names, parents = astruct.load_from_maya(root)
descendants = astruct.descendants_list(parents)
orients = Quaternions.id(len(names))
offsets = np.array([pm.xform(j, q=True, translation=True) for j in names])
for j, name in enumerate(names):
scale = pm.xform(pm.PyNode(name), q=True, scale=True, relative=True)
if len(descendants[j]) == 0: continue
offsets[descendants[j]] *= scale
""" Load Animation """
eulers = np.zeros((end - start, len(names), 3))
positions = np.zeros((end - start, len(names), 3))
rotations = Quaternions.id((end - start, len(names)))
for i in range(end - start):
pm.currentTime(start + i + 1, u=True)
scales = {}
for j, name, parent in zip(range(len(names)), names, parents):
node = pm.PyNode(name)
if i == 0 and pm.hasAttr(node, 'jointOrient'):
ort = node.getOrientation()
orients[j] = Quaternions(
np.array([ort[3], ort[0], ort[1], ort[2]]))
if pm.hasAttr(node, 'rotate'):
eulers[i, j] = np.radians(pm.xform(node, q=True,
rotation=True))
if pm.hasAttr(node, 'translate'):
positions[i, j] = pm.xform(node, q=True, translation=True)
if pm.hasAttr(node, 'scale'):
scales[j] = pm.xform(node, q=True, scale=True, relative=True)
for j in scales:
if len(descendants[j]) == 0: continue
positions[i, descendants[j]] *= scales[j]
positions[i, 0] = pm.xform(root,
q=True,
translation=True,
worldSpace=True)
rotations = orients[np.newaxis] * Quaternions.from_euler(
eulers, order='xyz', world=True)
""" Done """
pm.currentTime(original_time)
return Animation(rotations, positions, orients, offsets, parents), names
def load_to_maya(anim, names=None, radius=0.5):
"""
Load Animation Object into Maya as Joint Skeleton
loads each frame as a new keyfame in maya.
If the animation is too slow or too fast perhaps
the framerate needs adjusting before being loaded
such that it matches the maya scene framerate.
Parameters
----------
anim : Animation
Animation to load into Scene
names : [str]
Optional list of Joint names for Skeleton
Returns
-------
List of Maya Joint Nodes loaded into scene
"""
import pymel.core as pm
joints = []
frames = range(1, len(anim) + 1)
if names is None:
names = ["joint_" + str(i) for i in range(len(anim.parents))]
for i, offset, orient, parent, name in zip(range(len(anim.offsets)),
anim.offsets, anim.orients,
anim.parents, names):
if parent < 0:
pm.select(d=True)
else:
pm.select(joints[parent])
joint = pm.joint(n=name, p=offset, relative=True, radius=radius)
joint.setOrientation([orient[1], orient[2], orient[3], orient[0]])
curvex = pm.nodetypes.AnimCurveTA(n=name + "_rotateX")
curvey = pm.nodetypes.AnimCurveTA(n=name + "_rotateY")
curvez = pm.nodetypes.AnimCurveTA(n=name + "_rotateZ")
jrotations = (-Quaternions(orient[np.newaxis]) *
anim.rotations[:, i]).euler()
curvex.addKeys(frames, jrotations[:, 0])
curvey.addKeys(frames, jrotations[:, 1])
curvez.addKeys(frames, jrotations[:, 2])
pm.connectAttr(curvex.output, joint.rotateX)
pm.connectAttr(curvey.output, joint.rotateY)
pm.connectAttr(curvez.output, joint.rotateZ)
offsetx = pm.nodetypes.AnimCurveTU(n=name + "_translateX")
offsety = pm.nodetypes.AnimCurveTU(n=name + "_translateY")
offsetz = pm.nodetypes.AnimCurveTU(n=name + "_translateZ")
offsetx.addKeys(frames, anim.positions[:, i, 0])
offsety.addKeys(frames, anim.positions[:, i, 1])
offsetz.addKeys(frames, anim.positions[:, i, 2])
pm.connectAttr(offsetx.output, joint.translateX)
pm.connectAttr(offsety.output, joint.translateY)
pm.connectAttr(offsetz.output, joint.translateZ)
joints.append(joint)
return joints
def test_mixamo(model, path, device, max_steps=None, testset='nkn'):
suffix = ''
if max_steps:
suffix += "max{}frame".format(max_steps)
else:
suffix += "fullseq"
suffix += testset
min_steps = 120
if testset == 'paper':
min_steps = 0
with open('./datasets/mixamoquatdirect_train_large.bin', 'rb') as binary:
b = pickle.load(binary)
f_np = EasyDict(b['f'])
local_mean = f_np.local.mean
local_std = f_np.local.std
rtj_mean = f_np.rtj.mean
rtj_std = f_np.rtj.std
f = EasyDict({
'local': {
'mean': torch.from_numpy(local_mean).float().to(device),
'std': torch.from_numpy(local_std).float().to(device),
},
'rtj': {
'mean': torch.from_numpy(rtj_mean).float().to(device),
'std': torch.from_numpy(rtj_std).float().to(device),
},
})
diff_worlds, diff_locals, diff_rtjs, vels, \
labels, bl_diffs, blratio_diffs \
= list(), list(), list(), list(), list(), list(), list()
(inpjoints, inpanims, inpquats, inplocals, inprtjs, inpskels, from_names,
tgtjoints, tgtanims, tgtquats, tgtlocals, tgtrtjs, tgtskels, to_names,
gtanims) = \
load_testdata_root(min_steps, max_steps, b['scale'], testset)
gt_world_pos, keys = list(), list()
inp_heights = get_skel_height_np(np.asarray(inpskels))
gt_heights = get_skel_height_np(np.asarray(tgtskels))
for i in range(len(tgtlocals)):
gt = put_in_world(tgtlocals[i].copy(), tgtrtjs[i].copy(),
tgtanims[i][0].rotations[:, 0])
km = from_names[i].split("_")[0:2]
from_ = from_names[i][:sum([len(l) for l in km]) + 1]
kc = to_names[i].split("_")[0:2]
to_ = to_names[i][:sum([len(l) for l in kc]) + 1]
key = from_ + "/" + to_
gt_world_pos.append(gt)
keys.append(key)
n_inpskels, n_tgtskels = list(), list()
for i in range(len(inplocals)):
inplocals[i] = (inplocals[i] - local_mean) / local_std
inprtjs[i] = (inprtjs[i] - rtj_mean) / rtj_std
n_inpskels.append((inpskels[i] - local_mean) / local_std)
n_tgtskels.append((tgtskels[i] - local_mean) / local_std)
results_dir = os.path.join(path, "test_{}".format(suffix))
with trange(len(inplocals)) as tq:
for i in tq:
tq.set_description('Mixamo Testing')
steps = inplocals[i].shape[0]
tq.set_postfix(frame=steps)
if not os.path.exists(results_dir):
os.makedirs(results_dir)
quatA = torch.tensor(inpquats[i],
dtype=torch.float32,
device=device).view([1, steps, 22, 4])
rtjA = torch.tensor(inprtjs[i], dtype=torch.float32,
device=device).view(1, steps, 3)
skelA = np.asarray(n_inpskels[i])[0:1]
skelA = torch.tensor(skelA, dtype=torch.float32,
device=device).view(1, 1, 22, 3)
skelB = np.asarray(n_tgtskels[i])[0:1]
skelB = torch.tensor(skelB, dtype=torch.float32,
device=device).view(1, 1, 22, 3)
dlocalB = torch.tensor(tgtlocals[i],
dtype=torch.float32,
device=device)
drtjB = torch.tensor(tgtrtjs[i],
dtype=torch.float32,
device=device).view(1, steps, 3)
with torch.no_grad():
_, (clocalB, cdlocalB, crtjB, cquatB,
_) = model(quatA, rtjA, skelA, skelB)
cdrtjB = crtjB * f.rtj.std + f.rtj.mean
init_diff = drtjB[:, 0:1] - cdrtjB[:, 0:1]
cdrtjB = cdrtjB + init_diff
diff_local = ((cdlocalB -
dlocalB)**2).sum(dim=-1).cpu().numpy()[0]
diff_local = 1. / gt_heights[i] * diff_local
diff_rtj = (cdrtjB - drtjB)**2
diff_rtj = diff_rtj.sum(dim=-1).cpu().numpy()[0]
outputB = put_in_world_batch(cdlocalB, cdrtjB,
cquatB[:, :, 0]).cpu().numpy()[0]
diff_world = 1. / gt_heights[i] * (
(outputB - gt_world_pos[i])**2).sum(axis=-1)
variance = np.std(gt_world_pos[i], axis=0)**2
variance = 1. / gt_heights[i] * variance
tgtbls = get_bonelengths(tgtskels[i])
inpbls = get_bonelengths(inpskels[i])
bl_diff, blratio_diff = compare_bls(tgtbls, inpbls)
labels.append(keys[i])
diff_worlds.append(diff_world)
diff_locals.append(diff_local)
diff_rtjs.append(diff_rtj)
vels.append(variance)
bl_diffs.append(bl_diff)
blratio_diffs.append(blratio_diff)
cdlocalB = cdlocalB.cpu().numpy()[0]
cdrtjB = cdrtjB.cpu().numpy()[0]
cquatB = cquatB.cpu().numpy()[0]
# create bvh path
if "Big_Vegas" in from_names[i]:
from_bvh = from_names[i].replace("Big_Vegas", "Vegas")
else:
from_bvh = from_names[i]
if "Warrok_W_Kurniawan" in to_names[i]:
to_bvh = to_names[i].replace("Warrok_W_Kurniawan", "Warrok")
else:
to_bvh = to_names[i]
bvh_path = os.path.join(path, 'blender_files_{}'.format(suffix),
to_bvh.split("_")[-1])
if not os.path.exists(bvh_path):
os.makedirs(bvh_path)
bvh_path += "/{0:05d}".format(i)
# save input sequence
inpanim, inpnames, inpftime = inpanims[i]
BVH.save(
bvh_path + "_from=" + from_bvh + "_to=" + to_bvh + "_inp.bvh",
inpanim, inpnames, inpftime)
# save ground truth
gtanim, gtnames, gtftime = gtanims[i]
BVH.save(
bvh_path + "_from=" + from_bvh + "_to=" + to_bvh + "_gt.bvh",
gtanim, gtnames, gtftime)
# save the retargeted result
tgtanim, tgtnames, tgtftime = tgtanims[i]
"""Exclude angles in exclude_list as they will rotate non-existent
children During training."""
exclude_list = [5, 17, 21, 9, 13]
anim_joints = []
quat_joints = []
for l in range(len(tgtjoints[i])):
if l not in exclude_list:
anim_joints.append(
tgtjoints[i]
[l]) # excluded joint index w.r.t all joints
quat_joints.append(l) # excluded joint index
root_rotB = Quaternions(cquatB[:, 0])
worldB = put_in_world(cdlocalB, cdrtjB, root_rotB)
tgtanim.positions[:, 0] = worldB[:, 0].copy()
excluded_quatB = cquatB[:, quat_joints].copy()
excluded_quatB[:, 0, :] = root_rotB.qs
tgtanim.rotations.qs[:, anim_joints] = excluded_quatB
BVH.save(bvh_path + "_from=" + from_bvh + "_to=" + to_bvh + ".bvh",
tgtanim, tgtnames, tgtftime)
if testset == 'nkn':
print("Create report to {}...".format(results_dir))
create_report(diff_worlds, diff_locals, diff_rtjs, vels, labels,
bl_diffs, blratio_diffs, inp_heights, gt_heights, suffix,
results_dir)
print("Done.")