def __init__(self, batch_size, alpha, beta, gamma, omega, euler_ord, n_joints,
layers_units, max_len, dmean, dstd, omean, ostd,
parents, keep_prob, logs_dir, learning_rate, optim_name, margin,
d_arch, d_rand, norm_type, is_train=True):
self.n_joints = n_joints
self.batch_size = batch_size
self.alpha = alpha
self.beta = beta
self.gamma = gamma
self.omega = omega
self.euler_ord = euler_ord
self.kp = keep_prob
self.max_len = max_len
self.learning_rate = learning_rate
self.fk = FK()
self.d_arch = d_arch
self.d_rand = d_rand
self.margin = margin
self.fake_score = 0.5
self.norm_type = norm_type
self.realSeq_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len,
3 * n_joints + 4],
name="realSeq")
self.realSkel_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints],
name="realSkel")
self.seqA_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints + 4],
name="seqA")
self.seqB_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints + 4],
name="seqB")
self.skelA_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints],
name="skelA")
self.skelB_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints],
name="skelB")
self.aeReg_ = tf.placeholder(tf.float32,
shape=[batch_size, 1],
name="aeReg")
self.mask_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len],
name="mask")
self.kp_ = tf.placeholder(tf.float32, shape=[], name="kp")
enc_gru = self.gru_model(layers_units)
dec_gru = self.gru_model(layers_units)
b_outputs = []
b_offsets = []
b_quats = []
a_outputs = []
a_offsets = []
a_quats = []
reuse = False
statesA_AB = ()
statesB_AB = ()
statesA_BA = ()
statesB_BA = ()
for units in layers_units:
statesA_AB += (tf.zeros([batch_size, units]),)
statesB_AB += (tf.zeros([batch_size, units]),)
statesA_BA += (tf.zeros([batch_size, units]),)
statesB_BA += (tf.zeros([batch_size, units]),)
for t in range(max_len):
"""Retarget A to B"""
with tf.variable_scope("Encoder", reuse=reuse):
ptA_in = self.seqA_[:,t,:]
_, statesA_AB = tf.contrib.rnn.static_rnn(
enc_gru, [ptA_in], initial_state=statesA_AB,
dtype=tf.float32
)
with tf.variable_scope("Decoder", reuse=reuse):
if t == 0:
ptB_in = tf.zeros([batch_size, 3 * n_joints + 4])
else:
ptB_in = tf.concat([b_outputs[-1], b_offsets[-1]], axis=-1)
ptcombined = tf.concat(
values=[self.skelB_[:,0,3:], ptB_in, statesA_AB[-1]], axis=1
)
_, statesB_AB = tf.contrib.rnn.static_rnn(
dec_gru, [ptcombined], initial_state=statesB_AB,
dtype=tf.float32
)
angles_n_offset = self.mlp_out(statesB_AB[-1])
output_angles = tf.reshape(angles_n_offset[:,:-4],
[batch_size, n_joints, 4])
b_offsets.append(angles_n_offset[:,-4:])
b_quats.append(self.normalized(output_angles))
skel_in = tf.reshape(self.skelB_[:,0,:], [batch_size, n_joints, 3])
skel_in = skel_in * dstd + dmean
output = (self.fk.run(parents, skel_in, output_angles) - dmean) / dstd
output = tf.reshape(output, [batch_size, -1])
b_outputs.append(output)
"""Retarget B back to A"""
with tf.variable_scope("Encoder", reuse=True):
ptB_in = tf.concat([b_outputs[-1], b_offsets[-1]], axis=-1)
_, statesB_BA = tf.contrib.rnn.static_rnn(
enc_gru, [ptB_in], initial_state=statesB_BA,
dtype=tf.float32
)
with tf.variable_scope("Decoder", reuse=True):
if t == 0:
ptA_in = tf.zeros([batch_size, 3 * n_joints + 4])
else:
ptA_in = tf.concat([a_outputs[-1], a_offsets[-1]], axis=-1)
ptcombined = tf.concat(
values=[self.skelA_[:,0,3:], ptA_in, statesB_BA[-1]], axis=1
)
_, statesA_BA = tf.contrib.rnn.static_rnn(
dec_gru, [ptcombined], initial_state=statesA_BA,
dtype=tf.float32
)
angles_n_offset = self.mlp_out(statesA_BA[-1])
output_angles = tf.reshape(angles_n_offset[:,:-4],
[batch_size, n_joints, 4])
a_offsets.append(angles_n_offset[:,-4:])
a_quats.append(self.normalized(output_angles))
skel_in = tf.reshape(self.skelA_[:,0,:], [batch_size, n_joints, 3])
skel_in = skel_in * dstd + dmean
output = (self.fk.run(parents, skel_in, output_angles) - dmean) / dstd
output = tf.reshape(output, [batch_size, -1])
a_outputs.append(output)
reuse = True
self.outputB = tf.stack(b_outputs, axis=1)
self.offsetB = tf.stack(b_offsets, axis=1)
self.quatB = tf.stack(b_quats, axis=1)
self.outputA = tf.stack(a_outputs, axis=1)
self.offsetA = tf.stack(a_offsets, axis=1)
self.quatA = tf.stack(a_quats, axis=1)
if is_train:
dmean = dmean.reshape([1,1,-1])
dstd = dstd.reshape([1,1,-1])
with tf.variable_scope("DIS", reuse=False):
if self.d_rand:
dnorm_seq = self.realSeq_[:,:,:-4] * dstd + dmean
dnorm_off = self.realSeq_[:,:,-4:] * ostd + omean
skel_ = self.realSkel_[:,0:1,3:]
else:
dnorm_seq = self.seqA_[:,:,:-4] * dstd + dmean
dnorm_off = self.seqA_[:,:,-4:] * ostd + omean
skel_ = self.skelA_[:,0:1,3:]
diff_seq = dnorm_seq[:,1:,:] - dnorm_seq[:,:-1,:]
real_data = tf.concat(
[diff_seq, dnorm_off[:,:-1,:]], axis=-1
)
self.D_logits = self.discriminator(real_data, skel_)
self.D = tf.nn.sigmoid(self.D_logits)
self.seqB = tf.concat([self.outputB, self.offsetB], axis=-1)
with tf.variable_scope("DIS", reuse=True):
dnorm_seqB = self.seqB[:,:,:-4] * dstd + dmean
dnorm_offB = self.seqB[:,:,-4:] * ostd + omean
diff_seqB = dnorm_seqB[:,1:,:] - dnorm_seqB[:,:-1,:]
fake_data = tf.concat(
[diff_seqB, dnorm_offB[:,:-1,:]], axis=-1
)
self.D_logits_ = self.discriminator(fake_data, self.skelB_[:,0:1,3:])
self.D_ = tf.nn.sigmoid(self.D_logits_)
""" CYCLE OBJECTIVE """
output_seqA = self.outputA
target_seqA = self.seqA_[:,:,:-4]
self.cycle_local_loss = tf.reduce_sum(
tf.square(
tf.multiply(
self.mask_[:,:,None],
tf.subtract(output_seqA, target_seqA)
)
)
)
self.cycle_local_loss = tf.divide(
self.cycle_local_loss, tf.reduce_sum(self.mask_)
)
self.cycle_global_loss = tf.reduce_sum(
tf.square(
tf.multiply(
self.mask_[:,:,None],
tf.subtract(self.seqA_[:,:,-4:], self.offsetA)
)
)
)
self.cycle_global_loss = tf.divide(
self.cycle_global_loss, tf.reduce_sum(self.mask_)
)
dnorm_offA_ = self.offsetA * ostd + omean
self.cycle_smooth = tf.reduce_sum(
tf.square(
tf.multiply(
self.mask_[:,1:,None],
dnorm_offA_[:,1:] - dnorm_offA_[:,:-1]
)
)
)
self.cycle_smooth = tf.divide(
self.cycle_smooth, tf.reduce_sum(self.mask_)
)
""" INTERMEDIATE OBJECTIVE FOR REGULARIZATION """
output_seqB = self.outputB
target_seqB = self.seqB_[:,:,:-4]
self.interm_local_loss = tf.reduce_sum(
tf.square(
tf.multiply(
self.aeReg_[:,:,None] * self.mask_[:,:,None],
tf.subtract(output_seqB, target_seqB)
)
)
)
self.interm_local_loss = tf.divide(
self.interm_local_loss,
tf.maximum(tf.reduce_sum(self.aeReg_ * self.mask_), 1)
)
self.interm_global_loss = tf.reduce_sum(
tf.square(
tf.multiply(
self.aeReg_[:,:,None] * self.mask_[:,:,None],
tf.subtract(self.seqB_[:,:,-4:], self.offsetB)
)
)
)
self.interm_global_loss = tf.divide(
self.interm_global_loss,
tf.maximum(tf.reduce_sum(self.aeReg_ * self.mask_), 1)
)
dnorm_offB_ = self.offsetB * ostd + omean
self.interm_smooth = tf.reduce_sum(
tf.square(
tf.multiply(
self.mask_[:,1:,None],
dnorm_offB_[:,1:] - dnorm_offB_[:,:-1]
)
)
)
self.interm_smooth = tf.divide(
self.interm_smooth,
tf.maximum(tf.reduce_sum(self.mask_), 1)
)
rads = self.alpha / 180.0
self.twist_loss1 = tf.reduce_mean(
tf.square(
tf.maximum(
0.0, tf.abs(self.euler(self.quatB, euler_ord)) - rads * np.pi
)
)
)
self.twist_loss2 = tf.reduce_mean(
tf.square(
tf.maximum(
0.0, tf.abs(self.euler(self.quatA, euler_ord)) - rads * np.pi
)
)
)
"""Twist loss"""
self.twist_loss = 0.5 * (self.twist_loss1 + self.twist_loss2)
"""Acceleration smoothness loss"""
self.smoothness = 0.5 * (self.interm_smooth + self.cycle_smooth)
self.overall_loss = (self.cycle_local_loss + self.cycle_global_loss +
self.interm_local_loss + self.interm_global_loss +
self.gamma * self.twist_loss +
self.omega * self.smoothness)
self.L_disc_real = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.D_logits,
labels=tf.ones_like(self.D_logits)
)
)
self.L_disc_fake = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.D_logits_,
labels=tf.zeros_like(self.D_logits_)
)
)
self.L_disc = self.L_disc_real + self.L_disc_fake
self.L_gen = tf.reduce_sum(
tf.multiply(
(1 - self.aeReg_),
tf.nn.sigmoid_cross_entropy_with_logits(
logits=self.D_logits_,
labels=tf.ones_like(self.D_logits_)
)
)
)
self.L_gen = tf.divide(
self.L_gen, tf.maximum(tf.reduce_sum(1 - self.aeReg_), 1)
)
self.L = self.beta * self.L_gen + self.overall_loss
cycle_local_sum = tf.summary.scalar(
"losses/cycle_local_loss", self.cycle_local_loss
)
cycle_global_sum = tf.summary.scalar(
"losses/cycle_global_loss", self.cycle_global_loss
)
interm_local_sum = tf.summary.scalar(
"losses/interm_local_loss", self.interm_local_loss
)
interm_global_sum = tf.summary.scalar(
"losses/interm_global_loss", self.interm_global_loss
)
twist_sum = tf.summary.scalar("losses/twist_loss", self.twist_loss)
smooth_sum = tf.summary.scalar("losses/smoothness", self.smoothness)
Ldiscreal_sum = tf.summary.scalar("losses/disc_real", self.L_disc_real)
Ldiscfake_sum = tf.summary.scalar("losses/disc_fake", self.L_disc_fake)
Lgen_sum = tf.summary.scalar("losses/disc_gen", self.L_gen)
self.sum = tf.summary.merge([cycle_local_sum, cycle_global_sum,
interm_local_sum, interm_global_sum,
Lgen_sum, Ldiscreal_sum, Ldiscfake_sum,
twist_sum, smooth_sum])
self.writer = tf.summary.FileWriter(logs_dir, tf.get_default_graph())
self.allvars = tf.trainable_variables()
self.gvars = [v for v in self.allvars if "DIS" not in v.name]
self.dvars = [v for v in self.allvars if "DIS" in v.name]
if optim_name == "rmsprop":
goptimizer = tf.train.RMSPropOptimizer(
self.learning_rate, name="goptimizer"
)
doptimizer = tf.train.RMSPropOptimizer(
self.learning_rate, name="doptimizer"
)
elif optim_name == "adam":
goptimizer = tf.train.AdamOptimizer(
self.learning_rate, beta1=0.5, name="goptimizer"
)
doptimizer = tf.train.AdamOptimizer(
self.learning_rate, beta1=0.5, name="doptimizer"
)
else:
raise Exception("Unknown optimizer")
ggradients, gg = zip(*goptimizer.compute_gradients(
self.L, var_list=self.gvars
))
dgradients, dg = zip(*doptimizer.compute_gradients(
self.L_disc, var_list=self.dvars
))
ggradients, _ = tf.clip_by_global_norm(ggradients, 25)
dgradients, _ = tf.clip_by_global_norm(dgradients, 25)
self.goptim = goptimizer.apply_gradients(
zip(ggradients, gg)
)
self.doptim = doptimizer.apply_gradients(
zip(dgradients, dg)
)
num_param=0
for var in self.gvars:
num_param+=int(np.prod(var.get_shape()));
print "NUMBER OF G PARAMETERS: " + str(num_param)
num_param=0
for var in self.dvars:
num_param+=int(np.prod(var.get_shape()));
print "NUMBER OF D PARAMETERS: " + str(num_param)
self.saver = tf.train.Saver()
def __init__(self,
batch_size,
alpha,
gamma,
omega,
euler_ord,
n_joints,
layers_units,
max_len,
dmean,
dstd,
omean,
ostd,
parents,
keep_prob,
logs_dir,
learning_rate,
optim_name,
is_train=True):
self.n_joints = n_joints
self.batch_size = batch_size
self.alpha = alpha
self.gamma = gamma
self.omega = omega
self.euler_ord = euler_ord
self.kp = keep_prob
self.max_len = max_len
self.learning_rate = learning_rate
self.fk = FK()
self.seqA_ = tf.placeholder(
tf.float32,
shape=[batch_size, max_len, 3 * n_joints + 4],
name="seqA")
self.seqB_ = tf.placeholder(
tf.float32,
shape=[batch_size, max_len, 3 * n_joints + 4],
name="seqB")
self.skelA_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints],
name="skelA")
self.skelB_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len, 3 * n_joints],
name="skelB")
self.mask_ = tf.placeholder(tf.float32,
shape=[batch_size, max_len],
name="mask")
enc_gru = self.gru_model(layers_units)
dec_gru = self.gru_model(layers_units)
b_outputs = []
b_offsets = []
b_quats = []
a_outputs = []
a_offsets = []
a_quats = []
reuse = False
statesA_AB = ()
statesB_AB = ()
statesA_BA = ()
statesB_BA = ()
for units in layers_units:
statesA_AB += (tf.zeros([batch_size, units]), )
statesB_AB += (tf.zeros([batch_size, units]), )
statesA_BA += (tf.zeros([batch_size, units]), )
statesB_BA += (tf.zeros([batch_size, units]), )
for t in range(max_len):
""" Retarget A to B """
with tf.variable_scope("Encoder", reuse=reuse):
ptA_in = self.seqA_[:, t, :]
_, statesA_AB = tf.contrib.rnn.static_rnn(
enc_gru, [ptA_in],
initial_state=statesA_AB,
dtype=tf.float32)
with tf.variable_scope("Decoder", reuse=reuse):
if t == 0:
ptB_in = tf.zeros([batch_size, 3 * n_joints + 4])
else:
ptB_in = tf.concat([b_outputs[-1], b_offsets[-1]], axis=-1)
ptcombined = tf.concat(
values=[self.skelB_[:, 0, 3:], ptB_in, statesA_AB[-1]],
axis=1)
_, statesB_AB = tf.contrib.rnn.static_rnn(
dec_gru, [ptcombined],
initial_state=statesB_AB,
dtype=tf.float32)
angles_n_offset = self.mlp_out(statesB_AB[-1])
output_angles = tf.reshape(angles_n_offset[:, :-4],
[batch_size, n_joints, 4])
b_offsets.append(angles_n_offset[:, -4:])
b_quats.append(self.normalized(output_angles))
skel_in = tf.reshape(self.skelB_[:, 0, :],
[batch_size, n_joints, 3])
skel_in = skel_in * dstd + dmean
output = (self.fk.run(parents, skel_in, output_angles) -
dmean) / dstd
output = tf.reshape(output, [batch_size, -1])
b_outputs.append(output)
reuse = True
self.outputB = tf.stack(b_outputs, axis=1)
self.offsetB = tf.stack(b_offsets, axis=1)
self.quatB = tf.stack(b_quats, axis=1)
if is_train:
dmean = dmean.reshape([1, 1, -1])
dstd = dstd.reshape([1, 1, -1])
output_seqB = self.outputB
target_seqB = self.seqB_[:, :, :-4]
self.local_loss = tf.reduce_sum(
tf.square(
tf.multiply(self.mask_[:, :, None],
tf.subtract(output_seqB, target_seqB))))
self.local_loss = tf.divide(
self.local_loss, tf.maximum(tf.reduce_sum(self.mask_), 1))
self.global_loss = tf.reduce_sum(
tf.square(
tf.multiply(
self.mask_[:, :, None],
tf.subtract(self.seqB_[:, :, -4:], self.offsetB))))
self.global_loss = tf.divide(
self.global_loss, tf.maximum(tf.reduce_sum(self.mask_), 1))
dnorm_offB_ = self.offsetB * ostd + omean
self.smoothness = tf.reduce_sum(
tf.square(
tf.multiply(self.mask_[:, 1:, None],
dnorm_offB_[:, 1:] - dnorm_offB_[:, :-1])))
self.smoothness = tf.divide(
self.smoothness, tf.maximum(tf.reduce_sum(self.mask_), 1))
rads = self.alpha / 180.0
self.twist_loss = tf.reduce_mean(
tf.square(
tf.maximum(
0.0,
tf.abs(self.euler(self.quatB, euler_ord)) -
rads * np.pi)))
self.L = (self.local_loss + self.global_loss +
self.gamma * self.twist_loss +
self.omega * self.smoothness)
local_sum = tf.summary.scalar("losses/local_loss", self.local_loss)
global_sum = tf.summary.scalar("losses/global_loss",
self.global_loss)
twist_sum = tf.summary.scalar("losses/twist_loss", self.twist_loss)
smooth_sum = tf.summary.scalar("losses/smoothness",
self.smoothness)
self.sum = tf.summary.merge(
[local_sum, global_sum, twist_sum, smooth_sum])
self.writer = tf.summary.FileWriter(logs_dir,
tf.get_default_graph())
self.allvars = tf.trainable_variables()
self.gvars = [v for v in self.allvars if "DIS" not in v.name]
if optim_name == "rmsprop":
goptimizer = tf.train.RMSPropOptimizer(self.learning_rate,
name="goptimizer")
elif optim_name == "adam":
goptimizer = tf.train.AdamOptimizer(self.learning_rate,
beta1=0.5,
name="goptimizer")
else:
raise Exception("Unknown optimizer")
ggradients, gg = zip(
*goptimizer.compute_gradients(self.L, var_list=self.gvars))
ggradients, _ = tf.clip_by_global_norm(ggradients, 25)
self.goptim = goptimizer.apply_gradients(zip(ggradients, gg))
num_param = 0
for var in self.gvars:
num_param += int(np.prod(var.get_shape()))
print "NUMBER OF G PARAMETERS: " + str(num_param)
self.saver = tf.train.Saver()