def permute_res(self,
current,
out_size,
mode_node,
scope=None,
act_type='relu',
alpha=1.):
with tf.variable_scope(scope):
for layer in np.arange(0, len(out_size)):
sh = current.get_shape().as_list()
in_node = current
with tf.variable_scope('stage_1'):
current = BatchNorm(current, mode_node, scope=str(layer))
current = tf.nn.relu(current)
current = cell2D(current,
1,
sh[2],
sh[-1],
out_size[layer],
mode_node,
1,
'pointconv' + str(layer),
padding='VALID',
bn=False,
act=False)
pooled = tf.reduce_max(current, axis=1, keep_dims=True)
w1 = tf.get_variable('w1_' + str(layer),
initializer=[1.],
trainable=True,
dtype='float32')
current = current + w1 * pooled
with tf.variable_scope('stage_2'):
current = BatchNorm(current, mode_node, scope=str(layer))
current = tf.nn.relu(current)
current = cell2D(current,
1,
sh[2],
sh[-1],
out_size[layer],
mode_node,
1,
'pointconv' + str(layer),
padding='VALID',
bn=False,
act=False)
pooled2 = tf.reduce_max(current, axis=1, keep_dims=True)
w2 = tf.get_variable('w2_' + str(layer),
initializer=[1.],
trainable=True,
dtype='float32')
current = current + w2 * pooled2
current = current + in_node
return current
def resnet(data_node, mode_node, layers):
shape = data_node.get_shape().as_list()
weights = []
ch_in = shape[-1]
data_node_wrap = tf.concat(
(data_node, tf.slice(data_node, [0, 0, 0, 0], [-1, -1, 1, -1])),
axis=2)
# data_node_wrap = tf.concat((data_node_wrap,tf.slice(data_node_wrap,[0,0,0,0],[-1,1,-1,-1])),axis=1)
with tf.variable_scope("input") as SCOPE:
conv1_w, conv1_b = CONV2D([2, 2, ch_in, 32])
c1 = tf.nn.conv2d(data_node_wrap,
conv1_w,
strides=[1, 1, 1, 1],
padding='VALID')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 32, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 32, 64, mode_node, 2, 'r2')
current = cell2D_res(current, 3, 64, 64, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 7, 7, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("fully") as SCOPE:
features = tf.reshape(current, (shape[0], -1))
for ii in range(len(layers) - 1):
layer_in = layers[ii]
layer_out = layers[ii + 1]
w = tf.reshape(
cell1D(features,
layer_in * layer_out,
mode_node,
SCOPE='w' + str(ii),
with_act=False,
with_bn=False), (shape[0], layer_in, layer_out))
b = tf.reshape(
cell1D(features,
layer_out,
mode_node,
SCOPE='b' + str(ii),
with_act=False,
with_bn=False), (shape[0], 1, layer_out))
weights.append({'w': w, 'b': b})
return weights
def deep_prior4_IG(in_node, in_node2, mode_node, batch_size, grid_size,
grid_size_gt):
in_node = tf.reduce_max(in_node, axis=3, keep_dims=True)
in_node = tf.slice(in_node, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1])
in_node = tf.squeeze(in_node, axis=3)
ch_in = in_node.get_shape().as_list()[-1]
with tf.variable_scope("Input") as SCOPE:
conv1_w, conv1_b = CONV2D([5, 5, ch_in, 32])
c1 = tf.nn.conv2d(in_node,
conv1_w,
strides=[1, 1, 1, 1],
padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("Residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 64, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 64, 64, mode_node, 1, 'r2')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 128, 128, mode_node, 1, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = cell2D_res(current, 3, 256, 256, mode_node, 1, 'r6')
current = cell2D_res(current, 3, 256, 512, mode_node, 2, 'r7')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 6, 6, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("Features") as SCOPE:
features = tf.reshape(current, (batch_size, -1))
with tf.variable_scope("Kinematics") as SCOPE:
num_joints = 21
KL = KINETIC.Kinetic(batch_size, num_joints=num_joints)
with tf.variable_scope("Bones_GT") as SCOPE:
bones_gt = bone_logits(in_node2)
bones_gt = tf.transpose(bones_gt, (0, 2, 1))
bones_gt_scale = tf.div(
bones_gt, tf.slice(KL.model['Bones'], [0, 0, 1], [-1, 1, -1]))
bones_gt_scale = tf.concat((tf.zeros(
(batch_size, 1, 1)), bones_gt_scale),
axis=2)
pose_limits_scale = 180 * np.ones((1, 3, 21), dtype=np.float32)
# X axis
pose_limits_scale[:, 0, (4, 8, 12, 16, 20)] = 5 # Tip bones
pose_limits_scale[:, 0, (3, 7, 11, 15, 19)] = 5 #
pose_limits_scale[:, 0, (2, 6, 10, 14,
18)] = 40 # Most of the rotational freedom
pose_limits_scale[:, 0, (1, 5, 9, 13, 17)] = 30 # Metacarpal
# Z axis
pose_limits_scale[:, 2, (4, 8, 12, 16, 20)] = 2 # Tip bones
pose_limits_scale[:, 2, (3, 7, 11, 15, 19)] = 2
pose_limits_scale[:, 2, (2, 6, 10, 14,
18)] = 50 # Most of the rotational freedom
pose_limits_scale[:, 2, (1, 5, 9, 13, 17)] = 30 # Metacarpal
# Y axis
pose_limits_scale[:, 1, (4, 8, 12, 16, 20)] = 80 # Tip bones
pose_limits_scale[:, 1, (3, 7, 11, 15, 19)] = 80
pose_limits_scale[:, 1, (2, 6, 10, 14,
18)] = 80 # Most of the rotational freedom
pose_limits_scale[:, 1, (1, 5, 9, 13, 17)] = 50 # Metacarpal
# Wrist
pose_limits_scale[:, :, 0] = 190
pose_limits_scale_tf = tf.constant(pose_limits_scale / 180.,
dtype=tf.float32)
pose = pose_limits_scale_tf * tf.tanh(
tf.reshape(
cell1D(features,
63,
mode_node,
SCOPE='Pose',
with_act=False,
with_bn=False), (batch_size, 3, 21)))
# bones = 1. + 0.5*tf.tanh(tf.expand_dims(cell1D(features,21, mode_node, SCOPE='Bones', with_act=False, with_bn=False),1))
camera = tf.tanh(
tf.expand_dims(
cell1D(features,
3,
mode_node,
SCOPE='Camera',
with_act=False,
with_bn=False), -1))
pred_cloud, _ = KL.apply_trans(bones_gt_scale, pose, camera)
marks = tf.transpose(pred_cloud, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy") as SCOPE:
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
return logits, pred_cloud
def deep_prior4(in_node, in_node2, mode_node, batch_size, grid_size,
grid_size_gt):
in_node = tf.reduce_max(in_node, axis=3, keep_dims=True)
in_node = tf.slice(in_node, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1])
in_node = tf.squeeze(in_node, axis=3)
# in_node_max = tf.reduce_max(in_node,axis=(1,2,3),keep_dims=True)
# in_node = (tf.div(in_node,in_node_max) - 0.5)*2
ch_in = in_node.get_shape().as_list()[-1]
num_params = 63
with tf.variable_scope("input") as SCOPE:
conv1_w, conv1_b = CONV2D([5, 5, ch_in, 32])
c1 = tf.nn.conv2d(in_node,
conv1_w,
strides=[1, 1, 1, 1],
padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 64, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 64, 64, mode_node, 1, 'r2')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 128, 128, mode_node, 1, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = cell2D_res(current, 3, 256, 256, mode_node, 1, 'r6')
current = cell2D_res(current, 3, 256, 512, mode_node, 2, 'r7')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 6, 6, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("fully") as SCOPE:
features = tf.reshape(current, (batch_size, -1))
current = cell1D(features,
num_params,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
marks = tf.tanh(tf.reshape(current, (batch_size, 3, 21)))
pred_cloud = tf.transpose(marks, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy") as SCOPE:
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
return logits, pred_cloud