def resnet_50(example,params,mode):
# in_node = example['orthographic_image']
in_node = example
ch_in = in_node.get_shape().as_list()[-1]
with tf.variable_scope("Input"):
conv1_w, conv1_b = CONV2D([7,7,ch_in,64])
c1 = tf.nn.conv2d(in_node,conv1_w,strides=[1, 1, 1, 1],padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
c1 = max_pool(c1, kernel_size=3, stride=2, padding='SAME')
with tf.variable_scope("Residuals") as SCOPE:
current = cell2D_res_bottleneck(c1, 3, 64, 256, mode, 1, 'r1')
current = cell2D_res_bottleneck(current, 3, 64, 256, mode, 1, 'r2')
current = cell2D_res_bottleneck(current, 3, 64, 256, mode, 1, 'r3')
current = cell2D_res_bottleneck(current, 3, 128, 512, mode, 2, 'r4')
current = cell2D_res_bottleneck(current, 3, 128, 512, mode, 1, 'r5')
current = cell2D_res_bottleneck(current, 3, 128, 512, mode, 1, 'r6')
current = cell2D_res_bottleneck(current, 3, 128, 512, mode, 1, 'r7')
current = cell2D_res_bottleneck(current, 3, 256, 1024, mode, 2, 'r8')
current = cell2D_res_bottleneck(current, 3, 256, 1024, mode, 1, 'r9')
current = cell2D_res_bottleneck(current, 3, 256, 1024, mode, 1, 'r10')
current = cell2D_res_bottleneck(current, 3, 256, 1024, mode, 1, 'r11')
current = cell2D_res_bottleneck(current, 3, 256, 1024, mode, 1, 'r12')
current = cell2D_res_bottleneck(current, 3, 256, 1024, mode, 1, 'r13')
current = cell2D_res_bottleneck(current, 3, 512, 2048, mode, 2, 'r14')
current = cell2D_res_bottleneck(current, 3, 512, 2048, mode, 1, 'r15')
current = cell2D_res_bottleneck(current, 3, 512, 2048, mode, 1, 'r16')
# current = BatchNorm(current,mode,SCOPE)
featue_size = current.get_shape().as_list()[1]
current = tf.nn.avg_pool(current,[1,featue_size,featue_size,1],[1,1,1,1],padding='VALID')
size_pool = current.get_shape().as_list()
def resnet(data_node, mode_node, layers):
shape = data_node.get_shape().as_list()
weights = []
ch_in = shape[-1]
data_node_wrap = tf.concat(
(data_node, tf.slice(data_node, [0, 0, 0, 0], [-1, -1, 1, -1])),
axis=2)
# data_node_wrap = tf.concat((data_node_wrap,tf.slice(data_node_wrap,[0,0,0,0],[-1,1,-1,-1])),axis=1)
with tf.variable_scope("input") as SCOPE:
conv1_w, conv1_b = CONV2D([2, 2, ch_in, 32])
c1 = tf.nn.conv2d(data_node_wrap,
conv1_w,
strides=[1, 1, 1, 1],
padding='VALID')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 32, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 32, 64, mode_node, 2, 'r2')
current = cell2D_res(current, 3, 64, 64, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 7, 7, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("fully") as SCOPE:
features = tf.reshape(current, (shape[0], -1))
for ii in range(len(layers) - 1):
layer_in = layers[ii]
layer_out = layers[ii + 1]
w = tf.reshape(
cell1D(features,
layer_in * layer_out,
mode_node,
SCOPE='w' + str(ii),
with_act=False,
with_bn=False), (shape[0], layer_in, layer_out))
b = tf.reshape(
cell1D(features,
layer_out,
mode_node,
SCOPE='b' + str(ii),
with_act=False,
with_bn=False), (shape[0], 1, layer_out))
weights.append({'w': w, 'b': b})
return weights
def resnet_34(example,args_):
mode = args_[0]
base_size = args_[1]
in_node = example
ch_in = in_node.get_shape().as_list()[-1]
with tf.variable_scope("Input"):
conv0_w, conv0_b = CONV2D([5,5,ch_in,base_size])
c0 = tf.nn.conv2d(in_node,conv0_w,strides=[1, 1, 1, 1],padding='SAME')
c0 = tf.nn.bias_add(c0, conv0_b)
# c0 = tf.nn.max_pool(c0, ksize=[1,3,3,1], strides=(1,2,2,1), padding='SAME')
# c1 = BatchNorm(c1,mode,SCOPE)
# c1 = tf.nn.relu(c1)
with tf.variable_scope("Residuals"):
current = cell2D_res(c0, 3, base_size, base_size, mode, 2, 'r1')
current = cell2D_res(current, 3, base_size, base_size, mode, 1, 'r2')
current = cell2D_res(current, 3, base_size, base_size, mode, 1, 'r3')
current = cell2D_res(current, 3, base_size, base_size*2, mode, 2, 'r4')
current = cell2D_res(current, 3, base_size*2, base_size*2, mode, 1, 'r5')
current = cell2D_res(current, 3, base_size*2, base_size*2, mode, 1, 'r6')
current = cell2D_res(current, 3, base_size*2, base_size*2, mode, 1, 'r7')
current = cell2D_res(current, 3, base_size*2, base_size*4, mode, 2, 'r8')
current = cell2D_res(current, 3, base_size*4, base_size*4, mode, 1, 'r9')
current = cell2D_res(current, 3, base_size*4, base_size*4, mode, 1, 'r10')
current = cell2D_res(current, 3, base_size*4, base_size*4, mode, 1, 'r11')
current = cell2D_res(current, 3, base_size*4, base_size*4, mode, 1, 'r12')
current = cell2D_res(current, 3, base_size*4, base_size*4, mode, 1, 'r13')
current = cell2D_res(current, 3, base_size*4, base_size*8, mode, 2, 'r14')
current = cell2D_res(current, 3, base_size*8, base_size*8, mode, 1, 'r15')
current = cell2D_res(current, 3, base_size*8, base_size*8, mode, 1, 'r16')
with tf.variable_scope('Pooling'):
# current = BatchNorm(current,mode,SCOPE)
featue_size = current.get_shape().as_list()
current = tf.nn.avg_pool(current,[1,featue_size[1],featue_size[2],1],[1,1,1,1],padding='VALID')
# batch1 = BatchNorm(current,mode,scope)
# relu1 = tf.nn.relu(batch1)
# conv2_w, _ = CONV2D([featue_size[1],featue_size[2],featue_size[3],1],learn_bias=False)
# features = tf.nn.depthwise_conv2d_native(relu1,conv2_w,strides=[1, 1, 1, 1],padding='VALID')
return tf.squeeze(current,axis=(1,2))
def deep_prior4_IG(in_node, in_node2, mode_node, batch_size, grid_size,
grid_size_gt):
in_node = tf.reduce_max(in_node, axis=3, keep_dims=True)
in_node = tf.slice(in_node, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1])
in_node = tf.squeeze(in_node, axis=3)
ch_in = in_node.get_shape().as_list()[-1]
with tf.variable_scope("Input") as SCOPE:
conv1_w, conv1_b = CONV2D([5, 5, ch_in, 32])
c1 = tf.nn.conv2d(in_node,
conv1_w,
strides=[1, 1, 1, 1],
padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("Residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 64, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 64, 64, mode_node, 1, 'r2')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 128, 128, mode_node, 1, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = cell2D_res(current, 3, 256, 256, mode_node, 1, 'r6')
current = cell2D_res(current, 3, 256, 512, mode_node, 2, 'r7')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 6, 6, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("Features") as SCOPE:
features = tf.reshape(current, (batch_size, -1))
with tf.variable_scope("Kinematics") as SCOPE:
num_joints = 21
KL = KINETIC.Kinetic(batch_size, num_joints=num_joints)
with tf.variable_scope("Bones_GT") as SCOPE:
bones_gt = bone_logits(in_node2)
bones_gt = tf.transpose(bones_gt, (0, 2, 1))
bones_gt_scale = tf.div(
bones_gt, tf.slice(KL.model['Bones'], [0, 0, 1], [-1, 1, -1]))
bones_gt_scale = tf.concat((tf.zeros(
(batch_size, 1, 1)), bones_gt_scale),
axis=2)
pose_limits_scale = 180 * np.ones((1, 3, 21), dtype=np.float32)
# X axis
pose_limits_scale[:, 0, (4, 8, 12, 16, 20)] = 5 # Tip bones
pose_limits_scale[:, 0, (3, 7, 11, 15, 19)] = 5 #
pose_limits_scale[:, 0, (2, 6, 10, 14,
18)] = 40 # Most of the rotational freedom
pose_limits_scale[:, 0, (1, 5, 9, 13, 17)] = 30 # Metacarpal
# Z axis
pose_limits_scale[:, 2, (4, 8, 12, 16, 20)] = 2 # Tip bones
pose_limits_scale[:, 2, (3, 7, 11, 15, 19)] = 2
pose_limits_scale[:, 2, (2, 6, 10, 14,
18)] = 50 # Most of the rotational freedom
pose_limits_scale[:, 2, (1, 5, 9, 13, 17)] = 30 # Metacarpal
# Y axis
pose_limits_scale[:, 1, (4, 8, 12, 16, 20)] = 80 # Tip bones
pose_limits_scale[:, 1, (3, 7, 11, 15, 19)] = 80
pose_limits_scale[:, 1, (2, 6, 10, 14,
18)] = 80 # Most of the rotational freedom
pose_limits_scale[:, 1, (1, 5, 9, 13, 17)] = 50 # Metacarpal
# Wrist
pose_limits_scale[:, :, 0] = 190
pose_limits_scale_tf = tf.constant(pose_limits_scale / 180.,
dtype=tf.float32)
pose = pose_limits_scale_tf * tf.tanh(
tf.reshape(
cell1D(features,
63,
mode_node,
SCOPE='Pose',
with_act=False,
with_bn=False), (batch_size, 3, 21)))
# bones = 1. + 0.5*tf.tanh(tf.expand_dims(cell1D(features,21, mode_node, SCOPE='Bones', with_act=False, with_bn=False),1))
camera = tf.tanh(
tf.expand_dims(
cell1D(features,
3,
mode_node,
SCOPE='Camera',
with_act=False,
with_bn=False), -1))
pred_cloud, _ = KL.apply_trans(bones_gt_scale, pose, camera)
marks = tf.transpose(pred_cloud, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy") as SCOPE:
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
return logits, pred_cloud
def deep_prior4(in_node, in_node2, mode_node, batch_size, grid_size,
grid_size_gt):
in_node = tf.reduce_max(in_node, axis=3, keep_dims=True)
in_node = tf.slice(in_node, [0, 0, 0, 0, 2], [-1, -1, -1, -1, 1])
in_node = tf.squeeze(in_node, axis=3)
# in_node_max = tf.reduce_max(in_node,axis=(1,2,3),keep_dims=True)
# in_node = (tf.div(in_node,in_node_max) - 0.5)*2
ch_in = in_node.get_shape().as_list()[-1]
num_params = 63
with tf.variable_scope("input") as SCOPE:
conv1_w, conv1_b = CONV2D([5, 5, ch_in, 32])
c1 = tf.nn.conv2d(in_node,
conv1_w,
strides=[1, 1, 1, 1],
padding='SAME')
c1 = tf.nn.bias_add(c1, conv1_b)
with tf.variable_scope("residuals") as SCOPE:
current = cell2D_res(c1, 3, 32, 64, mode_node, 2, 'r1')
current = cell2D_res(current, 3, 64, 64, mode_node, 1, 'r2')
current = cell2D_res(current, 3, 64, 128, mode_node, 2, 'r3')
current = cell2D_res(current, 3, 128, 128, mode_node, 1, 'r4')
current = cell2D_res(current, 3, 128, 256, mode_node, 2, 'r5')
current = cell2D_res(current, 3, 256, 256, mode_node, 1, 'r6')
current = cell2D_res(current, 3, 256, 512, mode_node, 2, 'r7')
current = BatchNorm(current, mode_node, SCOPE)
current = tf.nn.avg_pool(current, [1, 6, 6, 1], [1, 1, 1, 1],
padding='SAME')
with tf.variable_scope("fully") as SCOPE:
features = tf.reshape(current, (batch_size, -1))
current = cell1D(features,
num_params,
mode_node,
SCOPE='logits',
with_act=False,
with_bn=False)
marks = tf.tanh(tf.reshape(current, (batch_size, 3, 21)))
pred_cloud = tf.transpose(marks, (0, 2, 1))
marks_norm = (marks / 2 + 0.5) * (grid_size_gt - 1)
with tf.variable_scope("xentropy") as SCOPE:
gt_meshgrid = np.meshgrid(
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt),
np.linspace(0, grid_size_gt - 1, grid_size_gt))
grid_x = (gt_meshgrid[1]).astype(np.float32)
grid_y = (gt_meshgrid[0]).astype(np.float32)
grid_z = (gt_meshgrid[2]).astype(np.float32)
grid_x_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='x',
initializer=grid_x,
trainable=False,
dtype='float32'), 0), -1)
grid_y_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='y',
initializer=grid_y,
trainable=False,
dtype='float32'), 0), -1)
grid_z_tf = tf.expand_dims(
tf.expand_dims(
tf.get_variable(name='z',
initializer=grid_z,
trainable=False,
dtype='float32'), 0), -1)
pred_x = grid_x_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 0, 0], [-1, 1, -1]), 1), 1)
pred_y = grid_y_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 1, 0], [-1, 1, -1]), 1), 1)
pred_z = grid_z_tf - tf.expand_dims(
tf.expand_dims(tf.slice(marks_norm, [0, 2, 0], [-1, 1, -1]), 1), 1)
logits = -1 * (tf.pow(pred_x, 2) + tf.pow(pred_y, 2) +
tf.pow(pred_z, 2))
init_scale = np.zeros((1, 1, 1, 1, 21), dtype='float32')
init_bias = np.zeros((1, 1, 1, 1, 21), dtype='float32')
# init_scale = np.zeros((1),dtype='float32')
# init_bias = np.zeros((1),dtype='float32')
scale = tf.get_variable(name='scale',
initializer=init_scale,
trainable=True,
dtype='float32')
bias = tf.get_variable(name='bias',
initializer=init_bias,
trainable=True,
dtype='float32')
logits = logits * scale + bias
return logits, pred_cloud