def hg_net(net, n, scope=None, reuse=None):
num = int(net.shape[-1].value)
sc_current = 'hg_net_{}'.format(n)
with tf.variable_scope(scope, sc_current, [net], reuse=reuse):
upper0 = incept_resnet.resnet_k(net)
lower0 = slim.max_pool2d(net, 3, stride=2)
lower0 = incept_resnet.resnet_k(lower0)
lower0 = slim.conv2d(lower0, num * 2, 1, stride=1)
if 1 < n:
lower1 = hourglass.hg_net(lower0, n - 1)
else:
lower1 = lower0
lower1 = slim.conv2d(lower1, num, 1, stride=1)
lower2 = incept_resnet.resnet_k(lower1)
upper1 = slim.conv2d_transpose(lower2,
int(net.shape[-1].value),
3,
stride=2)
return upper0 + upper1
def get_model(self,
input_tensor,
is_training,
bn_decay,
regu_scale,
hg_repeat=2,
scope=None):
""" input_tensor: BxHxWxC
out_dim: BxHxWx(J*5), where J is number of joints
"""
end_points = {}
self.end_point_list = []
final_endpoint = 'stage_out'
num_joint = self.join_num
# num_out_map = num_joint * 5 # hmap2, olmap, uomap
num_feature = 64
def add_and_check_final(name, net):
end_points[name] = net
return name == final_endpoint
from tensorflow.contrib import slim
from model.incept_resnet import incept_resnet
from model.hourglass import hourglass
# ~/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/layers/
with tf.variable_scope(scope, self.name_desc, [input_tensor]):
bn_epsilon = 0.001
with \
slim.arg_scope(
[slim.batch_norm],
is_training=is_training,
epsilon=bn_epsilon,
# # Make sure updates happen automatically
# updates_collections=None,
# Try zero_debias_moving_mean=True for improved stability.
# zero_debias_moving_mean=True,
decay=bn_decay), \
slim.arg_scope(
[slim.dropout],
is_training=is_training), \
slim.arg_scope(
[slim.fully_connected],
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm), \
slim.arg_scope(
[slim.max_pool2d, slim.avg_pool2d],
stride=2, padding='SAME'), \
slim.arg_scope(
[slim.conv2d_transpose],
stride=2, padding='SAME',
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm), \
slim.arg_scope(
[slim.conv2d],
stride=1, padding='SAME',
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm):
with tf.variable_scope('stage128'):
sc = 'stage128'
net = slim.conv2d(input_tensor, 8, 3)
net = incept_resnet.conv_maxpool(net, scope=sc)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage64'
net = incept_resnet.conv_maxpool(net, scope=sc)
# net = incept_resnet.resnet_k(
# net, scope='stage64_residual')
# net = incept_resnet.reduce_net(
# net, scope='stage64_reduce')
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage32_pre'
net = incept_resnet.resnet_k(net, scope='stage32_res')
net = slim.conv2d(net, num_feature, 1, scope='stage32_out')
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
for hg in range(hg_repeat):
sc = 'hourglass_{}'.format(hg)
with tf.variable_scope(sc):
branch0 = hourglass.hg_net(net, 2, scope=sc + '_hg')
branch0 = incept_resnet.resnet_k(branch0, scope='_res')
net_maps = slim.conv2d(
branch0,
num_joint * 3,
1,
# normalizer_fn=None, activation_fn=tf.nn.softmax)
normalizer_fn=None,
activation_fn=None)
self.end_point_list.append(sc)
if add_and_check_final(sc, net_maps):
return net_maps, end_points
branch1 = slim.conv2d(net_maps, num_feature, 1)
net = net + branch0 + branch1
with tf.variable_scope('stage32'):
sc = 'stage32_post'
# net = incept_resnet.conv_maxpool(net, scope=sc)
net = slim.max_pool2d(net, 3, scope=sc)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage16'):
sc = 'stage16'
net = incept_resnet.conv_maxpool(net, scope=sc)
# net = slim.max_pool2d(net, 3, scope=sc)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage8'):
sc = 'stage_out'
net = incept_resnet.pullout8(net,
self.out_dim,
is_training,
scope=sc)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
raise ValueError('final_endpoint (%s) not recognized', final_endpoint)
def get_model(self,
input_tensor,
is_training,
bn_decay,
regu_scale,
scope=None,
hg_repeat=2):
""" input_tensor: BxHxWxC
out_dim: Bx(Jx3), where J is number of joints
"""
end_points = {}
self.end_point_list = []
### Size of heatmap ###
#num_feature = 32
num_feature = 64
num_joint = self.join_num
final_endpoint = 'stage_out'
def add_and_check_final(name, net):
end_points[name] = net
return name == final_endpoint
from tensorflow.contrib import slim
from model.incept_resnet import incept_resnet
from model.style_content_mv import style_content_mv
from model.hourglass import hourglass
with tf.variable_scope(scope, self.name_desc, [input_tensor]):
bn_epsilon = 0.001
with \
slim.arg_scope( [slim.batch_norm],
is_training=is_training,
epsilon=bn_epsilon,
decay=bn_decay), \
slim.arg_scope( [slim.dropout],
is_training=is_training), \
slim.arg_scope( [slim.fully_connected],
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm), \
slim.arg_scope( [slim.max_pool2d, slim.avg_pool2d],
stride=2, padding='SAME'), \
slim.arg_scope( [slim.conv2d_transpose],
stride=2, padding='SAME',
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm), \
slim.arg_scope( [slim.conv2d],
stride=1, padding='SAME',
weights_regularizer=slim.l2_regularizer(regu_scale),
## For style_content testing
weights_initializer=tf.truncated_normal_initializer(stddev=0.02),
## For style_content testing
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm):
### FPN-styled conent Module ###
with tf.variable_scope('fpn_downscale_encoder'):
sc = 'content_code'
p1, p2, p3, content_code = style_content_mv.fpn_downscale(
input_tensor, scope=sc, reuse=False)
self.end_point_list.append(sc)
if add_and_check_final(sc, content_code):
return content_code, end_points
# (p1, p2, p3, p4) = (128, 64, 32, 16)
with tf.variable_scope('fpn_upscale_decoder'):
sc = 'styled_guidance_map'
# This should be smaller, like 8x8
# (?, 16, 16, 128)
d1, d2, d3, styled_guidance_map = style_content_mv.fpn_upscale(
p1, p2, p3, content_code, scope=sc, reuse=False)
self.end_point_list.append(sc)
if add_and_check_final(sc, styled_guidance_map):
return styled_guidance_map, end_points
# with tf.variable_scope('style_content_encoder'):
# sc = 'content_code'
# # (?, 128, 128, 3)
# style_aware_content_code = style_content_mv.encoder(styled_guidance_map, scope=sc, reuse=True)
# if add_and_check_final('style_aware_content_code', style_aware_content_code):
# return style_aware_content_code, end_points
with tf.variable_scope('multiscale_heat_map'):
sc = 'hmap128'
br0 = slim.conv2d(d3, 21 * 3, 3, stride=1)
br0 = slim.max_pool2d(br0, 3, stride=2)
# (?, 64, 64, 63)
sc = 'hmap64'
br1 = slim.conv2d(d2, 21 * 3, 3, stride=1)
# (?, 64, 64, 63)
net = br0 + br1
sc = 'hmap32'
net = slim.max_pool2d(net, 3, stride=2)
# ( net = 32, 32 63)
br2 = slim.conv2d(d1, 21 * 3, 3, stride=1)
# (?, 32, 32, 63)
sc = 'mv_hmap32'
net = net + br2
net = slim.conv2d(net, 21 * 3, 3, stride=1)
# (?, 32, 32, 63)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
### Residual Module ###
with tf.variable_scope('stage128'):
sc = 'stage128_image'
# (?, 128, 128, 3)
net = slim.conv2d(input_tensor, 8, 3)
# (?, 128, 128, 8)
net = incept_resnet.conv_maxpool(net, scope=sc)
# (? , 64, 64, 16)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage64_image'
net = incept_resnet.conv_maxpool(net, scope=sc)
# (?, 32, 32, 32)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage32_pre'
net = incept_resnet.resnet_k(net, scope='stage32_res')
net = slim.conv2d(net, num_feature, 1, scope='stage32_out')
#(?, 32, 32, 64)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
for hg in range(hg_repeat):
sc = 'hourglass_{}'.format(hg)
with tf.variable_scope(sc):
## TODO: check if hourglass useful
# Add hg_net, check this out later
branch0 = hourglass.hg_net(net, 2, scope=sc + '_hg')
branch0 = incept_resnet.resnet_k(branch0, scope='_res')
# Multiply bt 3 here
# Styled Map becomes 63
net_maps = slim.conv2d(branch0,
num_joint * 3,
1,
normalizer_fn=None,
activation_fn=None)
# (? 32, 32, 63)
self.end_point_list.append(sc)
if add_and_check_final(sc, net_maps):
return net_maps, end_points
branch1 = slim.conv2d(net_maps, num_feature, 1)
net = net + branch0 + branch1
with tf.variable_scope('stage32'):
sc = 'stage32_post'
## Why max pool only?
#net = incept_resnet.conv_maxpool(net, scope=sc)
net = slim.max_pool2d(net, 3, scope=sc)
#(?, 16, 16, 64)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage16'):
sc = 'stage16_image'
# (?, 8, 8, 128)
net = incept_resnet.conv_maxpool(net, scope=sc)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage8'):
sc = 'stage_out'
# (?, 63)
net = incept_resnet.pullout8(net,
self.out_dim,
is_training,
scope=sc)
self.end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
raise ValueError('final_endpoint (%s) not recognized', final_endpoint)
return net, end_points
def get_net(input_tensor,
out_dim,
is_training,
bn_decay,
regu_scale,
end_point_list,
hg_repeat=4,
scope=None,
final_endpoint='stage_out'):
""" input_tensor: BxHxWxC
"""
end_points = {}
def add_and_check_final(name, net):
end_points[name] = net
return name == final_endpoint
with tf.variable_scope(scope, 'hourglass_net', [input_tensor]):
bn_epsilon = 0.001
with \
slim.arg_scope(
[slim.batch_norm],
is_training=is_training,
epsilon=bn_epsilon,
# # Make sure updates happen automatically
# updates_collections=None,
# Try zero_debias_moving_mean=True for improved stability.
# zero_debias_moving_mean=True,
decay=bn_decay), \
slim.arg_scope(
[slim.dropout],
is_training=is_training), \
slim.arg_scope(
[slim.fully_connected],
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm), \
slim.arg_scope(
[slim.max_pool2d, slim.avg_pool2d],
stride=2, padding='SAME'), \
slim.arg_scope(
[slim.conv2d_transpose],
stride=2, padding='SAME',
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm), \
slim.arg_scope(
[slim.conv2d],
stride=1, padding='SAME',
weights_regularizer=slim.l2_regularizer(regu_scale),
biases_regularizer=slim.l2_regularizer(regu_scale),
activation_fn=tf.nn.relu,
normalizer_fn=slim.batch_norm):
with tf.variable_scope('stage128'):
sc = 'stage128'
net = slim.conv2d(input_tensor, 8, 3)
net = incept_resnet.conv_maxpool(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage64'
net = incept_resnet.reduce_net(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
for hg in range(hg_repeat):
sc = 'hourglass_{}'.format(hg)
with tf.variable_scope(sc):
net = hourglass.hg_net(net, 4, scope=sc + '_hg')
net = incept_resnet.resnet_k(net, scope=sc + '_res')
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage32'):
sc = 'stage32_1'
net = incept_resnet.block32(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage32_2'
net = incept_resnet.reduce_net(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage16'):
sc = 'stage16_1'
net = incept_resnet.block16(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage16_2'
net = incept_resnet.reduce_net(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
with tf.variable_scope('stage8'):
sc = 'stage8'
net = incept_resnet.block8(net, scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
sc = 'stage_out'
net = incept_resnet.pullout8(net,
out_dim,
is_training,
scope=sc)
end_point_list.append(sc)
if add_and_check_final(sc, net):
return net, end_points
raise ValueError('final_endpoint (%s) not recognized', final_endpoint)