def resnet_ssd(image, L2_reg, is_training=True):
resnet = resnet_v1_101 if '101' in cfg.MODEL.net_structure else resnet_v1_50
with slim.arg_scope(
resnet_arg_scope(weight_decay=L2_reg, bn_is_training=is_training)):
if cfg.TRAIN.lock_basenet_bn:
net, end_points = resnet(image,
is_training=False,
global_pool=False,
num_classes=None)
else:
net, end_points = resnet(image,
is_training=is_training,
global_pool=False,
num_classes=None)
for k, v in end_points.items():
print(k, v)
with slim.arg_scope(
resnet_arg_scope(weight_decay=L2_reg,
bn_is_training=is_training,
bn_method='BN')):
resnet_fms = [
end_points['%s/block1' % cfg.MODEL.net_structure],
end_points['%s/block2' % cfg.MODEL.net_structure],
end_points['%s/block3' % cfg.MODEL.net_structure],
end_points['%s/block4' % cfg.MODEL.net_structure]
]
net = slim.conv2d(resnet_fms[-1],
512, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_1_1')
net = slim.conv2d(net,
512, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_1_2')
resnet_fms.append(net)
net = slim.conv2d(net,
128, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_2_1')
net = slim.conv2d(net,
256, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_2_2')
resnet_fms.append(net)
print('extra resnet50 backbone output:', resnet_fms)
if cfg.MODEL.fpn:
enhanced_fms = create_fem_net(resnet_fms, L2_reg, is_training)
else:
enhanced_fms = None
return resnet_fms, enhanced_fms
def ssd_out(fms,
L2_reg,
training=True,
ratios_per_pixel=num_predict_per_level):
cla_set = []
reg_set = []
arg_scope = resnet_arg_scope(
weight_decay=L2_reg,
bn_is_training=training,
)
with slim.arg_scope(arg_scope):
with tf.variable_scope('ssdout'):
for i in range(len(fms)):
current_feature = fms[i]
dim_h = tf.shape(current_feature)[1]
dim_w = tf.shape(current_feature)[2]
#feature_halo=halo(current_feature,'fm%d'%i)
feature_halo = current_feature
reg_out = slim.conv2d(feature_halo,
ratios_per_pixel * 4, [3, 3],
stride=1,
activation_fn=None,
normalizer_fn=None,
scope='out_reg%d' % i)
if cfg.MODEL.maxout and i == 0:
cla_out = max_out_cla(feature_halo,
ratios_per_pixel,
cla_num=cfg.DATA.NUM_CLASS,
scope='maxout%d' % i)
else:
cla_out = slim.conv2d(feature_halo,
ratios_per_pixel *
cfg.DATA.NUM_CLASS, [3, 3],
stride=1,
activation_fn=None,
normalizer_fn=None,
scope='out_cla%d' % i)
reg_out = tf.reshape(reg_out,
([-1, dim_h, dim_w, ratios_per_pixel, 4]))
reg_out = tf.reshape(
reg_out, ([-1, dim_h * dim_w * ratios_per_pixel, 4]))
cla_out = tf.reshape(
cla_out,
([-1, dim_h, dim_w, ratios_per_pixel, cfg.DATA.NUM_CLASS]))
cla_out = tf.reshape(cla_out, ([
-1, dim_h * dim_w * ratios_per_pixel, cfg.DATA.NUM_CLASS
]))
cla_set.append(cla_out)
reg_set.append(reg_out)
reg = tf.concat(reg_set, axis=1)
cla = tf.concat(cla_set, axis=1)
return reg, cla
def __call__(self, fms, training=True):
arg_scope = resnet_arg_scope(bn_is_training=training, )
with slim.arg_scope(arg_scope):
with tf.variable_scope('CenternetHead'):
# c2, c3, c4, c5 = fms
# deconv_feature=c5
deconv_feature = self._unet_magic(fms)
#####
kps = slim.separable_conv2d(
deconv_feature,
cfg.DATA.num_class, [3, 3],
stride=1,
activation_fn=None,
normalizer_fn=None,
weights_initializer=tf.initializers.random_normal(
stddev=0.001),
biases_initializer=tf.initializers.constant(-2.19),
scope='centernet_cls_output')
wh = slim.separable_conv2d(
deconv_feature,
4, [3, 3],
stride=1,
activation_fn=None,
normalizer_fn=None,
weights_initializer=tf.initializers.random_normal(
stddev=0.001),
biases_initializer=tf.initializers.constant(0),
scope='centernet_wh_output')
return kps, wh * 16
def __call__(self,fms,L2_reg,training=True,ratios_per_pixel=num_predict_per_level):
cla_set=[]
reg_set=[]
arg_scope = resnet_arg_scope(weight_decay=L2_reg, bn_is_training=training, )
with slim.arg_scope(arg_scope):
with tf.variable_scope('ssdout'):
for i in range(len(fms)):
current_feature = fms[i]
dim_h=tf.shape(current_feature)[1]
dim_w = tf.shape(current_feature)[2]
#feature_halo=halo(current_feature,'fm%d'%i)
feature_halo=current_feature
reg_out = slim.conv2d(feature_halo, ratios_per_pixel*4, [1, 1], stride=1, activation_fn=None,normalizer_fn=None, scope='out_reg%d'%i)
cla_out = slim.conv2d(feature_halo, ratios_per_pixel, [1, 1], stride=1, activation_fn=None,normalizer_fn=None, scope='out_cla%d'%i)
reg_out = tf.reshape(reg_out, ([-1, dim_h, dim_w, ratios_per_pixel, 4]))
reg_out = tf.reshape(reg_out, ([-1, dim_h * dim_w * ratios_per_pixel, 4]))
cla_out = tf.reshape(cla_out, ([-1, dim_h, dim_w, ratios_per_pixel, 1]))
cla_out = tf.reshape(cla_out, ([-1, dim_h * dim_w* ratios_per_pixel,1]))
cla_set.append(cla_out)
reg_set.append(reg_out)
reg = tf.concat(reg_set, axis=1)
cla = tf.concat(cla_set, axis=1)
return reg,cla
def create_fem_net(blocks, L2_reg,is_training, trainable=True,data_format='NHWC'):
of0,of1,of2,of3,of4,of5=blocks
initializer = tf.contrib.layers.xavier_initializer()
with slim.arg_scope(resnet_arg_scope(weight_decay=L2_reg,bn_is_training=is_training)):
lateral = slim.conv2d(of2, dims_list[2], [1, 1],
trainable=trainable, weights_initializer=initializer,
padding='SAME',normalizer_fn=None,activation_fn=None,
scope='lateral/res{}'.format(2))
upsample = slim.conv2d(of3, dims_list[2], [1, 1],
trainable=trainable, weights_initializer=initializer,
padding='SAME',normalizer_fn=None,activation_fn=None,
scope='merge/res{}'.format(2), data_format=data_format)
upsample = tf.keras.layers.UpSampling2D(data_format='channels_last' if data_format=='NHWC' else 'channels_first')(upsample)
fem_2 = lateral* upsample
lateral = slim.conv2d(of1, dims_list[1], [1, 1],
trainable=trainable, weights_initializer=initializer,
padding='SAME',normalizer_fn=None,activation_fn=None,
scope='lateral/res{}'.format(1))
upsample = slim.conv2d(fem_2, dims_list[1], [1, 1],
trainable=trainable, weights_initializer=initializer,
padding='SAME',normalizer_fn=None,activation_fn=None,
scope='merge/res{}'.format(1), data_format=data_format)
upsample = tf.keras.layers.UpSampling2D(data_format='channels_last' if data_format == 'NHWC' else 'channels_first')(upsample)
fem_1 = lateral * upsample
lateral = slim.conv2d(of0, dims_list[0], [1, 1],
trainable=trainable, weights_initializer=initializer,
padding='SAME',normalizer_fn=None,activation_fn=None,
scope='lateral/res{}'.format(0))
upsample = slim.conv2d(fem_1, dims_list[0], [1, 1],
trainable=trainable, weights_initializer=initializer,
padding='SAME',normalizer_fn=None,activation_fn=None,
scope='merge/res{}'.format(0), data_format=data_format)
upsample = tf.keras.layers.UpSampling2D(data_format='channels_last' if data_format == 'NHWC' else 'channels_first')(upsample)
fem_0 = lateral * upsample
#####enhance model
fpn_fms=[fem_0,fem_1,fem_2,of3,of4,of5]
global_fems_fms=[]
for i, fem in enumerate(fpn_fms):
tmp_res=cpm(fem,scope='fems%d'%i)
global_fems_fms.append(tmp_res)
return global_fems_fms
def mobilenet_ssd(image, L2_reg, is_training=True):
assert 'MobilenetV2' in cfg.MODEL.net_structure
if cfg.TRAIN.lock_basenet_bn:
arg_scope = training_scope(weight_decay=L2_reg, is_training=False)
else:
arg_scope = training_scope(weight_decay=L2_reg,
is_training=is_training)
with tf.contrib.slim.arg_scope(arg_scope):
# _,endpoint = mobilenet_v2_050(image,is_training=is_training,base_only=True,finegrain_classification_mode=False)
_, endpoint = mobilenet_v2_140(image,
is_training=is_training,
base_only=True,
finegrain_classification_mode=False)
for k, v in endpoint.items():
print('mobile backbone output:', k, v)
mobilebet_fms = [
endpoint['layer_5/expansion_output'],
endpoint['layer_8/expansion_output'],
endpoint['layer_15/expansion_output'], endpoint['layer_18/output']
]
print('mobile backbone output:', mobilebet_fms)
with slim.arg_scope(
resnet_arg_scope(weight_decay=L2_reg, bn_is_training=is_training)):
net = slim.conv2d(mobilebet_fms[-1],
128, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_1_1')
net = slim.conv2d(net,
256, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_1_2')
mobilebet_fms.append(net)
net = slim.conv2d(net,
128, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_2_1')
net = slim.conv2d(net,
256, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_2_2')
mobilebet_fms.append(net)
print('extra backbone output:', mobilebet_fms)
if cfg.MODEL.fpn:
enhanced_fms = create_fem_net(mobilebet_fms, L2_reg, is_training)
else:
enhanced_fms = None
return mobilebet_fms, enhanced_fms
def __call__(self, fms, L2_reg, training=True):
arg_scope = resnet_arg_scope(weight_decay=L2_reg, bn_is_training=training, )
with slim.arg_scope(arg_scope):
with tf.variable_scope('CenternetHead'):
# c2, c3, c4, c5 = fms
# deconv_feature=c5
# for i in range(3):
# deconv_feature=self._upsample(deconv_feature,scope='upsample_%d'%i)
deconv_feature = self._unet_magic(fms)
#####
pre_fm = self._pre_head(deconv_feature, 'centernet_pre_feature')
kps = slim.conv2d(pre_fm,
cfg.DATA.num_class,
[1, 1],
stride=1,
activation_fn=None,
normalizer_fn=None,
weights_initializer=tf.initializers.random_normal(stddev=0.001),
biases_initializer=tf.initializers.constant(-2.19),
scope='centernet_cls_output')
wh = slim.conv2d(pre_fm,
2,
[1, 1],
stride=1,
activation_fn=None,
normalizer_fn=None,
weights_initializer=tf.initializers.random_normal(stddev=0.001),
biases_initializer=tf.initializers.constant(0.),
scope='centernet_wh_output')
if cfg.MODEL.offset:
reg = slim.conv2d(pre_fm,
2,
[1, 1],
stride=1,
activation_fn=None,
normalizer_fn=None,
weights_initializer=tf.initializers.random_normal(stddev=0.001),
biases_initializer=tf.initializers.constant(0.),
scope='centernet_reg_output')
else:
reg = None
return kps, wh, reg
def efficient_ssd(image, L2_reg, is_training=True, data_format='NHWC'):
net, endpoints = build_model_base(image,
model_name=cfg.MODEL.net_structure,
training=is_training)
for k, v in endpoints.items():
print('mobile backbone output:', k, v)
with slim.arg_scope(
resnet_arg_scope(weight_decay=L2_reg, bn_is_training=is_training)):
efficient_fms = [
endpoints['reduction_2/expansion_output'],
endpoints['reduction_3/expansion_output'],
endpoints['reduction_4/expansion_output'], endpoints['global_pool']
]
net = slim.conv2d(efficient_fms[-1],
512, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_1_1')
net = slim.conv2d(net,
512, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_1_2')
efficient_fms.append(net)
net = slim.conv2d(net,
128, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_2_1')
net = slim.conv2d(net,
256, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_2_2')
efficient_fms.append(net)
print('extra resnet50 backbone output:', efficient_fms)
if cfg.MODEL.fpn:
enhanced_fms = create_fem_net(efficient_fms, L2_reg, is_training)
else:
enhanced_fms = None
return efficient_fms, enhanced_fms
def mobilenet_ssd(image,L2_reg,is_training=True,data_format='NHWC'):
assert 'MobilenetV1' in cfg.MODEL.net_structure
if cfg.TRAIN.lock_basenet_bn:
arg_scope = mobilenet_v1_arg_scope(weight_decay=L2_reg, is_training=False)
else:
arg_scope = mobilenet_v1_arg_scope(weight_decay=L2_reg, is_training=is_training)
with tf.contrib.slim.arg_scope(arg_scope):
_,endpoint = mobilenet_v1_050(image,is_training=is_training,num_classes=None,global_pool=False)
for k,v in endpoint.items():
print('mobile backbone output:',k,v)
mobilebet_fms=[endpoint['Conv2d_3_pointwise'],endpoint['Conv2d_5_pointwise'],endpoint['Conv2d_11_pointwise'],endpoint['Conv2d_13_pointwise']]
print('mobile backbone output:',mobilebet_fms)
with slim.arg_scope(resnet_arg_scope(weight_decay=L2_reg, bn_is_training=is_training)):
# net = block(resnet_fms[-1], num_units=2, out_channels=512, scope='extra_Stage1')
# resnet_fms.append(net)
# net = block(net, num_units=2, out_channels=512, scope='extra_Stage2')
# resnet_fms.append(net)
net = slim.conv2d(mobilebet_fms[-1], 512, [1, 1], stride=1, activation_fn=tf.nn.relu, scope='extra_conv_1_1')
net = slim.conv2d(net, 512, [3, 3], stride=2, activation_fn=tf.nn.relu, scope='extra_conv_1_2')
mobilebet_fms.append(net)
net = slim.conv2d(net, 128, [1, 1], stride=1, activation_fn=tf.nn.relu, scope='extra_conv_2_1')
net = slim.conv2d(net, 256, [3, 3], stride=2, activation_fn=tf.nn.relu, scope='extra_conv_2_2')
mobilebet_fms.append(net)
print('extra backbone output:', mobilebet_fms)
if cfg.MODEL.fpn:
enhanced_fms = create_fem_net(mobilebet_fms, L2_reg, is_training)
else:
enhanced_fms =None
return mobilebet_fms,enhanced_fms
def effnet_ssd(image, L2_reg, is_training=True):
if cfg.TRAIN.lock_basenet_bn:
features, endpoints = build_model_base(image, cfg.MODEL.net_structure,
False)
else:
features, endpoints = build_model_base(image, cfg.MODEL.net_structure,
True) #, 'efficientnet-b0')
if 'efficientnet-b0' in cfg.MODEL.net_structure:
resnet_fms = [
endpoints['block_2/expansion_output'],
endpoints['block_4/expansion_output'],
endpoints['block_10/expansion_output'],
endpoints['block_14/expansion_output']
]
# resnet_fms = [endpoints['reduction_0'],
# endpoints['reduction_1'],
# endpoints['reduction_2'],
# endpoints['reduction_3']]
elif 'efficientnet-b3' in cfg.MODEL.net_structure:
resnet_fms = [
endpoints['block_4/expansion_output'],
endpoints['block_7/expansion_output'],
endpoints['block_17/expansion_output'],
endpoints['block_25/expansion_output']
]
elif 'efficientnet-b5' in cfg.MODEL.net_structure:
resnet_fms = [
endpoints['block_7/expansion_output'],
endpoints['block_12/expansion_output'],
endpoints['block_26/expansion_output'],
endpoints['block_38/expansion_output']
]
# for i in endpoints:
# print(endpoints[i])
# exit()
with slim.arg_scope(
resnet_arg_scope(weight_decay=L2_reg,
bn_is_training=is_training,
bn_method='BN')):
net = slim.conv2d(resnet_fms[-1],
512, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_1_1')
net = slim.conv2d(net,
512, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_1_2')
resnet_fms.append(net)
net = slim.conv2d(net,
128, [1, 1],
stride=1,
activation_fn=tf.nn.relu,
scope='extra_conv_2_1')
net = slim.conv2d(net,
256, [3, 3],
stride=2,
activation_fn=tf.nn.relu,
scope='extra_conv_2_2')
resnet_fms.append(net)
print('extra resnet50 backbone output:', resnet_fms)
if cfg.MODEL.fpn:
enhanced_fms = create_fem_net(resnet_fms, L2_reg, is_training)
else:
enhanced_fms = None
return resnet_fms, enhanced_fms
