def _stacked_separable_conv(net, stride, operation, filter_size):
"""Takes in an operations and parses it to the correct sep operation."""
num_layers, kernel_size = _operation_to_info(operation)
net_type = net.dtype
net = tf.cast(net, tf.float32) if net_type == tf.float16 else net
for layer_num in range(num_layers - 1):
net = tf.nn.relu(net)
net = slim.separable_conv2d(net,
filter_size,
kernel_size,
depth_multiplier=1,
scope='separable_{0}x{0}_{1}'.format(
kernel_size, layer_num + 1),
stride=stride)
net = slim.batch_norm(net,
scope='bn_sep_{0}x{0}_{1}'.format(
kernel_size, layer_num + 1))
stride = 1
net = tf.nn.relu(net)
net = slim.separable_conv2d(net,
filter_size,
kernel_size,
depth_multiplier=1,
scope='separable_{0}x{0}_{1}'.format(
kernel_size, num_layers),
stride=stride)
net = slim.batch_norm(net,
scope='bn_sep_{0}x{0}_{1}'.format(
kernel_size, num_layers))
net = tf.cast(net, net_type)
return net
def _stacked_separable_conv(net, stride, operation, filter_size,
use_bounded_activation):
"""Takes in an operations and parses it to the correct sep operation."""
num_layers, kernel_size = _operation_to_info(operation)
activation_fn = tf.nn.relu6 if use_bounded_activation else tf.nn.relu
for layer_num in range(num_layers - 1):
net = activation_fn(net)
net = slim.separable_conv2d(net,
filter_size,
kernel_size,
depth_multiplier=1,
scope='separable_{0}x{0}_{1}'.format(
kernel_size, layer_num + 1),
stride=stride)
net = slim.batch_norm(net,
scope='bn_sep_{0}x{0}_{1}'.format(
kernel_size, layer_num + 1))
stride = 1
net = activation_fn(net)
net = slim.separable_conv2d(net,
filter_size,
kernel_size,
depth_multiplier=1,
scope='separable_{0}x{0}_{1}'.format(
kernel_size, num_layers),
stride=stride)
net = slim.batch_norm(net,
scope='bn_sep_{0}x{0}_{1}'.format(
kernel_size, num_layers))
return net
def extract_features_resnet50(self,
im,
name,
is_training=True,
reuse=False):
use_global_pool = True
num_classes = 4096 if use_global_pool else 512
with tf.name_scope(name):
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
out, _ = resnet_v2.resnet_v2_50(inputs=im,
num_classes=num_classes,
global_pool=use_global_pool,
is_training=self.is_training,
spatial_squeeze=True,
scope='resnet_v2_50',
reuse=reuse)
if not use_global_pool:
args = {
'reuse': reuse,
'norm': None,
'activation': tf.nn.relu,
'padding': 'SAME',
'is_training': is_training
}
out_args = copy.deepcopy(args)
out_args['activation'] = None
out = ops.conv(out, 1024, 3, 2, name='conv1', **args)
out = slim.batch_norm(out)
out = ops.conv(out, 2048, 3, 2, name='conv2', **args)
out = slim.batch_norm(out)
out = ops.conv(out, 4096, 3, 2, name='conv3', **out_args)
out = slim.batch_norm(out)
out = tf.squeeze(out, [1, 2], name='SpatialSqueeze')
return out
def build(self, input_tensors, is_training, lengths=None):
"""Residual unit with 2 sub layers."""
input_tensor = input_tensors[-1]
net = tf_slim.conv2d(
input_tensor,
get_channel_dim(input_tensor),
kernel_size=self._kernel_size,
padding='same')
# Batch norm
if self._apply_batch_norm:
net = tf_slim.batch_norm(net, is_training=is_training)
net = tf.nn.leaky_relu(net)
net = tf_slim.conv2d(
input_tensor,
get_channel_dim(input_tensor),
kernel_size=self._kernel_size,
padding='same')
net += input_tensor
# Batch norm
if self._apply_batch_norm:
net = tf_slim.batch_norm(net, is_training=is_training)
net = tf.nn.leaky_relu(net)
return input_tensors + [net]
def _apply_conv_operation(self, net, operation, stride,
is_from_original_input):
"""Applies the predicted conv operation to net."""
# Dont stride if this is not one of the original hiddenstates
if stride > 1 and not is_from_original_input:
stride = 1
input_filters = get_channel_dim(net.shape)
filter_size = self._filter_size
if 'separable' in operation:
net = _stacked_separable_conv(net, stride, operation, filter_size)
elif operation in ['none']:
# Check if a stride is needed, then use a strided 1x1 here
if stride > 1 or (input_filters != filter_size):
net = tf.nn.relu(net)
net = slim.conv2d(net,
filter_size,
1,
stride=stride,
scope='1x1')
net = slim.batch_norm(net, scope='bn_1')
elif 'pool' in operation:
net = _pooling(net, stride, operation)
if input_filters != filter_size:
net = slim.conv2d(net, filter_size, 1, stride=1, scope='1x1')
net = slim.batch_norm(net, scope='bn_1')
else:
raise ValueError('Unimplemented operation', operation)
if operation != 'none':
net = self._apply_drop_path(net)
return net
def FullResolutionResidualUnit(pool_stream, res_stream, n_filters_3, n_filters_1, pool_scale):
"""
A full resolution residual unit
Arguments:
pool_stream: The inputs from the pooling stream
res_stream: The inputs from the residual stream
n_filters_3: Number of output feature maps for each 3x3 conv
n_filters_1: Number of output feature maps for each 1x1 conv
pool_scale: scale of the pooling layer i.e window size and stride
Returns:
Output of full resolution residual block
"""
G = tf.concat([pool_stream, slim.pool(res_stream, [pool_scale, pool_scale], stride=[pool_scale, pool_scale], pooling_type='MAX')], axis=-1)
net = slim.conv2d(G, n_filters_3, kernel_size=3, activation_fn=None)
net = slim.batch_norm(net, fused=True)
net = tf.nn.relu(net)
net = slim.conv2d(net, n_filters_3, kernel_size=3, activation_fn=None)
net = slim.batch_norm(net, fused=True)
pool_stream_out = tf.nn.relu(net)
net = slim.conv2d(pool_stream_out, n_filters_1, kernel_size=1, activation_fn=None)
net = Upsampling(net, scale=pool_scale)
res_stream_out = tf.add(res_stream, net)
return pool_stream_out, res_stream_out
def MultiscaleBlock_1(inputs, filters_1, filters_2, filters_3, p, d):
net = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net = slim.conv2d(net,
filters_1, [1, 1],
activation_fn=None,
normalizer_fn=None)
scale_1 = tf.nn.relu(slim.batch_norm(net, fused=True))
scale_1 = slim.conv2d(scale_1,
filters_3 // 2, [3, 3],
rate=p,
activation_fn=None,
normalizer_fn=None)
scale_2 = tf.nn.relu(slim.batch_norm(net, fused=True))
scale_2 = slim.conv2d(scale_2,
filters_3 // 2, [3, 3],
rate=d,
activation_fn=None,
normalizer_fn=None)
net = tf.concat((scale_1, scale_2), axis=-1)
net = tf.nn.relu(slim.batch_norm(net, fused=True))
net = slim.conv2d(net,
filters_2, [1, 1],
activation_fn=None,
normalizer_fn=None)
net = tf.add(inputs, net)
return net
def ResNetBlock_2(inputs, filters_1, filters_2, s=1):
net_1 = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net_1 = slim.conv2d(net_1,
filters_1, [1, 1],
stride=s,
activation_fn=None,
normalizer_fn=None)
net_1 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
net_1 = slim.conv2d(net_1,
filters_1, [3, 3],
activation_fn=None,
normalizer_fn=None)
net_1 = tf.nn.relu(slim.batch_norm(net_1, fused=True))
net_1 = slim.conv2d(net_1,
filters_2, [1, 1],
activation_fn=None,
normalizer_fn=None)
net_2 = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net_2 = slim.conv2d(net_2,
filters_2, [1, 1],
stride=s,
activation_fn=None,
normalizer_fn=None)
net = tf.add(net_1, net_2)
return net
def _build_nas_aux_head(inputs, dimension, data_format):
"""Builds the auxiliary head described in the NAS paper."""
shape = inputs.shape
if shape.rank < 4:
return None
shape = shape.as_list()
shape = shape[1:3] if data_format == "NHWC" else shape[2:4]
if np.any(np.array(shape) < np.array([5, 5])):
return None
with tf.compat.v1.variable_scope("aux_logits"):
with arg_scope(DATA_FORMAT_OPS, data_format=data_format):
aux_logits = tf_slim.avg_pool2d(inputs, [5, 5],
stride=3,
padding="SAME")
aux_logits = tf_slim.conv2d(aux_logits, 128, [1, 1], scope="proj")
aux_logits = tf_slim.batch_norm(aux_logits, scope="aux_bn0")
aux_logits = tf.nn.relu6(aux_logits)
# Shape of feature map before the final layer.
shape = aux_logits.shape
shape = shape[1:3] if data_format == "NHWC" else shape[2:4]
aux_logits = tf_slim.conv2d(aux_logits,
768,
shape,
padding="VALID")
aux_logits = tf_slim.batch_norm(aux_logits, scope="aux_bn1")
aux_logits = tf.nn.relu6(aux_logits)
aux_logits = tf.keras.layers.Flatten()(aux_logits)
aux_logits = tf_slim.fully_connected(aux_logits, dimension)
return aux_logits
def CFFBlock(F1, F2, num_classes):
F1_big = Upsampling_by_scale(F1, scale=2)
F1_out = slim.conv2d(F1_big, num_classes, [1, 1], activation_fn=None)
F1_big = slim.conv2d(F1_big, 2048, [3, 3], rate=2, activation_fn=None)
F1_big = slim.batch_norm(F1_big, fused=True)
F2_proj = slim.conv2d(F2, 512, [1, 1], rate=1, activation_fn=None)
F2_proj = slim.batch_norm(F2_proj, fused=True)
F2_out = tf.add([F1_big, F2_proj])
F2_out = tf.nn.relu(F2_out)
return F1_out, F2_out
def middle_flow_block(inpt, num_outputs=728, kernel_size=None, unit_num=None):
if kernel_size is None:
kernel_size = [3, 3]
unit_num = str(unit_num)
residual = inpt
net = tf.nn.relu(inpt)
net = slim.separable_conv2d(
net,
num_outputs,
kernel_size,
scope=
'xception_65/middle_flow/block1/unit_{}/xception_module/separable_conv1_depthwise'
.format(unit_num))
net = slim.batch_norm(
net,
scope=
'xception_65/middle_flow/block1/unit_{}/xception_module/separable_conv1_pointwise/BatchNorm'
.format(unit_num))
net = tf.nn.relu(net)
net = slim.separable_conv2d(
net,
num_outputs,
kernel_size,
scope=
'xception_65/middle_flow/block1/unit_{}/xception_module/separable_conv2_depthwise'
.format(unit_num))
net = slim.batch_norm(
net,
scope=
'xception_65/middle_flow/block1/unit_{}/xception_module/separable_conv2_pointwise/BatchNorm'
.format(unit_num))
net = tf.nn.relu(net)
net = slim.separable_conv2d(
net,
num_outputs,
kernel_size,
scope=
'xception_65/middle_flow/block1/unit_{}/xception_module/separable_conv3_depthwise'
.format(unit_num))
net = slim.batch_norm(
net,
scope=
'xception_65/middle_flow/block1/unit_{}/xception_module/separable_conv3_pointwise/BatchNorm'
.format(unit_num))
residual_next = tf.math.add(net, residual)
return residual_next
def DepthwiseSeparableConvBlock(inputs, n_filters, kernel_size=[3, 3]): """ Builds the Depthwise Separable conv block for MobileNets Apply successivly a 2D separable convolution, BatchNormalization relu, conv, BatchNormalization, relu """ # Skip pointwise by setting num_outputs=None net = slim.separable_convolution2d(inputs, num_outputs=None, depth_multiplier=1, kernel_size=[3, 3], activation_fn=None) net = slim.batch_norm(net, fused=True) net = tf.nn.relu(net) net = slim.conv2d(net, n_filters, kernel_size=[1, 1], activation_fn=None) net = slim.batch_norm(net, fused=True) net = tf.nn.relu(net) return net
def _imagenet_stem(inputs, hparams, stem_cell, current_step=None):
"""Stem used for models trained on ImageNet."""
num_stem_cells = 2
# 149 x 149 x 32
num_stem_filters = int(32 * hparams.stem_multiplier)
net = slim.conv2d(inputs,
num_stem_filters, [3, 3],
stride=2,
scope='conv0',
padding='VALID')
net = slim.batch_norm(net, scope='conv0_bn')
# Run the reduction cells
cell_outputs = [None, net]
filter_scaling = 1.0 / (hparams.filter_scaling_rate**num_stem_cells)
for cell_num in range(num_stem_cells):
net = stem_cell(net,
scope='cell_stem_{}'.format(cell_num),
filter_scaling=filter_scaling,
stride=2,
prev_layer=cell_outputs[-2],
cell_num=cell_num,
current_step=current_step)
cell_outputs.append(net)
filter_scaling *= hparams.filter_scaling_rate
return net, cell_outputs
def DilatedConvBlock(inputs, n_filters, rate=1, kernel_size=[3, 3]):
"""
Basic dilated conv block
Apply successivly BatchNormalization, ReLU nonlinearity, dilated convolution
"""
net = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net = slim.conv2d(net, n_filters, kernel_size, rate=rate, activation_fn=None, normalizer_fn=None)
return net
def batchnorm(inputs, is_training):
return slim.batch_norm(inputs,
decay=0.9,
epsilon=0.001,
updates_collections=tf.GraphKeys.UPDATE_OPS,
scale=False,
fused=True,
is_training=is_training)
def ConvUpscaleBlock(inputs, n_filters, kernel_size=[3, 3], scale=2):
"""
Basic conv transpose block for Encoder-Decoder upsampling
Apply successivly Transposed Convolution, BatchNormalization, ReLU nonlinearity
"""
net = slim.conv2d_transpose(inputs, n_filters, kernel_size=[3, 3], stride=[scale, scale], activation_fn=None)
net = tf.nn.relu(slim.batch_norm(net, fused=True))
return net
def __call__(self, x, train=True):
return slim.batch_norm(x,
decay=self.momentum,
updates_collections=None,
epsilon=self.epsilon,
scale=True,
is_training=train,
scope=self.name)
def ConvBlock(inputs, n_filters, kernel_size=[3, 3]):
"""
Basic conv block for Encoder-Decoder
Apply successivly Convolution, BatchNormalization, ReLU nonlinearity
"""
net = slim.conv2d(inputs, n_filters, kernel_size, activation_fn=None, normalizer_fn=None)
net = tf.nn.relu(slim.batch_norm(net, fused=True))
return net
def factorized_reduction(net, output_filters, stride, data_format=INVALID):
"""Reduces the shape of net without information loss due to striding."""
assert data_format != INVALID
if stride == 1:
net = slim.conv2d(net, output_filters, 1, scope='path_conv')
net = slim.batch_norm(net, scope='path_bn')
return net
if data_format == 'NHWC':
stride_spec = [1, stride, stride, 1]
else:
stride_spec = [1, 1, stride, stride]
# Skip path 1
path1 = tf.nn.avg_pool2d(net,
ksize=[1, 1, 1, 1],
strides=stride_spec,
padding='VALID',
data_format=data_format)
path1 = slim.conv2d(path1, int(output_filters / 2), 1, scope='path1_conv')
# Skip path 2
# First pad with 0's on the right and bottom, then shift the filter to
# include those 0's that were added.
if data_format == 'NHWC':
pad_arr = [[0, 0], [0, 1], [0, 1], [0, 0]]
path2 = tf.pad(tensor=net, paddings=pad_arr)[:, 1:, 1:, :]
concat_axis = 3
else:
pad_arr = [[0, 0], [0, 0], [0, 1], [0, 1]]
path2 = tf.pad(tensor=net, paddings=pad_arr)[:, :, 1:, 1:]
concat_axis = 1
path2 = tf.nn.avg_pool2d(path2,
ksize=[1, 1, 1, 1],
strides=stride_spec,
padding='VALID',
data_format=data_format)
# If odd number of filters, add an additional one to the second path.
final_filter_size = int(output_filters / 2) + int(output_filters % 2)
path2 = slim.conv2d(path2, final_filter_size, 1, scope='path2_conv')
# Concat and apply BN
final_path = tf.concat(values=[path1, path2], axis=concat_axis)
final_path = slim.batch_norm(final_path, scope='final_path_bn')
return final_path
def BatchNormClassifier(inputs, labels, scope=None, reuse=None):
with tf.compat.v1.variable_scope(scope, 'BatchNormClassifier', [inputs, labels],
reuse=reuse):
inputs = slim.batch_norm(inputs, decay=0.1)
predictions = slim.fully_connected(inputs, 1,
activation_fn=tf.sigmoid,
scope='fully_connected')
slim.losses.log_loss(predictions, labels)
return predictions
def build_pspnet(inputs,
label_size,
num_classes,
preset_model='PSPNet',
frontend="ResNet101",
pooling_type="MAX",
weight_decay=1e-5,
upscaling_method="conv",
is_training=True,
pretrained_dir="models"):
"""
Builds the PSPNet model.
Arguments:
inputs: The input tensor
label_size: Size of the final label tensor. We need to know this for proper upscaling
preset_model: Which model you want to use. Select which ResNet model to use for feature extraction
num_classes: Number of classes
pooling_type: Max or Average pooling
Returns:
PSPNet model
"""
logits, end_points, frontend_scope, init_fn = frontend_builder.build_frontend(
inputs,
frontend,
pretrained_dir=pretrained_dir,
is_training=is_training)
feature_map_shape = [int(x / 8.0) for x in label_size]
print(feature_map_shape)
psp = PyramidPoolingModule(end_points['pool3'],
feature_map_shape=feature_map_shape,
pooling_type=pooling_type)
net = slim.conv2d(psp, 512, [3, 3], activation_fn=None)
net = slim.batch_norm(net, fused=True)
net = tf.nn.relu(net)
if upscaling_method.lower() == "conv":
net = ConvUpscaleBlock(net, 256, kernel_size=[3, 3], scale=2)
net = ConvBlock(net, 256)
net = ConvUpscaleBlock(net, 128, kernel_size=[3, 3], scale=2)
net = ConvBlock(net, 128)
net = ConvUpscaleBlock(net, 64, kernel_size=[3, 3], scale=2)
net = ConvBlock(net, 64)
elif upscaling_method.lower() == "bilinear":
net = Upsampling(net, label_size)
net = slim.conv2d(net,
num_classes, [1, 1],
activation_fn=None,
scope='logits')
return net, init_fn
def _cifar_stem(inputs, hparams):
"""Stem used for models trained on Cifar."""
num_stem_filters = int(hparams.num_conv_filters * hparams.stem_multiplier)
net = slim.conv2d(
inputs,
num_stem_filters,
3,
scope='l1_stem_3x3')
net = slim.batch_norm(net, scope='l1_stem_bn')
return net, [None, net]
def ConvBlock(inputs, n_filters, kernel_size=[3, 3]): """ Builds the conv block for MobileNets Apply successivly a 2D convolution, BatchNormalization relu """ # Skip pointwise by setting num_outputs=Non net = slim.conv2d(inputs, n_filters, kernel_size=[1, 1], activation_fn=None) net = slim.batch_norm(net, fused=True) net = tf.nn.relu(net) return net
def ResidualUnit(inputs, n_filters=48, filter_size=3):
"""
A local residual unit
Arguments:
inputs: The input tensor
n_filters: Number of output feature maps for each conv
filter_size: Size of convolution kernel
Returns:
Output of local residual block
"""
net = slim.conv2d(inputs, n_filters, filter_size, activation_fn=None)
net = slim.batch_norm(net, fused=True)
net = tf.nn.relu(net)
net = slim.conv2d(net, n_filters, filter_size, activation_fn=None)
net = slim.batch_norm(net, fused=True)
return net
def conv_transpose_block(inputs, n_filters, strides=2, filter_size=[3, 3], dropout_p=0.0): """ Basic conv transpose block for Encoder-Decoder upsampling Apply successivly Transposed Convolution, BatchNormalization, ReLU nonlinearity Dropout (if dropout_p > 0) on the inputs """ conv = slim.conv2d_transpose(inputs, n_filters, kernel_size=[3, 3], stride=[strides, strides]) out = tf.nn.relu(slim.batch_norm(conv, fused=True)) if dropout_p != 0.0: out = slim.dropout(out, keep_prob=(1.0-dropout_p)) return out
def conv_block(inputs, n_filters, filter_size=[3, 3], dropout_p=0.0): """ Basic conv block for Encoder-Decoder Apply successivly Convolution, BatchNormalization, ReLU nonlinearity Dropout (if dropout_p > 0) on the inputs """ conv = slim.conv2d(inputs, n_filters, filter_size, activation_fn=None, normalizer_fn=None) out = tf.nn.relu(slim.batch_norm(conv, fused=True)) if dropout_p != 0.0: out = slim.dropout(out, keep_prob=(1.0-dropout_p)) return out
def AttentionRefinementModule(inputs, n_filters):
# Global average pooling
net = tf.reduce_mean(inputs, [1, 2], keep_dims=True)
net = slim.conv2d(net, n_filters, kernel_size=[1, 1])
net = slim.batch_norm(net, fused=True)
net = tf.sigmoid(net)
net = tf.multiply(inputs, net)
return net
def ResNetBlock_1(inputs, filters_1, filters_2):
net = tf.nn.relu(slim.batch_norm(inputs, fused=True))
net = slim.conv2d(net,
filters_1, [1, 1],
activation_fn=None,
normalizer_fn=None)
net = tf.nn.relu(slim.batch_norm(net, fused=True))
net = slim.conv2d(net,
filters_1, [3, 3],
activation_fn=None,
normalizer_fn=None)
net = tf.nn.relu(slim.batch_norm(net, fused=True))
net = slim.conv2d(net,
filters_2, [1, 1],
activation_fn=None,
normalizer_fn=None)
net = tf.add(inputs, net)
return net
def generator(self, t_z, t_text_embedding, t_training):
s = self.options['image_size']
s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16)
reduced_text_embedding = ops.lrelu(
ops.linear(t_text_embedding, self.options['t_dim'], 'g_embedding'))
z_concat = tf.concat([t_z, reduced_text_embedding], -1)
z_ = ops.linear(z_concat, self.options['gf_dim'] * 8 * s16 * s16,
'g_h0_lin')
h0 = tf.reshape(z_, [-1, s16, s16, self.options['gf_dim'] * 8])
h0 = tf.nn.relu(
slim.batch_norm(h0, is_training=t_training, scope="g_bn0"))
h1 = ops.deconv2d(
h0,
[self.options['batch_size'], s8, s8, self.options['gf_dim'] * 4],
name='g_h1')
h1 = tf.nn.relu(
slim.batch_norm(h1, is_training=t_training, scope="g_bn1"))
h2 = ops.deconv2d(
h1,
[self.options['batch_size'], s4, s4, self.options['gf_dim'] * 2],
name='g_h2')
h2 = tf.nn.relu(
slim.batch_norm(h2, is_training=t_training, scope="g_bn2"))
h3 = ops.deconv2d(
h2,
[self.options['batch_size'], s2, s2, self.options['gf_dim'] * 1],
name='g_h3')
h3 = tf.nn.relu(
slim.batch_norm(h3, is_training=t_training, scope="g_bn3"))
h4 = ops.deconv2d(h3, [self.options['batch_size'], s, s, 3],
name='g_h4')
return (tf.tanh(h4) / 2. + 0.5)
def _build_aux_head(net, end_points, num_classes, hparams, scope):
"""Auxiliary head used for all models across all datasets."""
with tf.compat.v1.variable_scope(scope):
aux_logits = tf.identity(net)
with tf.compat.v1.variable_scope('aux_logits'):
aux_logits = slim.avg_pool2d(aux_logits, [5, 5],
stride=3,
padding='VALID')
aux_logits = slim.conv2d(aux_logits, 128, [1, 1], scope='proj')
aux_logits = slim.batch_norm(aux_logits, scope='aux_bn0')
aux_logits = tf.nn.relu(aux_logits)
# Shape of feature map before the final layer.
shape = aux_logits.shape
if hparams.data_format == 'NHWC':
shape = shape[1:3]
else:
shape = shape[2:4]
aux_logits = slim.conv2d(aux_logits, 768, shape, padding='VALID')
aux_logits = slim.batch_norm(aux_logits, scope='aux_bn1')
aux_logits = tf.nn.relu(aux_logits)
aux_logits = slim.flatten(aux_logits)
aux_logits = slim.fully_connected(aux_logits, num_classes)
end_points['AuxLogits'] = aux_logits
