def deep_cnn_v1(x,
training,
nbof_labels,
regularizer_rate=0,
fmaps=[32, 64, 128]):
def conv_layer(name, x, fmaps):
with tf.compat.v1.variable_scope('Conv_{}'.format(name)):
x = tf.compat.v1.layers.batch_normalization(
tf.nn.elu(
layers.conv2d(x,
fmaps=fmaps,
kernel=3,
strides=1,
regularizer_rate=regularizer_rate)),
training=training,
momentum=0.99,
gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate),
beta_regularizer=tf.keras.regularizers.l2(regularizer_rate))
return x
x = conv_layer('Input', x, fmaps[0])
x = tf.nn.max_pool2d(x, 2, 2, 'SAME')
for i in range(len(fmaps)):
x = conv_layer('{}_{}'.format(i, 0), x, fmaps[i])
x = conv_layer('{}_{}'.format(i, 1), x, fmaps[i])
x = tf.nn.max_pool2d(x, 2, 2, 'SAME')
x = tf.cond(training, lambda: tf.nn.dropout(x, rate=0.2 + i * 0.1),
lambda: x)
x = layers.global_avg_pool(x)
with tf.compat.v1.variable_scope('Output'):
logit = layers.dense(x, fmaps=nbof_labels)
return logit
def keras_resnet(inputs, training, nbof_labels, regularizer_rate=0):
def batch_norm(x):
with tf.compat.v1.variable_scope('BN'):
x = tf.layers.batch_normalization(
x,
training=training,
momentum=0.9,
gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate),
beta_regularizer=tf.keras.regularizers.l2(regularizer_rate))
return x
def BAN(x, use_bias=True, use_act=True, use_norm=True):
if use_bias: x = layers.bias(x, regularizer_rate=regularizer_rate)
if use_norm: x = batch_norm(x)
if use_act: x = layers.leaky_relu(x, alpha=0.2)
return x
def conv_layer(name,
x,
fmaps,
kernel=3,
strides=1,
padding='SAME',
use_bias=True,
use_act=True,
use_norm=True):
with tf.compat.v1.variable_scope('Conv2D_{}'.format(name)):
x = layers.conv2d(x,
fmaps=fmaps,
kernel=kernel,
strides=strides,
padding=padding,
regularizer_rate=regularizer_rate)
x = BAN(x)
return x
def dense_layer(name, x, fmaps):
with tf.compat.v1.variable_scope('Dense_{}'.format(name)):
x = layers.dense(x, fmaps=fmaps, regularizer_rate=regularizer_rate)
x = BAN(x, use_bias=False, use_act=False, use_norm=True)
return x
def block_res(name, x, fmaps, strides=1, use_conv=False):
f1, f2, f3 = fmaps, fmaps, 4 * fmaps
with tf.compat.v1.variable_scope('Block_{}'.format(name)):
if use_conv: # conv block
s = conv_layer('shortcut',
x,
f3,
kernel=1,
strides=strides,
use_act=False)
else: # identity block
s = x
strides = 1
r = conv_layer(name + '_0', x, fmaps=f1, kernel=1, strides=strides)
r = conv_layer(name + '_1', r, fmaps=f2, kernel=3, strides=1)
r = conv_layer(name + '_2',
r,
fmaps=f3,
kernel=1,
strides=1,
use_act=False)
return layers.leaky_relu(r + s)
# Inputs
x = conv_layer('inputs', inputs, 64, 3, 1)
# Middle layers
fmaps = [64, 128, 256, 512]
nbof_unit = [3, 4, 6, 3]
strides = [1, 2, 2, 2]
for i in range(len(fmaps)):
x = block_res('{}_{}'.format(i, 0),
x,
fmaps[i],
strides=strides[i],
use_conv=True)
for j in range(nbof_unit[i] - 1):
x = block_res('{}_{}'.format(i, j + 1), x, fmaps[i])
x = layers.global_avg_pool(x)
logit = dense_layer(name='logit', x=x, fmaps=nbof_labels)
return logit
def wideresnet_se(inputs, training, nbof_labels, regularizer_rate=0):
def batch_norm(x):
with tf.compat.v1.variable_scope('BN'):
# x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate)
x = tf.compat.v1.layers.batch_normalization(
x,
training=training,
momentum=0.99,
gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate),
beta_regularizer=tf.keras.regularizers.l2(regularizer_rate))
# if len(x.shape)>2: x = layers.pixel_norm(x)
return x
def block_se(name, x, fmaps, se_ratio):
squeezed_fmaps = max(1, int(se_ratio * fmaps))
with tf.compat.v1.variable_scope('SEBlock_{}'.format(name)):
with tf.compat.v1.variable_scope('Squeeze'):
s = tf.compat.v1.reduce_mean(x, axis=[1, 2], keepdims=True)
s = tf.nn.swish(layers.conv2d(s, squeezed_fmaps, kernel=1))
with tf.compat.v1.variable_scope('Excite'):
s = tf.nn.sigmoid(layers.conv2d(s, fmaps, kernel=1))
return s * x
act = layers.leaky_relu
# act = tf.nn.selu
def block_basic(name, x, fmaps, strides, dropout_rate=0.0):
with tf.compat.v1.variable_scope('Block_{}'.format(name)):
if x.shape[-1] == fmaps:
r = act(batch_norm(x))
s = x if strides == 1 else tf.nn.max_pool2d(
x, ksize=1, strides=2, padding='SAME')
else:
x = act(batch_norm(x))
with tf.compat.v1.variable_scope('Shortcut'):
s = layers.conv2d(x, fmaps, kernel=1, strides=strides)
# r = block_se(name, r, fmaps, 0.25)
with tf.compat.v1.variable_scope('Conv2D_0'):
r = act(
batch_norm(
layers.conv2d(r if x.shape[-1] == fmaps else x,
fmaps=fmaps,
kernel=3,
strides=strides)))
if dropout_rate > 0:
r = tf.cond(training,
lambda: tf.nn.dropout(r, rate=dropout_rate),
lambda: r,
name='use_dropout')
with tf.compat.v1.variable_scope('Conv2D_1'):
r = layers.conv2d(r, fmaps=fmaps, kernel=3)
r = block_se(name, r, fmaps, 0.25)
return r + s
# Inputs
with tf.compat.v1.variable_scope('Conv2D_1'):
x = layers.conv2d(inputs, fmaps=16, kernel=3)
# Middle layers
fmaps = [160, 320, 640]
nbof_unit = [4, 4, 4]
# nbof_unit = [1,1,1]
strides = [1, 2, 2]
dropouts = [0., 0., 0.]
# dropouts = [0.025,0.05,0.1]
# dropouts = [0.1,0.2,0.3]
# strides = [2,2,2]
# nbof_unit = [2,3,4,6,3]
for i in range(len(fmaps)):
x = block_basic('{}_{}'.format(i, 0),
x,
fmaps[i],
strides=strides[i],
dropout_rate=dropouts[i])
for j in range(nbof_unit[i] - 1):
x = block_basic('{}_{}'.format(i, j + 1),
x,
fmaps[i],
strides=1,
dropout_rate=dropouts[i])
# Output
with tf.compat.v1.variable_scope('Output'):
x = act(batch_norm(x))
x = layers.global_avg_pool(x)
logit = layers.dense(x, fmaps=nbof_labels)
return logit
def wideresnet(inputs,
training,
nbof_labels,
regularizer_rate=0,
fmaps=[160, 320, 640],
nbof_unit=[4, 4, 4],
strides=[1, 2, 2],
dropouts=[0., 0., 0.]):
def batch_norm(x):
with tf.compat.v1.variable_scope('BN'):
# x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate)
x = tf.compat.v1.layers.batch_normalization(
x,
training=training,
momentum=0.99,
gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate),
beta_regularizer=tf.keras.regularizers.l2(regularizer_rate))
# x = tf.nn.l2_normalize(x, axis=-1)
# if len(x.shape)>2: x = layers.pixel_norm(x)
return x
def block_basic(name, x, fmaps, strides, dropout_rate=0.0):
with tf.compat.v1.variable_scope('Block_{}'.format(name)):
if x.shape[-1] == fmaps:
r = layers.leaky_relu(batch_norm(x))
s = x if strides == 1 else tf.nn.max_pool2d(
x, ksize=1, strides=2, padding='SAME')
else:
x = layers.leaky_relu(batch_norm(x))
with tf.compat.v1.variable_scope('Shortcut'):
s = layers.conv2d(x,
fmaps,
kernel=1,
strides=strides,
regularizer_rate=regularizer_rate)
with tf.compat.v1.variable_scope('Conv2D_0'):
r = layers.leaky_relu(
batch_norm(
layers.conv2d(r if x.shape[-1] == fmaps else x,
fmaps=fmaps,
kernel=3,
strides=strides,
regularizer_rate=regularizer_rate)))
if dropout_rate > 0:
r = tf.cond(training,
lambda: tf.nn.dropout(r, rate=dropout_rate),
lambda: r,
name='use_dropout')
with tf.compat.v1.variable_scope('Conv2D_1'):
r = layers.conv2d(r,
fmaps=fmaps,
kernel=3,
regularizer_rate=regularizer_rate)
return r + s
# Inputs
with tf.compat.v1.variable_scope('Conv2D_1'):
x = layers.conv2d(inputs,
fmaps=fmaps[0],
kernel=3,
regularizer_rate=regularizer_rate)
# Middle layers
for i in range(len(fmaps)):
x = block_basic('{}_{}'.format(i, 0),
x,
fmaps[i],
strides=strides[i],
dropout_rate=dropouts[i])
for j in range(nbof_unit[i] - 1):
x = block_basic('{}_{}'.format(i, j + 1),
x,
fmaps[i],
strides=1,
dropout_rate=dropouts[i])
# Output
with tf.compat.v1.variable_scope('Output'):
x = layers.leaky_relu(batch_norm(x))
x = layers.global_avg_pool(x)
logit = layers.dense(x,
fmaps=nbof_labels,
regularizer_rate=regularizer_rate)
return logit
def insightface_resnet(inputs,
training,
nbof_labels,
regularizer_rate=0,
dropout_rate=0.,
fmaps=[64, 128, 256, 256, 512],
nbof_unit=[1, 2, 3, 5, 2],
strides=[2, 2, 2, 2, 2]):
def batch_norm(x):
with tf.compat.v1.variable_scope('BN'):
# x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate)
x = tf.compat.v1.layers.batch_normalization(
x,
training=training,
momentum=0.99,
gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate),
beta_regularizer=tf.keras.regularizers.l2(regularizer_rate))
# if len(x.shape)>2: x = layers.pixel_norm(x)
return x
def BAN(x, use_bias=True, use_act=True, use_norm=True):
if use_bias: x = layers.bias(x, regularizer_rate=regularizer_rate)
if use_act: x = layers.leaky_relu(x, alpha=0.2)
if use_norm: x = batch_norm(x)
return x
def conv_layer(name, x, fmaps, kernel=3, strides=1, padding='SAME'):
with tf.compat.v1.variable_scope('Conv2D_{}'.format(name)):
x = layers.conv2d(x,
fmaps=fmaps,
kernel=kernel,
strides=strides,
padding=padding,
regularizer_rate=regularizer_rate)
x = BAN(x)
return x
def dense_layer(name, x, fmaps):
with tf.compat.v1.variable_scope('Dense_{}'.format(name)):
x = layers.dense(x, fmaps=fmaps, regularizer_rate=regularizer_rate)
x = BAN(x, use_bias=False, use_act=False, use_norm=True)
return x
def block_res(name, x, fmaps, strides):
with tf.compat.v1.variable_scope('Block_{}'.format(name)):
if x.shape[-1] == fmaps:
s = x if strides == 1 else tf.nn.max_pool2d(
x, ksize=1, strides=2, padding='SAME')
else:
s = conv_layer('shortcut', x, fmaps, kernel=1, strides=strides)
r = batch_norm(x)
r = conv_layer(name + '_0', r, fmaps)
r = conv_layer(name + '_1', r, fmaps, strides=strides)
return r + s
# For Cifar10
# fmaps = [32,64,128,256]
# nbof_unit = [1,2,3,5]
# strides = [1,2,2,2]
# For vggface2 with 112x112
# 22 millions of parameters (for face recognition)
# fmaps = [64,128,256,256,512]
# nbof_unit = [1,2,3,5,2]
# strides = [2,2,2,2,2]
# Inputs
x = conv_layer('inputs', inputs, fmaps[0], 3, 1)
# Middle layers
for i in range(len(fmaps)):
x = block_res('{}_{}'.format(i, 0), x, fmaps[i], strides=strides[i])
for j in range(nbof_unit[i] - 1):
x = block_res('{}_{}'.format(i, j + 1), x, fmaps[i], strides=1)
# Output layers
x = layers.global_avg_pool(x)
if dropout_rate > 0:
x = tf.cond(training,
lambda: tf.nn.dropout(x, rate=dropout_rate),
lambda: x,
name='use_dropout')
logit = dense_layer(name='logit', x=x, fmaps=nbof_labels)
return logit
def EfficientNet(
inputs,
training,
nbof_labels,
width_coefficient=1.0,
depth_coefficient=1.0,
#dropout_rate=0.,
depth_divisor=8,
min_depth=None):
block_args_list = get_default_block_list()
with tf.variable_scope('Input'):
x = layers.conv2d(inputs, 32, kernel=1, strides=1)
x = layers.batch_norm(x, training)
x = tf.nn.swish(x)
# Blocks part
for i, block_args in enumerate(block_args_list):
assert block_args.num_repeat > 0
# Update block input and output filters based on depth multiplier.
block_args.input_filters = round_filters(block_args.input_filters,
width_coefficient,
depth_divisor, min_depth)
block_args.output_filters = round_filters(block_args.output_filters,
width_coefficient,
depth_divisor, min_depth)
block_args.num_repeat = round_repeats(block_args.num_repeat,
depth_coefficient)
# The first block needs to take care of stride and filter size increase.
x = MBConvBlock('{}_0'.format(i), block_args.input_filters,
block_args.output_filters, block_args.kernel_size,
block_args.strides, block_args.expand_ratio,
block_args.se_ratio, block_args.identity_skip,
training)(x)
if block_args.num_repeat > 1:
block_args.input_filters = block_args.output_filters
block_args.strides = 1
for j in range(block_args.num_repeat - 1):
x = MBConvBlock('{}_{}'.format(i, j + 1), block_args.input_filters,
block_args.output_filters, block_args.kernel_size,
block_args.strides, block_args.expand_ratio,
block_args.se_ratio, block_args.identity_skip,
training)(x)
# Head part
with tf.variable_scope('HeadConv'):
x = layers.conv2d(x,
fmaps=round_filters(1280, width_coefficient,
depth_coefficient, min_depth),
kernel=1)
x = layers.batch_norm(x, training)
x = tf.nn.swish(x)
with tf.variable_scope('HeadDense'):
x = layers.global_avg_pool(x)
logit = layers.dense(x=x, fmaps=nbof_labels)
return logit
def shake_shake(inputs, training, nbof_labels, regularizer_rate=0):
def batch_norm(x):
with tf.compat.v1.variable_scope('BN'):
# x = layers.batch_norm(x, training=training, regularizer_rate=regularizer_rate)
x = tf.compat.v1.layers.batch_normalization(
x,
training=training,
momentum=0.99,
gamma_regularizer=tf.keras.regularizers.l2(regularizer_rate),
beta_regularizer=tf.keras.regularizers.l2(regularizer_rate))
# if len(x.shape)>2: x = layers.pixel_norm(x)
return x
def BAN(x, use_bias=True, use_act=True, use_norm=True):
if use_bias: x = layers.bias(x, regularizer_rate=regularizer_rate)
if use_act: x = layers.leaky_relu(x, alpha=0.2)
if use_norm: x = batch_norm(x)
return x
def conv_layer(name, x, fmaps, kernel=3, strides=1, padding='SAME'):
with tf.compat.v1.variable_scope('Conv2D_{}'.format(name)):
x = layers.conv2d(x,
fmaps=fmaps,
kernel=kernel,
strides=strides,
padding=padding,
regularizer_rate=regularizer_rate)
x = BAN(x)
return x
def dense_layer(name, x, fmaps):
with tf.compat.v1.variable_scope('Dense_{}'.format(name)):
x = layers.dense(x, fmaps=fmaps, regularizer_rate=regularizer_rate)
x = BAN(x, use_bias=False, use_act=False, use_norm=True)
return x
def block_res(name, x, fmaps, strides):
with tf.compat.v1.variable_scope('Block_{}'.format(name)):
if x.shape[-1] == fmaps:
s = x if strides == 1 else tf.nn.max_pool2d(
x, ksize=1, strides=2, padding='SAME')
else:
s = conv_layer('shortcut', x, fmaps, kernel=1, strides=strides)
with tf.variable_scope('R'):
r = batch_norm(x)
r = conv_layer(name + '_0', r, fmaps)
r = conv_layer(name + '_1', r, fmaps, strides=strides)
with tf.variable_scope('T'):
t = batch_norm(x)
t = conv_layer(name + '_0', t, fmaps)
t = conv_layer(name + '_1', t, fmaps, strides=strides)
alpha = tf.compat.v1.get_variable(
name='alpha',
shape=[3, 1, 1, r.shape[-1]],
dtype=r.dtype,
initializer=tf.compat.v1.initializers.constant(1 / 3),
trainable=True)
return alpha[0] * r + alpha[1] * t + alpha[2] * s
# alpha = tf.cond(training, lambda: tf.random.uniform([1])[0], lambda: tf.constant(0.5))
# return alpha*r + (1-alpha)*t + s
# Inputs
x = conv_layer('inputs', inputs, 32, 3, 1)
# Middle layers
fmaps = [32, 64, 128, 256]
# fmaps = [64,128,256]
nbof_unit = [1, 1, 1, 1]
# nbof_unit = [1,1,1]
strides = [1, 2, 2, 2]
# strides = [2,2,2]
# nbof_unit = [2,3,4,6,3]
for i in range(len(fmaps)):
x = block_res('{}_{}'.format(i, 0), x, fmaps[i], strides=strides[i])
for j in range(nbof_unit[i] - 1):
x = block_res('{}_{}'.format(i, j + 1), x, fmaps[i], strides=1)
x = layers.global_avg_pool(x)
x = tf.cond(training,
lambda: tf.nn.dropout(x, rate=0.2),
lambda: x,
name='use_dropout')
logit = dense_layer(name='logit', x=x, fmaps=nbof_labels)
return logit