def se_resnet_residual_block(
inputs,
n_filters,
size=3,
stride=1,
activation=tf.nn.relu,
ratio=16,
regularizer=None,
kernel_init=He_normal(seed=42),
se_kernel_init_1=He_normal(seed=42),
se_kernel_init_2=Kumar_normal(activation="sigmoid",
mode="FAN_AVG",
seed=42),
is_training=False,
name="se_resnet_residual_block"):
with tf.variable_scope(name):
if (inputs.shape[3] != n_filters) or (stride != 1):
shortcut = conv2d_bn(inputs,
size=1,
n_filters=n_filters,
stride=stride,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="shortcut")
else:
shortcut = tf.identity(inputs, name="shortcut")
x = conv2d_bn_act(inputs,
size=size,
n_filters=n_filters,
stride=stride,
activation=activation,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_act_1")
x = conv2d_bn(x,
size=size,
n_filters=n_filters,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_2")
x = squeeze_and_excite(x,
ratio=ratio,
regularizer=regularizer,
kernel_init_1=se_kernel_init_1,
kernel_init_2=se_kernel_init_2,
name="squeeze_excite")
x = tf.add(x, shortcut, name="add")
x = activation(x, name="activation_2")
return x
def bottleneck_block(inputs,
n_filters,
n_filters_reduce,
size=3,
stride=1,
activation=tf.nn.relu,
regularizer=None,
kernel_init=He_normal(seed=42),
is_training=False,
name="bottleneck_block"):
with tf.variable_scope(name):
if (inputs.shape[3] != n_filters) or (stride != 1):
shortcut = conv2d_bn(inputs,
size=1,
n_filters=n_filters,
stride=stride,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="shortcut")
else:
shortcut = tf.identity(inputs, name="shortcut")
x = conv2d_bn_act(inputs,
size=1,
n_filters=n_filters_reduce,
stride=stride,
activation=activation,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_act_1")
x = conv2d_bn_act(x,
size=size,
n_filters=n_filters_reduce,
stride=1,
activation=activation,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_act_2")
x = conv2d_bn(x,
size=1,
n_filters=n_filters,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_3")
x = tf.add(x, shortcut, name="add")
x = activation(x, name="activation_3")
return x
def bottleneck_block(
inputs,
cardinality,
group_width,
size = 3,
stride = 1,
activation = tf.nn.relu,
is_training = False,
regularizer = None,
kernel_init = He_normal(seed = 42),
name = "bottleneck_block"):
n_filters_reduce = cardinality*group_width
n_filters = n_filters_reduce*2
with tf.variable_scope(name):
if (inputs.shape[3] != n_filters) or (stride != 1):
shortcut = conv2d_bn(
inputs, size = 1, n_filters = n_filters, stride = stride,
is_training = is_training,
regularizer = regularizer,
kernel_init = kernel_init,
name = "shortcut")
else:
shortcut = tf.identity(inputs, name = "shortcut")
x = conv2d_bn_act(
inputs, size = 1, n_filters = n_filters_reduce, stride = 1,
activation = activation,
is_training = is_training,
regularizer = regularizer,
kernel_init = kernel_init,
name = "conv_bn_act_1")
x = group_conv2d(
x, size = size, stride = stride,
cardinality = cardinality,
n_filters = n_filters_reduce,
regularizer = regularizer,
kernel_init = kernel_init,
name = "group_conv_2"
)
x = tf.layers.batch_normalization(
x, training = is_training, name = "batch_norm_2")
x = activation(x, name = "activation_2")
x = conv2d_bn(
x, size = 1, n_filters = n_filters, stride = 1,
is_training = is_training,
regularizer = regularizer,
kernel_init = kernel_init,
name = "conv_bn_3")
x = tf.add(x, shortcut, name = "add")
x = activation(x, name = "activation_3")
return x
def auxiliary_classifier(inputs,
classes,
is_training=False,
regularizer=None,
activation=tf.nn.relu,
conv_kernel_init=He_normal(seed=42),
conv_bias_init=tf.zeros_initializer(),
dense_kernel_init=Kumar_normal(activation=None,
mode="FAN_IN",
seed=42),
name="nasnet_auxiliary_classifier"):
with tf.variable_scope(name):
x = inputs
if activation is not None:
x = activation(x, name="activation_1")
x = avg_pool2d(x, size=5, stride=3, padding="VALID", name="avg_pool")
x = conv2d_bn(x,
n_filters=128,
size=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=conv_kernel_init,
bias_init=conv_bias_init,
name="conv_projection")
if activation is not None:
x = activation(x, name="activation_2")
x = conv2d_bn(x,
n_filters=768,
size=[x.get_shape()[1].value,
x.get_shape()[2].value],
padding="VALID",
is_training=is_training,
regularizer=regularizer,
kernel_init=conv_kernel_init,
bias_init=conv_bias_init,
name="conv_reduction")
if activation is not None:
x = activation(x, name="activation_3")
x = global_avg_pool2d(x, name="global_avg_pool")
x = dense(x,
n_units=classes,
regularizer=regularizer,
kernel_init=dense_kernel_init,
name="dense")
return x
def exit_module(inputs,
size=3,
n_filters_1=[728, 1024],
n_filters_2=[1536, 2048],
pool_size=3,
is_training=False,
regularizer=None,
kernel_init=He_normal(seed=42),
bias_init=tf.zeros_initializer(),
name="xception_exit_module"):
with tf.variable_scope(name):
shortcut = conv2d_bn(inputs,
size=1,
n_filters=n_filters_1[-1],
stride=2,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="shortcut")
x = inputs
for r in range(len(n_filters_1)):
x = tf.nn.relu(x, name="relu_1_" + str(r))
x = separable_conv2d(x,
size=size,
n_filters=n_filters_1[r],
stride=1,
depth_multiplier=1,
regularizer=regularizer,
depth_init=kernel_init,
pointwise_init=kernel_init,
bias_init=bias_init,
name="separable_conv_1_" + str(r))
x = tf.layers.batch_normalization(x,
training=is_training,
name="bn_1_" + str(r))
x = max_pool2d(x, size=pool_size, stride=2, name="max_pool")
x = tf.add(x, shortcut, name="add_1")
for r in range(len(n_filters_2)):
x = separable_conv2d(x,
size=size,
n_filters=n_filters_2[r],
stride=1,
depth_multiplier=1,
regularizer=regularizer,
depth_init=kernel_init,
pointwise_init=kernel_init,
bias_init=bias_init,
name="separable_conv_2_" + str(r))
x = tf.layers.batch_normalization(x,
training=is_training,
name="bn_2_" + str(r))
x = tf.nn.relu(x, name="relu_2_" + str(r))
return x
def entry_block(inputs,
n_filters,
n_repeat=2,
conv_size=3,
pool_size=3,
init_activation=tf.nn.relu,
regularizer=None,
kernel_init=He_normal(seed=42),
bias_init=tf.zeros_initializer(),
is_training=False,
name="xception_entry_block"):
with tf.variable_scope(name):
shortcut = conv2d_bn(inputs,
size=1,
n_filters=n_filters,
stride=2,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="shortcut")
x = inputs
for r in range(n_repeat):
if r == 0:
activation = init_activation
else:
activation = tf.nn.relu
if activation is not None:
x = activation(x)
x = separable_conv2d(x,
size=conv_size,
n_filters=n_filters,
stride=1,
depth_multiplier=1,
regularizer=regularizer,
depth_init=kernel_init,
pointwise_init=kernel_init,
bias_init=bias_init,
name="separable_conv_" + str(r))
x = tf.layers.batch_normalization(x,
training=is_training,
name="bn_" + str(r))
x = max_pool2d(x, size=pool_size, stride=2, name="max_pool")
outputs = tf.add(x, shortcut, name="add")
return outputs
def squeeze(x,
n_filters,
activation=tf.nn.relu,
is_training=False,
regularizer=None,
kernel_init=He_normal(seed=42),
bias_init=tf.zeros_initializer(),
name="squeeze"):
with tf.variable_scope(name):
if activation is not None:
x = activation(x, name="activation")
x = conv2d_bn(x,
n_filters=n_filters,
size=1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="conv")
return x
def cifar10_nasnet(x, drop_rate = 0.0, seed = 42):
penultimate_filters = 768
nb_blocks = 6
stem_filters = 32
filters_multiplier = 2
filters = penultimate_filters // 24 # 2x2x6 -> increase two times 2x and concatenate 6 branches
layers = []
variables = []
training = tf.placeholder(tf.bool, name="training")
variables.append(("training", training))
x = conv2d_bn(
x, n_filters = stem_filters, size = 3, stride = 1,
is_training = training,
kernel_init = He_normal(seed = seed+1),
name = "initial_conv")
layers.append(("initial_conv", x))
p = None
for i in range(nb_blocks):
x, p = nasnet.Normal_A(
x, p,
n_filters = filters,
is_training = training,
kernel_init = He_normal(seed = seed+2+i),
name = "nasnet_normal_"+str(i)
)
layers.append(("nasnet_normal_"+str(i), x))
x, _ = nasnet.Reduction_A(
x, p,
n_filters = filters * filters_multiplier,
is_training = training,
kernel_init = He_normal(seed = seed+3+nb_blocks),
name = "nasnet_reduction_0"
)
layers.append(("nasnet_reduction_0", x))
for i in range(nb_blocks):
x, p = nasnet.Normal_A(
x, p,
n_filters = filters * filters_multiplier,
is_training = training,
kernel_init = He_normal(seed = seed+4+nb_blocks+i),
name = "nasnet_normal_"+str(nb_blocks+i)
)
layers.append(("nasnet_normal_"+str(nb_blocks+i), x))
x, _ = nasnet.Reduction_A(
x, p,
n_filters = filters * filters_multiplier ** 2,
is_training = training,
kernel_init = He_normal(seed = seed+5+2*nb_blocks),
name = "nasnet_reduction_1"
)
layers.append(("nasnet_reduction_1", x))
aux = nasnet.auxiliary_classifier(
x, classes = 10,
is_training = training,
conv_kernel_init = He_normal(seed = seed),
dense_kernel_init = Kumar_normal(activation = None, mode = "FAN_IN", seed = seed),
name = "nasnet_aux_classifier"
)
layers.append(("aux_logit", aux))
aux_prob = tf.nn.softmax(aux, name = "prob")
layers.append(("aux_prob", aux_prob))
for i in range(nb_blocks):
x, p = nasnet.Normal_A(
x, p,
n_filters = filters * filters_multiplier ** 2,
is_training = training,
kernel_init = He_normal(seed = seed+6+2*nb_blocks+i),
name = "nasnet_normal_"+str(2*nb_blocks+i)
)
layers.append(("nasnet_normal_"+str(2*nb_blocks+i), x))
x = tf.nn.relu(x, name = "relu")
layers.append(("relu", x))
x = global_avg_pool2d(x, name = "pool")
layers.append(("pool", x))
if drop_rate > 0.0:
x = tf.layers.dropout(
x, rate = drop_rate, training = training,
seed = seed+7+3*nb_blocks, name = "dropout")
layers.append(("dropout", x))
x = dense(
x, n_units = 10,
kernel_init = Kumar_normal(activation = None, mode = "FAN_IN", seed = seed+8+3*nb_blocks),
name = "dense")
layers.append(("logit", x))
prob = tf.nn.softmax(x, name = "prob")
layers.append(("prob", prob))
return layers, variables
def se_resnext_bottleneck_block(inputs,
cardinality,
group_width,
size=3,
stride=1,
ratio=16,
activation=tf.nn.relu,
is_training=False,
regularizer=None,
kernel_init=He_normal(seed=42),
se_kernel_init_1=He_normal(seed=42),
se_kernel_init_2=Kumar_normal(
activation="sigmoid",
mode="FAN_AVG",
seed=42),
name="se_resnext_bottleneck_block"):
n_filters_reduce = cardinality * group_width
n_filters = n_filters_reduce * 2
with tf.variable_scope(name):
if (inputs.shape[3] != n_filters) or (stride != 1):
shortcut = conv2d_bn(inputs,
size=1,
n_filters=n_filters,
stride=stride,
is_training=is_training,
kernel_init=kernel_init,
name="shortcut")
else:
shortcut = tf.identity(inputs, name="shortcut")
x = conv2d_bn_act(inputs,
size=1,
n_filters=n_filters_reduce,
stride=1,
activation=activation,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_act_1")
x = group_conv2d_fixdepth(x,
size=size,
stride=stride,
cardinality=cardinality,
group_width=group_width,
regularizer=regularizer,
kernel_init=kernel_init,
name="group_conv_2")
x = tf.layers.batch_normalization(x,
training=is_training,
name="batch_norm_2")
x = activation(x, name="activation_2")
x = conv2d_bn(x,
size=1,
n_filters=n_filters,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_bn_3")
x = squeeze_and_excite(x,
ratio=ratio,
regularizer=regularizer,
kernel_init_1=se_kernel_init_1,
kernel_init_2=se_kernel_init_2,
name="squeeze_excite")
x = tf.add(x, shortcut, name="add")
x = activation(x, name="activation_3")
return x
def inverted_residual(inputs,
n_filters,
expand_ratio=1.0,
size=3,
stride=1,
activation=tf.nn.relu6,
regularizer=None,
kernel_init=He_normal(seed=42),
bias_init=tf.zeros_initializer(),
is_training=False,
name="inverted_residual"):
n_filters_expand = int(n_filters * expand_ratio)
with tf.variable_scope(name):
if stride == 1:
if inputs.shape[3] != n_filters:
shortcut = conv2d_bn(inputs,
size=1,
n_filters=n_filters,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="shortcut")
else:
shortcut = tf.identity(inputs, name="shortcut")
else:
shortcut = None
# pointwise
x = conv2d_bn_act(inputs,
size=1,
n_filters=n_filters_expand,
stride=1,
activation=activation,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="conv_1")
# depthwise
x = separable_conv2d(x,
size=size,
n_filters=n_filters_expand,
stride=stride,
regularizer=regularizer,
depth_init=kernel_init,
pointwise_init=kernel_init,
bias_init=bias_init,
name="separable_conv")
x = tf.layers.batch_normalization(x,
training=is_training,
name="batch_norm_1")
if activation is not None:
x = activation(x, name="activation")
# pointwise
x = conv2d_bn(inputs,
size=1,
n_filters=n_filters,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="conv_2")
if shortcut is not None:
x = tf.add(x, shortcut, name="add")
return x
def mnist_resnet_cbn1r3d1(x):
"""Creates residual neural network for MNIST. The network
uses 1 batch normalized convolutional and 3 residual layers to create
the representation part of the network. The output probabilities are
generated by one dense layer followed by a softmax function.
Args:
x: A tensor representing the input.
Returns:
A tuple containing the layers of the network graph and additional
placeholders if any. Layers are represented as list of named tuples.
"""
layers = []
variables = []
training = tf.placeholder(tf.bool, name="training")
variables.append(("training", training))
conv1 = conv2d_bn(x,
size=3,
n_filters=16,
kernel_init=Kumar_initializer(mode="FAN_AVG"),
name="initial_conv")
layers.append(("initial_conv", conv1))
res1 = resnet.residual_layer(conv1,
n_filters=16,
n_blocks=2,
stride=1,
is_training=training,
name="residual1")
layers.append(("residual1", res1))
res2 = resnet.residual_layer(res1,
n_filters=32,
n_blocks=2,
stride=2,
is_training=training,
name="residual2")
layers.append(("residual2", res2))
res3 = resnet.residual_layer(res2,
n_filters=64,
n_blocks=2,
stride=2,
is_training=training,
name="residual3")
layers.append(("residual3", res3))
pool1 = tf.reduce_mean(res3, [1, 2]) # global average pooling
layers.append(("pool", pool1))
fc2 = dense(pool1,
n_units=10,
activation=None,
kernel_init=Kumar_initializer(mode="FAN_IN"),
name="fc2")
layers.append(("fc2", fc2))
prob = tf.nn.softmax(fc2, name="prob")
layers.append(("prob", prob))
return layers, variables
def adjust(p,
ref,
n_filters,
activation=tf.nn.relu,
is_training=False,
regularizer=None,
kernel_init=He_normal(seed=42),
bias_init=tf.zeros_initializer(),
name="adjust"):
#NHWC
channel_dim = 3
img_dim = 1
with tf.variable_scope(name):
if p is None:
p = ref
elif p.get_shape()[img_dim].value != ref.get_shape()[img_dim].value:
with tf.variable_scope("reduction"):
p = activation(p, name="activation")
p1 = avg_pool2d(p,
size=1,
stride=2,
padding="VALID",
name="avg_pool_1")
p1 = conv2d(p1,
n_filters=n_filters // 2,
size=1,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="conv_1")
p2 = zero_pad2d(p, pad=[[0, 1], [0, 1]], name="zero_pad")
p2 = crop2d(p2, crop=[[1, 0], [1, 0]], name="crop")
p2 = avg_pool2d(p2,
size=1,
stride=2,
padding="VALID",
name="avg_pool_2")
p2 = conv2d(p2,
n_filters=n_filters // 2,
size=1,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="conv_2")
p = tf.concat([p1, p2], axis=3, name="concat")
p = tf.layers.batch_normalization(p,
training=is_training,
name="batch_norm")
elif p.get_shape()[channel_dim].value != n_filters:
with tf.variable_scope("projection"):
p = activation(p, name="activation")
p = conv2d_bn(p,
n_filters=n_filters,
size=1,
regularizer=regularizer,
kernel_init=kernel_init,
bias_init=bias_init,
name="conv_bn")
return p