def auxiliary_classifier(inputs,
pool=avg_pool2d,
pool_size=5,
pool_stride=3,
n_filters_1x1=128,
n_units=1024,
drop_rate=0.7,
seed=42,
is_training=False,
regularizer_conv=None,
regularizer_dense=None,
kernel_init_conv=He_normal(seed=42),
kernel_init_dense=He_normal(seed=42),
name="inception_auxiliary_classifier"):
with tf.variable_scope(name):
# pool
pool = pool(inputs,
size=pool_size,
stride=pool_stride,
padding="VALID",
name="pool")
# 1x1
x_1x1 = conv2d_relu(pool,
size=1,
n_filters=n_filters_1x1,
stride=1,
regularizer=regularizer_conv,
kernel_init=kernel_init_conv,
name="conv_1x1")
# dense
flat = flatten(x_1x1, name="flatten")
dense = dense_relu(flat,
n_units=n_units,
regularizer=regularizer_dense,
kernel_init=kernel_init_dense,
name="dense")
# dropout
if drop_rate > 0.0:
dense = tf.layers.dropout(dense,
rate=drop_rate,
training=is_training,
seed=seed,
name="dropout")
return dense
def grid_fact2d(inputs,
n_filters_1x1=64,
n_reduce_3x3=96,
n_filters_3x3=128,
n_reduce_5x5=16,
n_filters_5x5=32,
n_filters_pool=32,
pool=max_pool2d,
pool_size=3,
regularizer=None,
kernel_init=He_normal(seed=42),
kernel_init_reduce=He_normal(seed=42),
name="inception_grid_2d_factorized"):
with tf.variable_scope(name):
# 1x1
x_1x1 = conv2d_relu(inputs,
size=1,
n_filters=n_filters_1x1,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_1x1")
# 3x3
reduce_3x3 = conv2d_relu(inputs,
size=1,
n_filters=n_reduce_3x3,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init_reduce,
name="conv_reduce_3x3")
x_3x3 = conv2d_relu(reduce_3x3,
size=3,
n_filters=n_filters_3x3,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_3x3")
# factorized 5x5
reduce_5x5 = conv2d_relu(inputs,
size=1,
n_filters=n_reduce_5x5,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init_reduce,
name="conv_reduce_5x5")
fact_5x5_1 = conv2d_relu(reduce_5x5,
size=3,
n_filters=n_filters_5x5,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init,
name="fact_conv_5x5_1")
fact_5x5_2 = conv2d_relu(fact_5x5_1,
size=3,
n_filters=n_filters_5x5,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init,
name="fact_conv_5x5_2")
# pool projection
pool = pool(inputs, size=pool_size, stride=1, name="pool")
proj_pool = conv2d_relu(pool,
size=1,
n_filters=n_filters_pool,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init_reduce,
name="conv_pool_proj")
inception = tf.concat([x_1x1, x_3x3, fact_5x5_2, proj_pool], axis=3)
return inception
def expanded_filterbank_fact1d_bn(
inputs,
n_filters_1x1=256,
n_reduce_1x3_3x1=384,
n_filters_1x3_3x1=256,
n_reduce_1x3=384,
n_filters_1x3=448,
n_filters_3x1=512,
n_filters_3x3_1x3_3x1=256,
n_filters_pool=256,
pool=avg_pool2d,
pool_size=3,
is_training=False,
regularizer=None,
kernel_init=He_normal(seed=42),
kernel_init_reduce=He_normal(seed=42),
name="inception_expanded_filterbank_batchnorm"):
with tf.variable_scope(name):
# 1x1
x_1x1 = conv2d_bn_relu(inputs,
size=1,
n_filters=n_filters_1x1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_1x1")
# 1x3 + 3x1
reduce_1x3_3x1 = conv2d_bn_relu(inputs,
size=1,
n_filters=n_reduce_1x3_3x1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init_reduce,
name="conv_reduce_1x3_3x1")
x_1x3 = conv2d_bn_relu(reduce_1x3_3x1,
size=[1, 3],
n_filters=n_filters_1x3_3x1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_1x3_1")
x_3x1 = conv2d_bn_relu(reduce_1x3_3x1,
size=[3, 1],
n_filters=n_filters_1x3_3x1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_3x1_1")
# 1x3 + 3x1 + (1x3 + 3x1)
reduce_1x3 = conv2d_bn_relu(inputs,
size=1,
n_filters=n_reduce_1x3,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init_reduce,
name="conv_reduce_1x3")
x_1x3_2 = conv2d_bn_relu(reduce_1x3,
size=[1, 3],
n_filters=n_filters_1x3,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_1x3_2")
x_3x1_2 = conv2d_bn_relu(x_1x3_2,
size=[3, 1],
n_filters=n_filters_3x1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_3x1_2")
x_3x3_1x3 = conv2d_relu(x_3x1_2,
size=[1, 3],
n_filters=n_filters_3x3_1x3_3x1,
stride=1,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_3x3_1x3")
x_3x3_3x1 = conv2d_bn_relu(x_3x1_2,
size=[3, 1],
n_filters=n_filters_3x3_1x3_3x1,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init,
name="conv_3x3_3x1")
# pool projection
pool = pool(inputs, size=pool_size, stride=1, name="pool")
proj_pool = conv2d_bn_relu(pool,
size=1,
n_filters=n_filters_pool,
stride=1,
is_training=is_training,
regularizer=regularizer,
kernel_init=kernel_init_reduce,
name="conv_pool_proj")
inception = tf.concat(
[x_1x1, x_1x3, x_3x1, x_3x3_1x3, x_3x3_3x1, proj_pool], axis=3)
return inception
def mnist_siamese_base(x, training=False, weight_decay=0.0001, seed=42):
layers = []
variables = []
conv1 = conv2d_relu(
x,
size=7,
n_filters=32,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 1),
name="conv1")
layers.append(("conv1", conv1))
pool1 = max_pool2d(conv1, size=2, stride=2, name="pool1")
layers.append(("pool1", pool1)) # 14x14
conv2 = conv2d_relu(
pool1,
size=5,
n_filters=64,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 2),
name="conv2")
layers.append(("conv2", conv2))
pool2 = max_pool2d(conv2, size=2, stride=2, name="pool2")
layers.append(("pool2", pool2)) # 7x7
conv3 = conv2d_relu(
pool2,
size=3,
n_filters=128,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 3),
name="conv3")
layers.append(("conv3", conv3))
pool3 = max_pool2d(conv3, size=2, stride=2, name="pool3")
layers.append(("pool3", pool3)) # 4x4
conv4 = conv2d_relu(
pool3,
size=1,
n_filters=256,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 4),
name="conv4")
layers.append(("conv4", conv4))
pool4 = max_pool2d(conv4, size=2, stride=2, name="pool4")
layers.append(("pool4", pool4)) # 2x2
conv5 = conv2d(pool4,
size=1,
n_filters=2,
kernel_init=He_normal(seed=seed + 5),
name="conv5")
layers.append(("conv5", conv5))
pool5 = max_pool2d(conv5, size=2, stride=2, name="pool5")
layers.append(("pool5", pool5)) # 1x1
flat1 = flatten(pool5, name="flatten1")
layers.append(("output", flat1))
return layers, variables
def cifar10_sequential_allconvC(x,
drop_rate=0.5,
drop_rate_input=0.2,
seed=42):
layers = []
variables = []
training = tf.placeholder(tf.bool, name="training")
variables.append(("training", training))
dropout_in = tf.layers.dropout(x,
rate=drop_rate_input,
training=training,
seed=seed + 0,
name="dropout_input")
layers.append(("dropout_in", dropout_in))
conv1 = conv2d_relu(
dropout_in,
size=3,
n_filters=96,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 1),
name="conv_1")
layers.append(("conv_1", conv1))
conv2 = conv2d_relu(
conv1,
size=3,
n_filters=96,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 2),
name="conv_2")
layers.append(("conv_2", conv2))
conv3 = conv2d_relu(
conv2,
stride=2,
size=3,
n_filters=96,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 3),
name="conv_3")
layers.append(("conv_3", conv3))
dropout1 = tf.layers.dropout(conv3,
rate=drop_rate,
training=training,
seed=seed + 3,
name="dropout_1")
layers.append(("dropout_1", dropout1))
conv4 = conv2d_relu(
dropout1,
size=3,
n_filters=192,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 4),
name="conv_4")
layers.append(("conv_4", conv4))
conv5 = conv2d_relu(
conv4,
size=3,
n_filters=192,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 5),
name="conv_5")
layers.append(("conv_5", conv5))
conv6 = conv2d_relu(
conv5,
stride=2,
size=3,
n_filters=192,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 6),
name="conv_6")
layers.append(("conv_6", conv6))
dropout2 = tf.layers.dropout(conv6,
rate=drop_rate,
training=training,
seed=seed + 6,
name="dropout_2")
layers.append(("dropout_2", dropout2))
conv7 = conv2d_relu(
dropout2,
size=3,
n_filters=192,
padding="VALID",
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 7),
name="conv_7")
layers.append(("conv_7", conv7))
conv8 = conv2d_relu(
conv7,
size=1,
n_filters=192,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 8),
name="conv_8")
layers.append(("conv_8", conv8))
conv9 = conv2d_relu(
conv8,
size=1,
n_filters=10,
kernel_init=tf.contrib.layers.xavier_initializer(seed=seed + 9),
name="conv_9")
layers.append(("conv_9", conv9))
pool = global_avg_pool2d(conv9)
layers.append(("logit", pool))
prob = tf.nn.softmax(pool, name="prob")
layers.append(("prob", prob))
return layers, variables
def cifar10_sequential_clrn5d3_wd(x,
drop_rate=0.5,
weight_decay=0.001,
seed=42):
layers = []
variables = []
training = tf.placeholder(tf.bool, name="training")
variables.append(("training", training))
# 2x conv3x3 selu + pool
conv1 = conv2d_relu(
x,
size=5,
n_filters=64,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 1),
name="conv_1")
layers.append(("conv_1", conv1))
norm1 = tf.nn.lrn(conv1,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_1")
layers.append(("norm_1", norm1))
pool1 = max_pool2d(norm1, size=3, stride=2, name="pool_1")
layers.append(("pool_1", pool1))
# 2x conv3x3 selu + pool
conv2 = conv2d_relu(
pool1,
size=5,
n_filters=64,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 2),
name="conv_2")
layers.append(("conv_2", conv2))
norm2 = tf.nn.lrn(conv2,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_2")
layers.append(("norm_2", norm2))
pool2 = max_pool2d(norm2, size=3, stride=2, name="pool_2")
layers.append(("pool_2", pool2))
# 2x conv3x3 selu + pool
conv3 = conv2d_relu(
pool2,
size=3,
n_filters=128,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 3),
name="conv_3")
layers.append(("conv_3", conv3))
norm3 = tf.nn.lrn(conv3,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_3")
layers.append(("norm_3", norm3))
conv4 = conv2d_relu(
norm3,
size=3,
n_filters=128,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 4),
name="conv_4")
layers.append(("conv_4", conv4))
norm4 = tf.nn.lrn(conv4,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_1")
layers.append(("norm_4", norm4))
conv5 = conv2d_relu(
norm4,
size=3,
n_filters=128,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 5),
name="conv_5")
layers.append(("conv_5", conv5))
norm5 = tf.nn.lrn(conv5,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_5")
layers.append(("norm_5", norm5))
pool3 = max_pool2d(norm5, size=2, stride=2, name="pool_3")
layers.append(("pool_3", pool3))
flat = flatten(pool3, name="flatten")
layers.append(("flatten", flat))
dense1 = dense_relu(
flat,
n_units=384,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 6),
name="dense_1")
layers.append(("dense_1", dense1))
if drop_rate > 0.0:
dense1 = tf.layers.dropout(dense1,
rate=drop_rate,
training=training,
seed=seed + 6,
name="dropout_1")
dense2 = dense_relu(
dense1,
n_units=192,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=He_normal(seed=seed + 7),
name="dense_2")
layers.append(("dense_2", dense2))
if drop_rate > 0.0:
dense2 = tf.layers.dropout(dense2,
rate=drop_rate,
training=training,
seed=seed + 7,
name="dropout_2")
# dense softmax
dense3 = dense(dense2,
n_units=10,
regularizer=tf.contrib.layers.l2_regularizer(weight_decay),
kernel_init=Kumar_normal(activation=None,
mode="FAN_IN",
seed=seed + 8),
name="dense_3")
layers.append(("logit", dense3))
prob = tf.nn.softmax(dense3, name="prob")
layers.append(("prob", prob))
return layers, variables
def cifar10_sequential_clrn3d(x,
drop_rate_1=0.2,
drop_rate_2=0.3,
drop_rate_3=0.4,
seed=42):
layers = []
variables = []
training = tf.placeholder(tf.bool, name="training")
variables.append(("training", training))
# conv5x5 relu + lrn + pool
conv1 = conv2d_relu(x,
size=5,
n_filters=32,
kernel_init=tf.truncated_normal_initializer(
stddev=5e-2, seed=seed + 1),
name="conv_1")
layers.append(("conv_1", conv1))
norm1 = tf.nn.lrn(conv1,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_1")
layers.append(("norm_1", norm1))
pool1 = max_pool2d(norm1, size=3, stride=2, name="pool_1")
layers.append(("pool_1", pool1))
dropout1 = tf.layers.dropout(pool1,
rate=drop_rate_1,
training=training,
seed=seed + 1,
name="dropout_1")
layers.append(("dropout_1", dropout1))
# conv5x5 relu + lrn + pool
conv2 = conv2d_relu(dropout1,
size=5,
n_filters=64,
kernel_init=tf.truncated_normal_initializer(
stddev=5e-2, seed=seed + 2),
name="conv_2")
layers.append(("conv_2", conv2))
norm2 = tf.nn.lrn(conv2,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_2")
layers.append(("norm_2", norm2))
pool2 = max_pool2d(norm2, size=3, stride=2, name="pool_2")
layers.append(("pool_2", pool2))
dropout2 = tf.layers.dropout(pool2,
rate=drop_rate_2,
training=training,
seed=seed + 2,
name="dropout_2")
layers.append(("dropout_2", dropout2))
# conv3x3 relu + lrn + pool
conv3 = conv2d_relu(dropout2,
size=3,
n_filters=128,
kernel_init=tf.truncated_normal_initializer(
stddev=5e-2, seed=seed + 3),
name="conv_3")
layers.append(("conv_3", conv3))
norm3 = tf.nn.lrn(conv3,
depth_radius=4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name="norm_3")
layers.append(("norm_3", norm3))
pool3 = max_pool2d(norm3, size=2, stride=2, name="pool_3")
layers.append(("pool_3", pool3))
dropout3 = tf.layers.dropout(pool3,
rate=drop_rate_3,
training=training,
seed=seed + 3,
name="dropout_3")
layers.append(("dropout_3", dropout3))
flat = flatten(dropout3, name="flatten")
layers.append(("flatten", flat))
# dense softmax
dense1 = dense(flat,
n_units=10,
kernel_init=tf.truncated_normal_initializer(stddev=1 /
192.0,
seed=seed + 4),
name="dense_1")
layers.append(("logit", dense1))
prob = tf.nn.softmax(dense1, name="prob")
layers.append(("prob", prob))
return layers, variables