def conv_battery(x,
global_step,
convk=3,
n=32,
mpk=2,
mpstride=1,
dropout=0.0,
is_training=False,
name="C",
verbose=False):
# BATCH NORM SETTINGS
bn_offset = 0.0
bn_scale = 1.0
# DROPOUT
conv_dropout = tf.cond(is_training, lambda: tf.constant(dropout),
lambda: tf.constant(0.0))
# CONV STACK
with tf.name_scope(name) as scope:
x = conv_layer(x, k=convk, n=n, bias=None, stride=1, name="conv")
x = batchnormC(x,
is_training=is_training,
iteration=global_step,
conv=True,
offset=bn_offset,
scale=bn_scale)
x = tf.nn.dropout(x, keep_prob=1 - conv_dropout)
x = max_pool_layer(x, k=mpk, stride=mpstride, name="maxpool")
x = leaky_relu(x, name="relu")
print_tensor_shape(x, name=scope, verbose=verbose)
return x
def fc_battery(x,
global_step,
n=1024,
bias=0.1,
is_training=False,
dropout=0.0,
winit=None,
verbose=False,
name="FC"):
# BATCH NORM SETTINGS
bn_offset = 0.0
bn_scale = 1.0
# DROPOUT
dropout = tf.cond(is_training, lambda: tf.constant(dropout),
lambda: tf.constant(0.0))
# DEFAULT WEIGHTS INITIALIZATION
if winit is None:
winit = he_weights_initializer() # identity_weights_initializer()
# FC STACK
with tf.name_scope(name) as scope:
x = fc_layer(x, n=n, bias=bias, winit=winit, name="FC")
x = batchnormC(x,
is_training=is_training,
iteration=global_step,
conv=False,
offset=bn_offset,
scale=bn_scale)
x = tf.nn.dropout(x, keep_prob=1 - dropout)
x = leaky_relu(x, rate=0.01, name="relu")
print_tensor_shape(x, name=scope, verbose=verbose)
return x
def model_a(x, is_training, global_step, settings=None, verbose=True):
"""
"""
# BATCH NORM SETTINGS
bn_offset = 0.0
bn_scale = 1.0
# MISC SETTINGS
bval = 0.01 # Bias value
leak = 0.01 # leakiness of leaky relus
# WEIGHTS INITIALIZERS
# st_winit = zero_weights_initializer()
conv_winit = he_weights_initializer()
fc_winit = he_weights_initializer() # identity_weights_initializer()
# DROPOUT SETTINGS
conv_dropout = tf.cond(is_training,
lambda: tf.constant(settings.conv_dropout),
lambda: tf.constant(0.0))
fc_dropout = tf.cond(is_training, lambda: tf.constant(settings.fc_dropout),
lambda: tf.constant(0.0))
# --------------------------------------------------------------------------
# TRUNK
# --------------------------------------------------------------------------
# CONV LAYERS
x = conv_battery(x,
global_step=global_step,
convk=5,
n=48,
mpk=2,
mpstride=2,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=5,
n=64,
mpk=2,
mpstride=1,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=5,
n=128,
mpk=2,
mpstride=2,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=5,
n=160,
mpk=2,
mpstride=1,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=3,
n=192,
mpk=2,
mpstride=2,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=3,
n=192,
mpk=2,
mpstride=1,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=3,
n=192,
mpk=2,
mpstride=2,
is_training=is_training,
verbose=verbose)
x = conv_battery(x,
global_step=global_step,
convk=2,
n=192,
mpk=2,
mpstride=1,
is_training=is_training,
verbose=verbose)
# FC LAYER
x = flatten(x)
print_tensor_shape(x, verbose=verbose)
x = fc_battery(x,
global_step=global_step,
n=1024,
bias=None,
is_training=is_training,
dropout=settings.fc_dropout,
winit=fc_winit,
verbose=verbose,
name="FC")
# --------------------------------------------------------------------------
# DIGIT BRANCHES
# --------------------------------------------------------------------------
max_digits = 5
d = [None] * max_digits
for i in range(max_digits):
d[i] = fc_layer(x,
n=11,
bias=0.1,
winit=fc_winit,
name="branch_{}".format(i + 1))
print_tensor_shape(d[i], verbose=verbose)
digits = tf.stack(d, axis=0, name="digit_logits")
print_tensor_shape(digits, verbose=verbose)
# --------------------------------------------------------------------------
# BBOX BRANCHES
# --------------------------------------------------------------------------
bboxes = fc_layer(x, n=24, bias=0.1, winit=fc_winit, name="bbox_logits")
print_tensor_shape(bboxes, verbose=verbose)
return digits, bboxes
def batchnormC(x, is_training, iteration, conv=False, offset=0.0, scale=1.0):
"""
Given some logits `x`, apply batch normalization to them.
Parameters
----------
x
is_training
iteration
conv: (boolean)(default=False)
Applying it to a convolutional layer?
Returns
-------
Credits
-------
This code is based on code written by Martin Gorner:
- https://github.com/martin-gorner/tensorflow-mnist-tutorial/blob/master/mnist_4.2_batchnorm_convolutional.py
https://www.youtube.com/watch?v=vq2nnJ4g6N0
"""
# adding the iteration prevents from averaging across non-existing iterations
exp_moving_avg = tf.train.ExponentialMovingAverage(0.9999, iteration)
bnepsilon = 1e-5
# calculate mean and variance for batch of logits
if conv:
mean, variance = tf.nn.moments(x, [0, 1, 2])
else:
# mean and variance along the batch
mean, variance = tf.nn.moments(x, [0])
update_moving_averages = exp_moving_avg.apply([mean, variance])
tf.add_to_collection("update_moving_averages", update_moving_averages)
# Mean and Variance (how it get it is dependent on whether it is training)
# TODO: Change the following to use the `is_trianing` directly without logical_not()
# to make it more intuitive.
m = tf.cond(tf.logical_not(is_training),
lambda: exp_moving_avg.average(mean), lambda: mean)
v = tf.cond(tf.logical_not(is_training),
lambda: exp_moving_avg.average(variance), lambda: variance)
# Offset
param_shape = mean.get_shape().as_list()
beta_init = tf.constant_initializer(offset)
beta = tf.Variable(initial_value=beta_init(param_shape), name="beta")
# Scale
gamma_init = tf.constant_initializer(scale)
gamma = tf.Variable(initial_value=gamma_init(param_shape), name="gamma")
# Apply Batch Norm
Ybn = tf.nn.batch_normalization(x,
m,
v,
offset=beta,
scale=gamma,
variance_epsilon=bnepsilon)
return Ybn
