def conv_layer(x,
k=3,
n=8,
stride=1,
bias=0.1,
winit=he_weights_initializer(),
pad='SAME',
name="conv_layer"):
num_filters_in = int(x.get_shape()[-1]) # The number of chanels coming in
with tf.name_scope(name) as scope:
weights = tf.Variable(winit([k, k, num_filters_in, n]),
name="weights",
dtype=tf.float32)
if bias is not None:
conv = tf.nn.conv2d(x,
weights,
strides=[1, stride, stride, 1],
padding=pad,
name="conv")
bias = tf.Variable(tf.constant(bias, shape=[n]), name="bias")
preactivation = tf.add(conv, bias, name=scope)
else:
preactivation = tf.nn.conv2d(x,
weights,
strides=[1, stride, stride, 1],
padding=pad,
name=scope)
return preactivation
def fc_layer(x,
n=32,
bias=0.1,
winit=he_weights_initializer(),
dtype=tf.float32,
name="fc_layer"):
"""Fully Connected Layer
Args:
x: The tensor from the previous layer.
n: The number of nodes for this layer (width).
bias: (float or None)(default=0.1)
Initial value for bias units.
If None, then no Biases will be used.
winit: weights initializer
name: Name for this operation
"""
in_nodes = int(x.get_shape()[-1]) # The number of nodes from input x
with tf.name_scope(name) as scope:
weights = tf.Variable(winit([in_nodes, n]),
name="weights",
dtype=dtype)
if bias is not None:
bias = tf.Variable(tf.constant(bias, shape=[n], dtype=dtype),
name="bias")
preactivation = tf.add(tf.matmul(x, weights), bias, name=scope)
else:
preactivation = tf.matmul(x, weights, name=scope)
return preactivation
def xavier_weights(shape, name="weights"):
# print("creating XAVIER_WEIGHTS")
shape = list(shape)
receptive_field_size = np.prod(shape[:2])
fanin = shape[-2] # * receptive_field_size
fanout = shape[-1]
# print("WEIGHTS: shape: {} Fanin {} fanout {} receptive field {}".format(shape, fanin, fanout, receptive_field_size))
#W = np.random.randn(fanin, fanout) / np.sqrt(fanin)
W = tf.truncated_normal(shape) / np.sqrt(fanin)
return tf.Variable(W, dtype=tf.float32, name=name)
def create_graph(logit_func, settings):
""" Creates a Tensorflow graph for the multi-digit classification + bounding
box task.
Args:
logit_func: (function) A function that returns two tensors:
- digit_logits
- bbox_logits
settings: (object) A Settings object that contains attribute values
for the model.
Returns:
(tensorflow graph)
"""
print_headers("GRAPH", border="=", width=PRINT_WIDTH)
graph = tf.Graph()
with graph.as_default():
# PLACEHOLDERS
X = tf.placeholder(tf.float32, shape=[None, 54, 54],
name="X") # Images
Y = tf.placeholder(tf.int32, shape=[None, 5], name="Y") # Digits
BBOX = tf.placeholder(tf.float32, shape=[None, 24],
name="BBOX") # Bboxes
# OPTIONAL PLACEHOLDERS
alpha = tf.placeholder_with_default(0.001, shape=None, name="alpha")
is_training = tf.placeholder_with_default(False,
shape=None,
name="is_training")
# VARIABLES
global_step = tf.Variable(0, name='global_step', trainable=False)
# PREPROCESS
x = X / 255.0 # Rescale values to be 0-1
x = tf.reshape(x, shape=[-1, 54, 54, 1]) # Reshape for Conv Layers
print("x after reshaping to 4D: ", x.get_shape().as_list())
# MODEL
digit_logits, bbox_logits = logit_func(x=x,
is_training=is_training,
settings=settings,
global_step=global_step)
# BBOX LOSS
bbox_loss = tf.sqrt(tf.reduce_mean(tf.square(1 *
(bbox_logits - BBOX))),
name="bbox_loss")
# DIGITS LOSS
digits_loss = multi_digit_loss(logits_list=digit_logits,
Y=Y,
max_digits=5,
name="digit_loss")
# TOTAL LOSS
loss = tf.add(bbox_loss, digits_loss, name="loss")
# TRAIN
train = trainer(loss,
alpha=alpha,
global_step=global_step,
name="train")
# PREDICTIONS
digit_preds = tf.transpose(tf.argmax(digit_logits, dimension=2))
digit_preds = tf.to_int32(digit_preds, name="digit_preds")
return graph
def std_weights(shape, std=0.01, name="weights"):
# print("creating STD_WEIGHTS")
shape = list(shape)
return tf.Variable(tf.truncated_normal(shape, stddev=std), name=name)
def noisy_identity_weights(shape, noise=0.0001, name="weights"):
#return he_normal_weights(shape, name="name")
# print("creating NOISY_IDENTITY_WEIGHTS")
shape = list(shape)
noise = noise * np.random.randn(*shape)
return tf.Variable(noise + np.eye(*shape), dtype=tf.float32, name=name)
def batchnormC(x, is_training, iteration, conv=False, offset=0.0, scale=1.0):
"""
Given some logits `x`, apply batch normalization to them.
Parameters
----------
x
is_training
iteration
conv: (boolean)(default=False)
Applying it to a convolutional layer?
Returns
-------
Credits
-------
This code is based on code written by Martin Gorner:
- https://github.com/martin-gorner/tensorflow-mnist-tutorial/blob/master/mnist_4.2_batchnorm_convolutional.py
https://www.youtube.com/watch?v=vq2nnJ4g6N0
"""
# adding the iteration prevents from averaging across non-existing iterations
exp_moving_avg = tf.train.ExponentialMovingAverage(0.9999, iteration)
bnepsilon = 1e-5
# calculate mean and variance for batch of logits
if conv:
mean, variance = tf.nn.moments(x, [0, 1, 2])
else:
# mean and variance along the batch
mean, variance = tf.nn.moments(x, [0])
update_moving_averages = exp_moving_avg.apply([mean, variance])
tf.add_to_collection("update_moving_averages", update_moving_averages)
# Mean and Variance (how it get it is dependent on whether it is training)
# TODO: Change the following to use the `is_trianing` directly without logical_not()
# to make it more intuitive.
m = tf.cond(tf.logical_not(is_training),
lambda: exp_moving_avg.average(mean), lambda: mean)
v = tf.cond(tf.logical_not(is_training),
lambda: exp_moving_avg.average(variance), lambda: variance)
# Offset
param_shape = mean.get_shape().as_list()
beta_init = tf.constant_initializer(offset)
beta = tf.Variable(initial_value=beta_init(param_shape), name="beta")
# Scale
gamma_init = tf.constant_initializer(scale)
gamma = tf.Variable(initial_value=gamma_init(param_shape), name="gamma")
# Apply Batch Norm
Ybn = tf.nn.batch_normalization(x,
m,
v,
offset=beta,
scale=gamma,
variance_epsilon=bnepsilon)
return Ybn