def batch_normalized_conv_layer(state_below, scope_name, n_inputs, n_outputs, filter_shape, stddev, wd, eps=.00001, test=False):
"""
Convolutional layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights", shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
stddev=stddev, wd=wd
)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
# get moments
conv_mean, conv_variance = tf.nn.moments(conv, [0, 1, 2])
# get mean and variance variables
mean = _variable_on_cpu("bn_mean", [n_outputs], tf.constant_initializer(0.0), False)
variance = _variable_on_cpu("bn_variance", [n_outputs], tf.constant_initializer(1.0), False)
# assign the moments
if not test:
assign_mean = mean.assign(conv_mean)
assign_variance = variance.assign(conv_variance)
conv_bn = tf.mul((conv - conv_mean), tf.rsqrt(conv_variance + eps), name=scope.name+"_bn")
else:
conv_bn = tf.mul((conv - mean), tf.rsqrt(variance + eps), name=scope.name+"_bn")
beta = _variable_on_cpu("beta", [n_outputs], tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs], tf.constant_initializer(1.0))
bn = tf.add(tf.mul(conv_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn, .1, name=scope.name, is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(dependencies=[assign_mean, assign_variance], output_tensor=output)
_activation_summary(output)
return output
def batch_normalized_linear_layer(state_below, scope_name, n_inputs, n_outputs, stddev, wd, eps=.00001, test=False):
"""
A linear layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
weight = _variable_with_weight_decay(
"weights", shape=[n_inputs, n_outputs],
stddev=stddev, wd=wd
)
act = tf.matmul(state_below, weight)
# get moments
act_mean, act_variance = tf.nn.moments(act, [0])
# get mean and variance variables
mean = _variable_on_cpu('bn_mean', [n_outputs], tf.constant_initializer(0.0), trainable=False)
variance = _variable_on_cpu('bn_variance', [n_outputs], tf.constant_initializer(1.0), trainable=False)
# assign the moments
if not test:
assign_mean = mean.assign(act_mean)
assign_variance = variance.assign(act_variance)
act_bn = tf.mul((act - act_mean), tf.rsqrt(act_variance + eps), name=scope.name+"_bn")
else:
act_bn = tf.mul((act - mean), tf.rsqrt(variance + eps), name=scope.name+"_bn")
beta = _variable_on_cpu("beta", [n_outputs], tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs], tf.constant_initializer(1.0))
bn = tf.add(tf.mul(act_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn, .1, name=scope.name, is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(dependencies=[assign_mean, assign_variance], output_tensor=output)
_activation_summary(output)
return output
def conv2d_stack(feed,
kernel_list,
stride_list,
padding_list,
batch_norm=False):
if not ((len(kernel_list) == len(stride_list)) and
(len(stride_list) == len(padding_list))):
return
inputs = []
inputs.append(feed)
for i in range(len(kernel_list)):
with tf.variable_scope('conv%d' % (i + 1)) as scope:
kernel = _variable_with_weight_decay('weights',
shape=kernel_list[i],
stddev=5e-2,
wd=None)
conv = conv2d(inputs[-1],
kernel,
stride_list[i],
padding=padding_list[i])
biases = _variable_on_cpu('biases', kernel_list[i][-1],
tf.constant_initializer(0.0))
if batch_norm:
mean, variance = tf.nn.moments(conv, axes=[0])
epsilon = 1e-5
gamma = _variable_on_cpu('gammas', kernel_list[i][-1],
tf.constant_initializer(1.0))
pre_activation = tf.nn.batch_normalization(
conv, mean, variance, biases, gamma, epsilon)
else:
pre_activation = tf.nn.bias_add(conv, biases)
after_activation = tf.nn.relu(pre_activation, name='activated_out')
_activation_summary(after_activation)
inputs.append(after_activation)
return inputs[-1]
def conv2d(input_data,
kernel_height,
kernel_width,
in_channels,
out_channels,
strides=[1, 1, 1, 1],
padding='SAME',
regularization=False,
kernel_summary=False,
name=None):
if regularization is not True:
kernel = ut._variable_with_weight_decay(
'kernels',
[kernel_height, kernel_width, in_channels, out_channels],
tf.truncated_normal_initializer(stddev=5e-2), 0.0)
else:
kernel = ut._variable_with_weight_decay(
'kernels',
[kernel_height, kernel_width, in_channels, out_channels],
tf.truncated_normal_initializer(stddev=5e-2), 0.001)
biases = ut._variable_on_gpu('biases', [out_channels],
tf.constant_initializer(0.0))
conv = tf.nn.conv2d(input_data, kernel, strides, padding)
conv = tf.nn.bias_add(conv, biases)
conv = tf.nn.relu(conv, name=name)
ut._activation_summary(conv)
if kernel_summary is True:
ut._kernel_summary(kernel, name + '/kernel', out_channels,
kernel_width, kernel_height)
return conv
def dense_layer(feed,
input_dim,
output_dim,
dropout=False,
keep_prob=None,
batch_norm=False,
weight_decay=None):
weights = _variable_with_weight_decay('weights',
shape=[input_dim, output_dim],
stddev=0.04,
wd=weight_decay)
biases = _variable_on_cpu('biases', [output_dim],
tf.constant_initializer(0.1))
intermediate = tf.matmul(feed, weights)
if batch_norm:
mean, variance = tf.nn.moments(intermediate, axes=[0])
epsilon = 1e-5
gamma = _variable_on_cpu('gammas', [output_dim],
tf.constant_initializer(1.0))
pre_activation = tf.nn.batch_normalization(intermediate, mean,
variance, biases, gamma,
epsilon)
else:
pre_activation = intermediate + biases
if dropout:
pre_activation = tf.nn.dropout(pre_activation,
keep_prob=keep_prob,
name="dropout")
after_activation = tf.nn.relu(pre_activation, name='activated_out')
_activation_summary(after_activation)
return after_activation
def linear_layer(state_below,
scope_name,
n_inputs,
n_outputs,
stddev,
wd,
use_nonlinearity=True):
"""
Standard linear neural network layer
"""
with tf.variable_scope(scope_name) as scope:
weights = _variable_with_weight_decay('weights', [n_inputs, n_outputs],
stddev=stddev,
wd=wd)
biases = _variable_on_cpu('biases', [n_outputs],
tf.constant_initializer(0.0))
activation = tf.nn.xw_plus_b(state_below,
weights,
biases,
name="activation")
if use_nonlinearity:
output = tf.nn.relu(activation, name=scope.name)
else:
output = activation
_activation_summary(output)
return output
def batch_normalized_conv_layer(state_below,
scope_name,
n_inputs,
n_outputs,
filter_shape,
stddev,
wd,
eps=.00001,
test=False):
"""
Convolutional layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights",
shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
stddev=stddev,
wd=wd)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
# get moments
conv_mean, conv_variance = tf.nn.moments(conv, [0, 1, 2])
# get mean and variance variables
mean = _variable_on_cpu("bn_mean", [n_outputs],
tf.constant_initializer(0.0), False)
variance = _variable_on_cpu("bn_variance", [n_outputs],
tf.constant_initializer(1.0), False)
# assign the moments
if not test:
assign_mean = mean.assign(conv_mean)
assign_variance = variance.assign(conv_variance)
conv_bn = tf.mul((conv - conv_mean),
tf.rsqrt(conv_variance + eps),
name=scope.name + "_bn")
else:
conv_bn = tf.mul((conv - mean),
tf.rsqrt(variance + eps),
name=scope.name + "_bn")
beta = _variable_on_cpu("beta", [n_outputs],
tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs],
tf.constant_initializer(1.0))
bn = tf.add(tf.mul(conv_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn,
.1,
name=scope.name,
is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(
dependencies=[assign_mean, assign_variance],
output_tensor=output)
_activation_summary(output)
return output
def batch_normalized_linear_layer(state_below,
scope_name,
n_inputs,
n_outputs,
stddev,
wd,
eps=.00001,
test=False):
"""
A linear layer with batch normalization
"""
with tf.variable_scope(scope_name) as scope:
weight = _variable_with_weight_decay("weights",
shape=[n_inputs, n_outputs],
stddev=stddev,
wd=wd)
act = tf.matmul(state_below, weight)
# get moments
act_mean, act_variance = tf.nn.moments(act, [0])
# get mean and variance variables
mean = _variable_on_cpu('bn_mean', [n_outputs],
tf.constant_initializer(0.0),
trainable=False)
variance = _variable_on_cpu('bn_variance', [n_outputs],
tf.constant_initializer(1.0),
trainable=False)
# assign the moments
if not test:
assign_mean = mean.assign(act_mean)
assign_variance = variance.assign(act_variance)
act_bn = tf.mul((act - act_mean),
tf.rsqrt(act_variance + eps),
name=scope.name + "_bn")
else:
act_bn = tf.mul((act - mean),
tf.rsqrt(variance + eps),
name=scope.name + "_bn")
beta = _variable_on_cpu("beta", [n_outputs],
tf.constant_initializer(0.0))
gamma = _variable_on_cpu("gamma", [n_outputs],
tf.constant_initializer(1.0))
bn = tf.add(tf.mul(act_bn, gamma), beta)
# output = tf.nn.relu(bn, name=scope.name)
output = randomized_relu(bn,
.1,
name=scope.name,
is_training=(not test))
if not test:
output = control_flow_ops.with_dependencies(
dependencies=[assign_mean, assign_variance],
output_tensor=output)
_activation_summary(output)
return output
def fc(input_data, in_channels, out_channels, regularization=True, name=None):
if regularization is True:
weights = ut._variable_with_weight_decay(
'weights', [in_channels, out_channels],
tf.truncated_normal_initializer(stddev=0.05), 0.001)
else:
weights = ut._variable_with_weight_decay(
'weights', [in_channels, out_channels],
tf.truncated_normal_initializer(stddev=0.05), 0.0)
biases = ut._variable_on_gpu('biases', [out_channels],
tf.constant_initializer(0.1))
fc = tf.nn.relu(tf.matmul(input_data, weights) + biases, name=name)
ut._activation_summary(fc)
return fc
def conv_layer(state_below, scope_name, n_inputs, n_outputs, filter_shape, stddev, wd):
"""
A Standard convolutional layer
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights", shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
wd=wd
)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu("biases", [n_outputs], tf.constant_initializer(0.0))
bias = tf.add(conv, biases)
output = tf.nn.relu(bias, name=scope.name)
_activation_summary(output)
return output
def linear_layer(state_below, scope_name, n_inputs, n_outputs, stddev, wd):
"""
Standard linear neural network layer
"""
with tf.variable_scope(scope_name) as scope:
weights = _variable_with_weight_decay(
'weights', [n_inputs, n_outputs],
stddev=stddev, wd=wd
)
biases = _variable_on_cpu(
'biases', [n_outputs], tf.constant_initializer(0.0)
)
output = tf.nn.xw_plus_b(state_below, weights, biases, name=scope.name)
_activation_summary(output)
return output
def conv_layer(state_below, scope_name, n_inputs, n_outputs, filter_shape,
stddev, wd):
"""
A Standard convolutional layer
"""
with tf.variable_scope(scope_name) as scope:
kernel = _variable_with_weight_decay(
"weights",
shape=[filter_shape[0], filter_shape[1], n_inputs, n_outputs],
wd=wd)
conv = tf.nn.conv2d(state_below, kernel, [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu("biases", [n_outputs],
tf.constant_initializer(0.0))
bias = tf.add(conv, biases)
output = tf.nn.relu(bias, name=scope.name)
_activation_summary(output)
return output
def inception_v1_module(feed,
feed_dim=256,
map_size=(128, 192, 96, 64),
reduce1x1_size=64,
batch_norm=False):
"""
:param feed:
:param map_size: lists of number of feature maps output by each tower (1x1, 3x3, 5x5, 1x1) inside the Inception module
:param reduce1x1_size: number of feature maps output by each 1×1 convolution that precedes a large convolution
:return:
"""
def conv2d_s1(x, W):
return conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')
def max_pool_3x3_s1(x):
return tf.nn.max_pool(x,
ksize=[1, 3, 3, 1],
strides=[1, 1, 1, 1],
padding='SAME')
# follows input
W_conv_1x1_1 = _variable_with_weight_decay(
'W_conv_1x1_1',
shape=[1, 1, feed_dim, map_size[0]],
stddev=5e-2,
wd=None)
b_conv_1x1_1 = _variable_on_cpu('b_conv_1x1_1', [map_size[0]],
tf.constant_initializer(0.0))
# follows input
W_conv_1x1_2 = _variable_with_weight_decay(
'W_conv_1x1_2',
shape=[1, 1, feed_dim, reduce1x1_size],
stddev=5e-2,
wd=None)
b_conv_1x1_2 = _variable_on_cpu('b_conv_1x1_2', [reduce1x1_size],
tf.constant_initializer(0.0))
# follows input
W_conv_1x1_3 = _variable_with_weight_decay(
'W_conv_1x1_3',
shape=[1, 1, feed_dim, reduce1x1_size],
stddev=5e-2,
wd=None)
b_conv_1x1_3 = _variable_on_cpu('b_conv_1x1_3', [reduce1x1_size],
tf.constant_initializer(0.0))
# follows 1x1_2
# attention to the shape paras!!!!
W_conv_3x3 = _variable_with_weight_decay(
'W_conv_3x3',
shape=[3, 3, reduce1x1_size, map_size[1]],
stddev=5e-2,
wd=None)
b_conv_3x3 = _variable_on_cpu('b_conv_3x3', [map_size[1]],
tf.constant_initializer(0.0))
# follows 1x1_3
W_conv_5x5 = _variable_with_weight_decay(
'W_conv_5x5',
shape=[5, 5, reduce1x1_size, map_size[2]],
stddev=5e-2,
wd=None)
b_conv_5x5 = _variable_on_cpu('b_conv_5x5', [map_size[2]],
tf.constant_initializer(0.0))
# follows max pooling
W_conv_1x1_4 = _variable_with_weight_decay(
'W_conv_1x1_4',
shape=[1, 1, feed_dim, map_size[3]],
stddev=5e-2,
wd=None)
b_conv_1x1_4 = _variable_on_cpu('b_conv_1x1_4', [map_size[3]],
tf.constant_initializer(0.0))
# Inception Module
conv_1x1_1 = conv2d_s1(feed, W_conv_1x1_1) + b_conv_1x1_1
conv_1x1_2 = tf.nn.relu(conv2d_s1(feed, W_conv_1x1_2) + b_conv_1x1_2)
conv_1x1_3 = tf.nn.relu(conv2d_s1(feed, W_conv_1x1_3) + b_conv_1x1_3)
conv_3x3 = conv2d_s1(conv_1x1_2, W_conv_3x3) + b_conv_3x3
conv_5x5 = conv2d_s1(conv_1x1_3, W_conv_5x5) + b_conv_5x5
maxpool1 = max_pool_3x3_s1(feed)
conv_1x1_4 = conv2d_s1(maxpool1, W_conv_1x1_4) + b_conv_1x1_4
# concatenate all the feature maps and hit them with a relu
concat = tf.concat([conv_1x1_1, conv_3x3, conv_5x5, conv_1x1_4], 3)
if batch_norm:
biases = _variable_on_cpu('biases', sum(map_size),
tf.constant_initializer(0.0))
mean, variance = tf.nn.moments(concat, axes=[0])
epsilon = 1e-5
gamma = _variable_on_cpu('gammas', sum(map_size),
tf.constant_initializer(1.0))
pre_activation = tf.nn.batch_normalization(concat, mean, variance,
biases, gamma, epsilon)
else:
pre_activation = concat
after_activation = tf.nn.relu(pre_activation, name='activated_out')
_activation_summary(after_activation)
return after_activation
def inference(examples):
# CONV1
with tf.variable_scope('conv1') as scope:
# conv weights [filter_height, filter_width, filter_depth, num_filters]
kernel = _variable('weights', [CONV1_HEIGHT, CONV1_WIDTH, 1, CONV1_FILTERS], tf.contrib.layers.xavier_initializer_conv2d())
biases = _variable('biases', [CONV1_FILTERS], tf.constant_initializer(0.1))
conv = conv2d(examples, kernel)
conv1 = tf.nn.relu(conv + biases, name=scope.name)
_activation_summary(conv1)
# pool1 dim: [n, time, freq after pooling, num_filters]
pool1 = tf.nn.max_pool(conv1, ksize=[1, POOL1_HEIGHT, POOL1_WIDTH, 1],
strides=[1, POOL1_STRIDE_HEIGHT, POOL1_STRIDE_WIDTH, 1], padding='SAME', name='pool1')
## TODO: add batch norm 1 here
batch_norm1_object = BatchNorm(name='batch_norm1', shape=[CONV1_FILTERS])
batch_norm1 = batch_norm1_object(pool1)
# CONV2
with tf.variable_scope('conv2') as scope:
kernel = _variable('weights', [CONV2_HEIGHT, CONV2_WIDTH, CONV1_FILTERS, CONV2_FILTERS], tf.contrib.layers.xavier_initializer_conv2d())
biases = _variable('biases', [CONV2_FILTERS], tf.constant_initializer(0.1))
conv = conv2d(batch_norm1, kernel)
conv2 = tf.nn.relu(conv + biases, name=scope.name)
_activation_summary(conv2)
# POOL2
pool2 = tf.nn.max_pool(conv2, ksize=[1, POOL2_HEIGHT, POOL2_WIDTH, 1],
strides=[1, POOL2_STRIDE_HEIGHT, POOL2_STRIDE_WIDTH, 1], padding='SAME', name='pool2')
## TODO: add batch norm 2 here
batch_norm2_object = BatchNorm(name='batch_norm2', shape=[CONV2_FILTERS])
batch_norm2 = batch_norm2_object(pool2)
# FC3
with tf.variable_scope('fc3') as scope:
reshape = tf.reshape(batch_norm2, [BATCH_SIZE, -1])
dim = (DIM_TIME/POOL1_HEIGHT/POOL2_HEIGHT) * (DIM_FREQ/POOL1_WIDTH/POOL2_WIDTH) * CONV2_FILTERS
weights = _variable('weights', [dim, FC3_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC3_SIZE], tf.constant_initializer(0.1))
fc3 = tf.nn.relu(tf.matmul(reshape, weights) + biases, name=scope.name)
_activation_summary(fc3)
# FC4
with tf.variable_scope('fc4') as scope:
weights = _variable('weights', [FC3_SIZE, FC4_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC4_SIZE], tf.constant_initializer(0.1))
fc4 = tf.nn.relu(tf.matmul(fc3, weights) + biases, name=scope.name)
_activation_summary(fc4)
# FC5
with tf.variable_scope('fc5') as scope:
weights = _variable('weights', [FC4_SIZE, FC5_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC5_SIZE], tf.constant_initializer(0.1))
fc5 = tf.nn.relu(tf.matmul(fc4, weights) + biases, name=scope.name)
_activation_summary(fc5)
# FC6
with tf.variable_scope('fc6') as scope:
weights = _variable('weights', [FC5_SIZE, FC6_SIZE], tf.contrib.layers.xavier_initializer(), wd=config.fc_wd)
biases = _variable('biases', [FC6_SIZE], tf.constant_initializer(0.1))
fc6 = tf.nn.relu(tf.matmul(fc5, weights) + biases, name=scope.name)
_activation_summary(fc6)
# softmax
with tf.variable_scope('softmax_linear') as scope:
weights = _variable('weights', [FC6_SIZE, NUM_CLASSES], tf.contrib.layers.xavier_initializer())
biases = _variable('biases', [NUM_CLASSES], tf.constant_initializer(0.0))
# shape of y_conv is (N,3)
softmax_linear = tf.add(tf.matmul(fc6, weights), biases, name=scope.name)
_activation_summary(softmax_linear)
return softmax_linear