def simple_global_bn(inp, name):
"""Global batch normalization
This tensor is nomalized by the global mean of the input tensor along the last axis.
Args:
inp : A 4D-Tensor.
name (str): Name of the operation.
Returns:
global batch normalized tensor.
"""
ksize = inp.get_shape().as_list()
ksize = [ksize[-1]]
mean, variance = tf.nn.moments(inp, [0, 1, 2], name=name + '_moments')
scale = _variable_on_cpu(
name + "_scale",
shape=ksize,
initializer=tf.contrib.layers.variance_scaling_initializer())
offset = _variable_on_cpu(
name + "_offset",
shape=ksize,
initializer=tf.contrib.layers.variance_scaling_initializer())
return tf.nn.batch_normalization(inp,
mean=mean,
variance=variance,
scale=scale,
offset=offset,
variance_epsilon=1e-5)
def rnn_layers_one_direction(x,
seq_length,
training,
hidden_num=200,
layer_num=3,
class_n=5,
dtype=tf.float32):
"""Create a single direction RNN layer
Args:
x (Float): Input 3D-Tensor of shape [batch_size, max_time, channel]
seq_length (Int): A 1D-Tensor of shape [batch_size], real length of each sequence.
training (Boolean): A 0D-Tenosr indicate if it's in training.
hidden_num (int, optional): Defaults to 200. Size of the hidden state.
layer_num (int, optional): Defaults to 3. Number of layers in RNN.
class_n (int, optional): Defaults to 5. Number of output class.
Returns:
logits: A 3D Tensor of shape [batch_size, max_time, class_n]
"""
cells = list()
for i in range(layer_num):
cell = BNLSTMCell(hidden_num, training)
cells.append(cell)
cell_wrap = tf.contrib.rnn.MultiRNNCell(cells)
with tf.variable_scope('LSTM_rnn') as scope:
lasth, _ = tf.nn.dynamic_rnn(cell_wrap,
x,
sequence_length=seq_length,
dtype=tf.float32,
scope=scope)
# shape of lasth [batch_size,max_time,hidden_num*2]
batch_size = lasth.get_shape().as_list()[0][type]
max_time = lasth.get_shape().as_list()[1]
with tf.variable_scope('rnn_fnn_layer'):
weight_class = _variable_on_cpu(
name='weights_class',
shape=[hidden_num, class_n],
initializer=tf.truncated_normal_initializer(
stddev=np.sqrt(2.0 / hidden_num)),
dtype=dtype)
bias_class = _variable_on_cpu(name='bias_class',
shape=[class_n],
initializer=tf.zeros_initializer(),
dtype=dtype)
lasth_rs = tf.reshape(lasth, [batch_size * max_time, hidden_num],
name='lasth_rs')
logits = tf.reshape(tf.nn.bias_add(tf.matmul(lasth_rs, weight_class),
bias_class),
[batch_size, max_time, class_n],
name="rnn_logits_rs")
return logits
def rnn_layers(x,
seq_length,
training,
hidden_num=100,
layer_num=3,
class_n=5,
cell='LSTM',
dtype=tf.float32):
"""Generate RNN layers.
Args:
x (Float): A 3D-Tensor of shape [batch_size,max_time,channel]
seq_length (Int): A 1D-Tensor of shape [batch_size], real length of each sequence.
training (Boolean): A 0D-Tenosr indicate if it's in training.
hidden_num (int, optional): Defaults to 100. Size of the hidden state,
hidden unit will be deep concatenated, so the final hidden state will be size of 200.
layer_num (int, optional): Defaults to 3. Number of layers in RNN.
class_n (int, optional): Defaults to 5. Number of output class.
cell(str): A String from 'LSTM','GRU','BNLSTM', the RNN Cell used.
BNLSTM stand for Batch normalization LSTM Cell.
Returns:
logits: A 3D Tensor of shape [batch_size, max_time, class_n]
"""
cells_fw = list()
cells_bw = list()
for i in range(layer_num):
if cell == 'LSTM':
cell_fw = LSTMCell(hidden_num)
cell_bw = LSTMCell(hidden_num)
elif cell == 'GRU':
cell_fw = GRUCell(hidden_num)
cell_bw = GRUCell(hidden_num)
elif cell == 'BNLSTM':
cell_fw = BNLSTMCell(hidden_num, training=training)
cell_bw = BNLSTMCell(hidden_num, training=training)
else:
raise ValueError("Cell type unrecognized.")
cells_fw.append(cell_fw)
cells_bw.append(cell_bw)
multi_cells_fw = tf.nn.rnn_cell.MultiRNNCell(cells_fw)
multi_cells_bw = tf.nn.rnn_cell.MultiRNNCell(cells_bw)
with tf.variable_scope('BDGRU_rnn') as scope:
outputs, _ = tf.nn.bidirectional_dynamic_rnn(
cell_fw=multi_cells_fw,
cell_bw=multi_cells_bw,
inputs=x,
sequence_length=seq_length,
dtype=dtype,
scope=scope)
lasth = tf.concat(outputs, 2, name='birnn_output_concat')
# shape of lasth [batch_size,max_time,hidden_num*2]
batch_size = lasth.get_shape().as_list()[0]
max_time = lasth.get_shape().as_list()[1]
with tf.variable_scope('rnn_fnn_layer'):
weight_out = _variable_on_cpu(
name='weights',
shape=[2, hidden_num],
initializer=tf.truncated_normal_initializer(
stddev=np.sqrt(2.0 / (2 * hidden_num))),
dtype=dtype)
biases_out = _variable_on_cpu(name='bias',
shape=[hidden_num],
initializer=tf.zeros_initializer(),
dtype=dtype)
weight_class = _variable_on_cpu(
name='weights_class',
shape=[hidden_num, class_n],
initializer=tf.truncated_normal_initializer(
stddev=np.sqrt(2.0 / hidden_num)),
dtype=dtype)
bias_class = _variable_on_cpu(name='bias_class',
shape=[class_n],
initializer=tf.zeros_initializer(),
dtype=dtype)
lasth_rs = tf.reshape(lasth, [batch_size, max_time, 2, hidden_num],
name='lasth_rs')
lasth_output = tf.nn.bias_add(tf.reduce_sum(tf.multiply(
lasth_rs, weight_out),
axis=2),
biases_out,
name='lasth_bias_add')
lasth_output_rs = tf.reshape(lasth_output,
[batch_size * max_time, hidden_num],
name='lasto_rs')
logits = tf.reshape(tf.nn.bias_add(
tf.matmul(lasth_output_rs, weight_class), bias_class),
[batch_size, max_time, class_n],
name="rnn_logits_rs")
return logits
def conv_layer(indata, ksize, padding, training, name, dilate=1, strides=None, bias_term=False, active=True,
BN=True, active_function='relu',wd = None):
"""A convolutional layer
Args:
indata: A input 4D-Tensor of shape [batch_size, Height, Width, Channel].
ksize: A length 4 list.
padding: A String from: "SAME","VALID"
training: Scalar Tensor of type boolean, indicate if in training or not.
name: A String give the name of this layer, other variables and options created in this layer will have this name as prefix.
dilate (int, optional): Defaults to 1. Dilation of the width.
strides (list, optional): Defaults to [1, 1, 1, 1]. A list of length 4.
bias_term (bool, optional): Defaults to False. If True, a bais Tensor is added.
active (bool, optional): Defaults to True. If True, output is activated by a activation function.
BN (bool, optional): Defaults to True. If True, batch normalization will be applied.
active_function (str, optional): Defaults to 'relu'. A String from 'relu','sigmoid','tanh'.
wd: weight decay, if None no weight decay will be added.
Returns:
conv_out: A output 4D-Tensor.
"""
if strides is None:
strides = [1, 1, 1, 1]
else:
if type(strides) is int:
strides = [1,strides,1,1]
with tf.variable_scope(name):
W = _variable_with_weight_decay("weights",
shape=ksize,
wd=wd,
initializer = tf.contrib.layers.xavier_initializer(uniform = False, ))
if bias_term:
b = _variable_on_cpu("bias",
shape=[ksize[-1]],
initializer = tf.constant_initializer(0.0))
if dilate > 1:
if bias_term:
conv_out = b + tf.nn.convolution(input=indata, filter=W, dilation_rate=np.asarray([1, dilate]),
padding=padding, name=name)
else:
conv_out = tf.nn.convolution(input=indata, filter=W, dilation_rate=np.asarray([1, dilate]),
padding=padding, name=name)
else:
if bias_term:
conv_out = b + \
tf.nn.conv2d(indata, W, strides=strides,
padding=padding, name=name)
else:
conv_out = tf.nn.conv2d(
indata, W, strides=strides, padding=padding, name=name)
if BN:
with tf.variable_scope(name + '_bn') as scope:
#conv_out = batchnorm(conv_out,scope=scope,training = training)
conv_out = simple_global_bn(conv_out, name=name + '_bn')
#conv_out = tf.layers.batch_normalization(conv_out,axis = -1,training = training,name = 'bn')
if active:
if active_function == 'relu':
with tf.variable_scope(name + '_relu'):
conv_out = tf.nn.relu(conv_out, name='relu')
elif active_function == 'sigmoid':
with tf.variable_scope(name + '_sigmoid'):
conv_out = tf.sigmoid(conv_out, name='sigmoid')
elif active_function == 'tanh':
with tf.variable_scope(name + '_tanh'):
conv_out = tf.tanh(conv_out, name='tanh')
return conv_out
