def simple_global_bn(inp, name): """Global batch normalization This tensor is nomalized by the global mean of the input tensor along the last axis. Args: inp : A 4D-Tensor. name (str): Name of the operation. Returns: global batch normalized tensor. """ ksize = inp.get_shape().as_list() ksize = [ksize[-1]] mean, variance = tf.nn.moments(inp, [0, 1, 2], name=name + '_moments') scale = _variable_on_cpu( name + "_scale", shape=ksize, initializer=tf.contrib.layers.variance_scaling_initializer()) offset = _variable_on_cpu( name + "_offset", shape=ksize, initializer=tf.contrib.layers.variance_scaling_initializer()) return tf.nn.batch_normalization(inp, mean=mean, variance=variance, scale=scale, offset=offset, variance_epsilon=1e-5)
def rnn_layers_one_direction(x, seq_length, training, hidden_num=200, layer_num=3, class_n=5, dtype=tf.float32): """Create a single direction RNN layer Args: x (Float): Input 3D-Tensor of shape [batch_size, max_time, channel] seq_length (Int): A 1D-Tensor of shape [batch_size], real length of each sequence. training (Boolean): A 0D-Tenosr indicate if it's in training. hidden_num (int, optional): Defaults to 200. Size of the hidden state. layer_num (int, optional): Defaults to 3. Number of layers in RNN. class_n (int, optional): Defaults to 5. Number of output class. Returns: logits: A 3D Tensor of shape [batch_size, max_time, class_n] """ cells = list() for i in range(layer_num): cell = BNLSTMCell(hidden_num, training) cells.append(cell) cell_wrap = tf.contrib.rnn.MultiRNNCell(cells) with tf.variable_scope('LSTM_rnn') as scope: lasth, _ = tf.nn.dynamic_rnn(cell_wrap, x, sequence_length=seq_length, dtype=tf.float32, scope=scope) # shape of lasth [batch_size,max_time,hidden_num*2] batch_size = lasth.get_shape().as_list()[0][type] max_time = lasth.get_shape().as_list()[1] with tf.variable_scope('rnn_fnn_layer'): weight_class = _variable_on_cpu( name='weights_class', shape=[hidden_num, class_n], initializer=tf.truncated_normal_initializer( stddev=np.sqrt(2.0 / hidden_num)), dtype=dtype) bias_class = _variable_on_cpu(name='bias_class', shape=[class_n], initializer=tf.zeros_initializer(), dtype=dtype) lasth_rs = tf.reshape(lasth, [batch_size * max_time, hidden_num], name='lasth_rs') logits = tf.reshape(tf.nn.bias_add(tf.matmul(lasth_rs, weight_class), bias_class), [batch_size, max_time, class_n], name="rnn_logits_rs") return logits
def rnn_layers(x, seq_length, training, hidden_num=100, layer_num=3, class_n=5, cell='LSTM', dtype=tf.float32): """Generate RNN layers. Args: x (Float): A 3D-Tensor of shape [batch_size,max_time,channel] seq_length (Int): A 1D-Tensor of shape [batch_size], real length of each sequence. training (Boolean): A 0D-Tenosr indicate if it's in training. hidden_num (int, optional): Defaults to 100. Size of the hidden state, hidden unit will be deep concatenated, so the final hidden state will be size of 200. layer_num (int, optional): Defaults to 3. Number of layers in RNN. class_n (int, optional): Defaults to 5. Number of output class. cell(str): A String from 'LSTM','GRU','BNLSTM', the RNN Cell used. BNLSTM stand for Batch normalization LSTM Cell. Returns: logits: A 3D Tensor of shape [batch_size, max_time, class_n] """ cells_fw = list() cells_bw = list() for i in range(layer_num): if cell == 'LSTM': cell_fw = LSTMCell(hidden_num) cell_bw = LSTMCell(hidden_num) elif cell == 'GRU': cell_fw = GRUCell(hidden_num) cell_bw = GRUCell(hidden_num) elif cell == 'BNLSTM': cell_fw = BNLSTMCell(hidden_num, training=training) cell_bw = BNLSTMCell(hidden_num, training=training) else: raise ValueError("Cell type unrecognized.") cells_fw.append(cell_fw) cells_bw.append(cell_bw) multi_cells_fw = tf.nn.rnn_cell.MultiRNNCell(cells_fw) multi_cells_bw = tf.nn.rnn_cell.MultiRNNCell(cells_bw) with tf.variable_scope('BDGRU_rnn') as scope: outputs, _ = tf.nn.bidirectional_dynamic_rnn( cell_fw=multi_cells_fw, cell_bw=multi_cells_bw, inputs=x, sequence_length=seq_length, dtype=dtype, scope=scope) lasth = tf.concat(outputs, 2, name='birnn_output_concat') # shape of lasth [batch_size,max_time,hidden_num*2] batch_size = lasth.get_shape().as_list()[0] max_time = lasth.get_shape().as_list()[1] with tf.variable_scope('rnn_fnn_layer'): weight_out = _variable_on_cpu( name='weights', shape=[2, hidden_num], initializer=tf.truncated_normal_initializer( stddev=np.sqrt(2.0 / (2 * hidden_num))), dtype=dtype) biases_out = _variable_on_cpu(name='bias', shape=[hidden_num], initializer=tf.zeros_initializer(), dtype=dtype) weight_class = _variable_on_cpu( name='weights_class', shape=[hidden_num, class_n], initializer=tf.truncated_normal_initializer( stddev=np.sqrt(2.0 / hidden_num)), dtype=dtype) bias_class = _variable_on_cpu(name='bias_class', shape=[class_n], initializer=tf.zeros_initializer(), dtype=dtype) lasth_rs = tf.reshape(lasth, [batch_size, max_time, 2, hidden_num], name='lasth_rs') lasth_output = tf.nn.bias_add(tf.reduce_sum(tf.multiply( lasth_rs, weight_out), axis=2), biases_out, name='lasth_bias_add') lasth_output_rs = tf.reshape(lasth_output, [batch_size * max_time, hidden_num], name='lasto_rs') logits = tf.reshape(tf.nn.bias_add( tf.matmul(lasth_output_rs, weight_class), bias_class), [batch_size, max_time, class_n], name="rnn_logits_rs") return logits
def conv_layer(indata, ksize, padding, training, name, dilate=1, strides=None, bias_term=False, active=True, BN=True, active_function='relu',wd = None): """A convolutional layer Args: indata: A input 4D-Tensor of shape [batch_size, Height, Width, Channel]. ksize: A length 4 list. padding: A String from: "SAME","VALID" training: Scalar Tensor of type boolean, indicate if in training or not. name: A String give the name of this layer, other variables and options created in this layer will have this name as prefix. dilate (int, optional): Defaults to 1. Dilation of the width. strides (list, optional): Defaults to [1, 1, 1, 1]. A list of length 4. bias_term (bool, optional): Defaults to False. If True, a bais Tensor is added. active (bool, optional): Defaults to True. If True, output is activated by a activation function. BN (bool, optional): Defaults to True. If True, batch normalization will be applied. active_function (str, optional): Defaults to 'relu'. A String from 'relu','sigmoid','tanh'. wd: weight decay, if None no weight decay will be added. Returns: conv_out: A output 4D-Tensor. """ if strides is None: strides = [1, 1, 1, 1] else: if type(strides) is int: strides = [1,strides,1,1] with tf.variable_scope(name): W = _variable_with_weight_decay("weights", shape=ksize, wd=wd, initializer = tf.contrib.layers.xavier_initializer(uniform = False, )) if bias_term: b = _variable_on_cpu("bias", shape=[ksize[-1]], initializer = tf.constant_initializer(0.0)) if dilate > 1: if bias_term: conv_out = b + tf.nn.convolution(input=indata, filter=W, dilation_rate=np.asarray([1, dilate]), padding=padding, name=name) else: conv_out = tf.nn.convolution(input=indata, filter=W, dilation_rate=np.asarray([1, dilate]), padding=padding, name=name) else: if bias_term: conv_out = b + \ tf.nn.conv2d(indata, W, strides=strides, padding=padding, name=name) else: conv_out = tf.nn.conv2d( indata, W, strides=strides, padding=padding, name=name) if BN: with tf.variable_scope(name + '_bn') as scope: #conv_out = batchnorm(conv_out,scope=scope,training = training) conv_out = simple_global_bn(conv_out, name=name + '_bn') #conv_out = tf.layers.batch_normalization(conv_out,axis = -1,training = training,name = 'bn') if active: if active_function == 'relu': with tf.variable_scope(name + '_relu'): conv_out = tf.nn.relu(conv_out, name='relu') elif active_function == 'sigmoid': with tf.variable_scope(name + '_sigmoid'): conv_out = tf.sigmoid(conv_out, name='sigmoid') elif active_function == 'tanh': with tf.variable_scope(name + '_tanh'): conv_out = tf.tanh(conv_out, name='tanh') return conv_out