def encoder_decoder(self, inputdata): """ LSTM-based encoder-decoder module. """ with tf.variable_scope('LSTMLayers'): [batch_size, width, _] = inputdata.get_shape().as_list() with tf.variable_scope('encoder'): forward_cells = [] backward_cells = [] for _ in range(2): forward_cells.append(tf.nn.rnn_cell.LSTMCell(self.lstm_dim)) backward_cells.append(tf.nn.rnn_cell.LSTMCell(self.lstm_dim)) encoder_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn( forward_cells, backward_cells, inputdata, dtype=tf.float32) with tf.variable_scope('decoder'): forward_cells = [] backward_cells = [] for _ in range(2): forward_cells.append(tf.nn.rnn_cell.LSTMCell(self.lstm_dim)) backward_cells.append(tf.nn.rnn_cell.LSTMCell(self.lstm_dim)) decoder_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn( forward_cells, backward_cells, encoder_layer, dtype=tf.float32) rnn_reshaped = tf.reshape(decoder_layer, [batch_size * width, -1]) logits = slim.fully_connected(rnn_reshaped, self.num_classes, activation_fn=None) logits = tf.reshape(logits, [batch_size, width, self.num_classes]) rnn_out = tf.transpose(logits, (1, 0, 2)) return rnn_out
def BiRNN_new(x, c, l, num_layers, num_hidden, meta_data, num_classes, timesteps, keep_prob, uncertainty, is_train=True, use_sequence_lengths=False, use_embedding=True, sequence_length=66, dict_size=32): if use_embedding: emb = tf.keras.layers.Embedding(dict_size, num_hidden, input_length=sequence_length) x = emb(x) num_layers = num_layers with tf.name_scope("birnn"): #lstm_fw_cell = [tf.contrib.rnn.DropoutWrapper(rnn.BasicLSTMCell(num_hidden), input_keep_prob = keep_prob) for _ in range(num_layers)] #lstm_bw_cell = [tf.contrib.rnn.DropoutWrapper(rnn.BasicLSTMCell(num_hidden), input_keep_prob = keep_prob) for _ in range(num_layers)] #lstm_fw_cell = [rnn.ConvLSTMCell(1,[66,32],128, (5,1)) for _ in range(num_layers)] #lstm_bw_cell = [rnn.ConvLSTMCell(1,[66,32],128, (5,1)) for _ in range(num_layers)] lstm_fw_cell = [ rnn.BasicLSTMCell(num_hidden) for _ in range(num_layers) ] lstm_bw_cell = [ rnn.BasicLSTMCell(num_hidden) for _ in range(num_layers) ] # if use_sequence_lengths: rnn_outputs_all, final_fw, final_bw = rnn.stack_bidirectional_dynamic_rnn( lstm_fw_cell, lstm_bw_cell, x, sequence_length=l, dtype=tf.float32) else: rnn_outputs_all, final_fw, final_bw = rnn.stack_bidirectional_dynamic_rnn( lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) final_fw = final_fw[-1].h final_bw = final_bw[-1].h out = tf.concat([final_fw, final_bw], 1) print(out, final_bw, final_fw) feat = tf.concat([out, c], axis=1) #add another layer ! l1 = tf.contrib.slim.fully_connected(feat, num_hidden) # l1 = tf.layers.dropout(l1, rate=keep_prob, training=is_train) # l2 = tf.contrib.slim.fully_connected(l1, num_hidden) preds = tf.contrib.slim.fully_connected(l1, num_classes, activation_fn=None) # maybe we need to return somethin for attention score return preds, l1, None, None, None, None
def bi_gru(self, inputs, hidden_size, res_add=True): """build the bi-GRU network. Return the encoder represented vector. X_inputs: [batch_size, n_step] n_step: 句子的词数量;或者文档的句子数。 outputs: [batch_size, n_step, hidden_size*2+embedding_size(if res_add)] """ cells_fw = [self.gru_cell(hidden_size) for _ in range(1)] cells_bw = [self.gru_cell(hidden_size) for _ in range(1)] initial_states_fw = [ cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw ] initial_states_bw = [ cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw ] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn( cells_fw, cells_bw, inputs, initial_states_fw=initial_states_fw, initial_states_bw=initial_states_bw, dtype=tf.float32) if res_add: outputs = tf.concat([outputs, inputs], axis=2) return outputs
def BiLSTM(x, is_train): # lstm_fw_cell = rnn.BasicLSTMCell(cfg.hidden_units_num) # lstm_bw_cell = rnn.BasicLSTMCell(cfg.hidden_units_num) stacked_lstm = [] stacked_bw_lstm = [] for i in range(1): lstm_cell = tf.contrib.rnn.BasicLSTMCell(cfg.hidden_units_num) if tensor_util.constant_value(is_train): lstm_cell = tf.contrib.rnn.DropoutWrapper( lstm_cell, output_keep_prob=cfg.dropout) lstm_cell_bw = tf.contrib.rnn.BasicLSTMCell(cfg.hidden_units_num) if tensor_util.constant_value(is_train): lstm_cell_bw = tf.contrib.rnn.DropoutWrapper( lstm_cell_bw, output_keep_prob=cfg.dropout) stacked_lstm.append(lstm_cell) stacked_bw_lstm.append(lstm_cell_bw) # 建立前向和后向的多层LSTM Gmcell = tf.contrib.rnn.MultiRNNCell(stacked_lstm) Gmcell_bw = tf.contrib.rnn.MultiRNNCell(stacked_bw_lstm) outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn([Gmcell], [Gmcell_bw], x, dtype=tf.float32) return outputs
def _sequence_lstm(self, inputdata, name): """ Implements the sequence label part of the network :param inputdata: :param name: :return: """ with tf.variable_scope(name_or_scope=name): # construct stack lstm rcnn layer # forward lstm cell fw_cell_list = [ tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self._hidden_nums] * self._layers_nums ] # Backward direction cells bw_cell_list = [ tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self._hidden_nums] * self._layers_nums ] stack_lstm, _, _ = rnn.stack_bidirectional_dynamic_rnn( fw_cell_list, bw_cell_list, inputdata, dtype=tf.float32) stack_lstm = self.dropout(inputdata=stack_lstm, keep_prob=0.5, is_training=self._is_training, name='sequence_drop_out') return stack_lstm
def BidirectionalLSTM(self, inputdata): """ 创建双向lstm网络 :param inputdata: :return: """ # 多层网络 论文实现为两层256个隐层单元 # 前向cell fw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in [self.hidden_nums, self.hidden_nums]] # 后向 direction cells bw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in [self.hidden_nums, self.hidden_nums]] stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn(fw_cell_list, bw_cell_list, inputdata, dtype=tf.float32) # 添加dropout层 stack_lstm_layer = tf.nn.dropout(stack_lstm_layer, keep_prob=0.5, noise_shape=None, name=None) [batch_s, _, hidden_nums] = inputdata.get_shape().as_list() outputs = tf.reshape(stack_lstm_layer, [-1, hidden_nums]) weights = tf.get_variable(name='W_out', shape=[hidden_nums, self.num_class], dtype=tf.float32, initializer=tf.glorot_uniform_initializer()) # tf.glorot_normal_initializer bias = tf.get_variable(name='b_out', shape=[self.num_class], dtype=tf.float32, initializer=tf.constant_initializer()) # 仿射投影到num_class上 # 未进入到softmax之前的概率 logits = tf.matmul(outputs, weights) + bias logits = tf.reshape(logits, [batch_s, -1, self.num_class]) # 交换batch和轴 转置 raw_pred = tf.argmax(tf.nn.softmax(logits), axis=2, name='raw_prediction') logits = tf.transpose(logits, (1, 0, 2)) return logits, raw_pred
def lstm_layer(inputs, batch_size, num_units, lengths=None, stack_size=1, use_cudnn=False, rnn_dropout_drop_amt=0, is_training=True, bidirectional=True): """Create a LSTM layer using the specified backend.""" if use_cudnn: return cudnn_lstm_layer(inputs, batch_size, num_units, lengths, stack_size, rnn_dropout_drop_amt, is_training, bidirectional) else: assert rnn_dropout_drop_amt == 0 cells_fw = [ contrib_cudnn_rnn.CudnnCompatibleLSTMCell(num_units) for _ in range(stack_size) ] cells_bw = [ contrib_cudnn_rnn.CudnnCompatibleLSTMCell(num_units) for _ in range(stack_size) ] with tf.variable_scope('cudnn_lstm'): (outputs, unused_state_f, unused_state_b) = contrib_rnn.stack_bidirectional_dynamic_rnn( cells_fw, cells_bw, inputs, dtype=tf.float32, sequence_length=lengths, parallel_iterations=1) return outputs
def BiGRU(x, hidden_size, dropout_prob=None, n_layer=None): x_len = length_seq(x) if n_layer: # multilayer GRU GRU_forwards = [rnn.GRUCell(hidden_size) for _ in range(n_layer)] GRU_backwards = [rnn.GRUCell(hidden_size) for _ in range(n_layer)] if dropout_prob: GRU_forwards = [ rnn.DropoutWrapper(i, input_keep_prob=dropout_prob) for i in GRU_forwards ] GRU_backwards = [ rnn.DropoutWrapper(i, input_keep_prob=dropout_prob) for i in GRU_backwards ] out, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw=GRU_forwards, cells_bw=GRU_backwards, inputs=x, sequence_length=x_len, dtype=tf.float32) else: GRU_forward = rnn.GRUCell(hidden_size) GRU_backword = rnn.GRUCell(hidden_size) if dropout_prob: GRU_forward = rnn.DropoutWrapper(GRU_forward, input_keep_prob=dropout_prob) GRU_backword = rnn.DropoutWrapper(GRU_backword, input_keep_prob=dropout_prob) (forward_out, backward_out), _ = tf.nn.bidirectional_dynamic_rnn( cell_fw=GRU_forward, cell_bw=GRU_backword, inputs=x, sequence_length=x_len, dtype=tf.float32) out = tf.concat((forward_out, backward_out), axis=2) return out
def encode(self, sequence, sequence_length): cells_fw, cells_bw = self._cells if self._use_cudnn: # Implements stacked bidirectional LSTM for variable-length sequences, # which are not supported by the CudnnLSTM layer. inputs_fw = tf.transpose(sequence, [1, 0, 2]) for lstm_fw, lstm_bw in zip(cells_fw, cells_bw): outputs_fw, _ = lstm_fw(inputs_fw, training=self._is_training) inputs_bw = tf.reverse_sequence( inputs_fw, sequence_length, seq_axis=0, batch_axis=1) outputs_bw, _ = lstm_bw(inputs_bw, training=self._is_training) outputs_bw = tf.reverse_sequence( outputs_bw, sequence_length, seq_axis=0, batch_axis=1) inputs_fw = tf.concat([outputs_fw, outputs_bw], axis=2) last_h_fw = _get_final(outputs_fw, sequence_length) # outputs_bw has already been reversed, so we can take the first element. last_h_bw = outputs_bw[0] else: _, states_fw, states_bw = rnn.stack_bidirectional_dynamic_rnn( cells_fw, cells_bw, sequence, sequence_length=sequence_length, time_major=False, dtype=tf.float32, scope=self._name_or_scope) # Note we access the outputs (h) from the states since the backward # ouputs are reversed to the input order in the returned outputs. last_h_fw = states_fw[-1][-1].h last_h_bw = states_bw[-1][-1].h return tf.concat([last_h_fw, last_h_bw], 1)
def add_birnn_layer(model, input): # Check required hyperparams try: num_cells = model.hyperparams["num_cells"] except KeyError: num_cells = 50 try: num_layers = model.hyperparams["num_layers"] except KeyError: num_layers = 1 fw_cells = [rnn.BasicLSTMCell(num_cells) for _ in range(num_layers)] bw_cells = [rnn.BasicLSTMCell(num_cells) for _ in range(num_layers)] fw_cells = [ rnn.DropoutWrapper(cell, output_keep_prob=model.dropout_keep_prob) for cell in fw_cells ] bw_cells = [ rnn.DropoutWrapper(cell, output_keep_prob=model.dropout_keep_prob) for cell in bw_cells ] model.rnn_outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn( fw_cells, bw_cells, input, sequence_length=model.x_len, dtype=tf.float32) return model.rnn_outputs
def bidi_gru_layers(config, inputs, hidden_sizes, seq_lens=None, bidirectional=False, phase=Phase.Predict): fcells = [rnn.GRUCell(size) for size in hidden_sizes] if phase == Phase.Train: fcells = [ rnn.DropoutWrapper(cell, output_keep_prob=config.rnn_output_dropout, state_keep_prob=config.rnn_state_dropout) for cell in fcells ] bcells = [rnn.GRUCell(size) for size in hidden_sizes] if phase == Phase.Train: bcells = [ rnn.DropoutWrapper(cell, output_keep_prob=config.rnn_output_dropout, state_keep_prob=config.rnn_state_dropout) for cell in bcells ] return rnn.stack_bidirectional_dynamic_rnn(fcells, bcells, inputs, dtype=tf.float32, sequence_length=seq_lens)
def __sequence_label(self, inputdata: tf.Tensor) -> Tuple[tf.Tensor, tf.Tensor]: """ Implements the sequence label part of the network :param inputdata: :return: """ with tf.variable_scope('LSTMLayers'): # construct stack lstm rcnn layer # forward lstm cell fw_cell_list = [tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self.__hidden_nums]*self.__layers_nums] # Backward direction cells bw_cell_list = [tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self.__hidden_nums]*self.__layers_nums] stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn(fw_cell_list, bw_cell_list, inputdata, dtype=tf.float32) if self.phase.lower() == 'train': stack_lstm_layer = self.dropout(inputdata=stack_lstm_layer, keep_prob=0.5) [batch_s, _, hidden_nums] = inputdata.get_shape().as_list() # [batch, width, 2*n_hidden] rnn_reshaped = tf.reshape(stack_lstm_layer, [-1, hidden_nums]) # [batch x width, 2*n_hidden] w = tf.Variable(tf.truncated_normal([hidden_nums, self.__num_classes], stddev=0.1), name="w") # Doing the affine projection logits = tf.matmul(rnn_reshaped, w) logits = tf.reshape(logits, [batch_s, -1, self.__num_classes]) raw_pred = tf.argmax(tf.nn.softmax(logits), axis=2, name='raw_prediction') # Swap batch and batch axis rnn_out = tf.transpose(logits, (1, 0, 2), name='transpose_time_major') # [width, batch, n_classes] return rnn_out, raw_pred
def encode(self, sequence, sequence_length): if self._use_cudnn: with tf.control_dependencies([ tf.assert_equal( sequence_length, tf.shape(sequence)[1], message= '`use_cudnn_enc` must be False if sequence lengths vary.' ) ]): _, (states_h, _) = self._cudnn_enc_lstm( tf.transpose(sequence, [1, 0, 2]), training=self._is_training) # Note we access the outputs (h) from the states since the backward # ouputs are reversed to the input order in the returned outputs. last_h_fw, last_h_bw = states_h[-2], states_h[-1] else: _, states_fw, states_bw = rnn.stack_bidirectional_dynamic_rnn( self._enc_cells_fw, self._enc_cells_bw, sequence, sequence_length=sequence_length, time_major=False, dtype=tf.float32, scope='encoder') # Note we access the outputs (h) from the states since the backward # ouputs are reversed to the input order in the returned outputs. last_h_fw = states_fw[-1][-1].h last_h_bw = states_bw[-1][-1].h # Concatenate the final outputs for each direction. last_h = tf.concat([last_h_fw, last_h_bw], 1) return last_h
def encode(self, sequence, sequence_length): if self._use_cudnn: with tf.control_dependencies( [tf.assert_equal( sequence_length, tf.shape(sequence)[1], message='`use_cudnn_enc` must be False if sequence lengths vary.') ]): _, (states_h, _) = self._cudnn_enc_lstm( tf.transpose(sequence, [1, 0, 2]), training=self._is_training) # Note we access the outputs (h) from the states since the backward # ouputs are reversed to the input order in the returned outputs. last_h_fw, last_h_bw = states_h[-2], states_h[-1] else: _, states_fw, states_bw = rnn.stack_bidirectional_dynamic_rnn( self._enc_cells_fw, self._enc_cells_bw, sequence, sequence_length=sequence_length, time_major=False, dtype=tf.float32, scope='encoder') # Note we access the outputs (h) from the states since the backward # ouputs are reversed to the input order in the returned outputs. last_h_fw = states_fw[-1][-1].h last_h_bw = states_bw[-1][-1].h # Concatenate the final outputs for each direction. last_h = tf.concat([last_h_fw, last_h_bw], 1) return last_h
def LSTM(inputs): # 定义LSTM网络 _hidden_nums = g_num_hidden _layers_nums = g_num_layers fw_cell_list = [ tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [_hidden_nums] * _layers_nums ] # Backward direction cells bw_cell_list = [ tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [_hidden_nums] * _layers_nums ] stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn( fw_cell_list, bw_cell_list, inputs, dtype=tf.float32) stack_lstm_layer = dropout(inputdata=stack_lstm_layer, keep_prob=0.5, is_training=g_is_training, name='sequence_drop_out') [batch_s, _, hidden_nums] = inputs.get_shape().as_list() # [batch, width, 2*n_hidden] shape = tf.shape(stack_lstm_layer) rnn_reshaped = tf.reshape(stack_lstm_layer, [shape[0] * shape[1], shape[2]]) return shape, hidden_nums, rnn_reshaped
def __init__(self,vocab_size,emb_dim,num_classes=4): # Hyper parameters self.dropout = 0.4 self.num_unit=100 self.num_layer=1 self.init_lr=0.001 self.clip_norm=5 Base.__init__(self,vocab_size,emb_dim) def cell(): cell=rnn.DropoutWrapper(rnn.GRUCell(num_units=self.num_unit)) return cell cell_fw=[cell() for _ in range(self.num_layer)] cell_bw =[cell() for _ in range(self.num_layer)] outputs,state_fw,state_bw = rnn.stack_bidirectional_dynamic_rnn( cells_fw=cell_fw, cells_bw=cell_bw, inputs=self.emb_inputs, sequence_length=self.lengths, dtype=tf.float32 ) self.merge = tf.concat([state_fw[-1],state_bw[-1]],axis=-1) # шонч╗Г self.train(num_classes)
def _sequence_label(self, inputdata, name): with tf.variable_scope(name_or_scope=name): fw_cell_list = [tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self._hidden_nums] * self._layers_nums] # Backward direction cells bw_cell_list = [tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self._hidden_nums] * self._layers_nums] stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn(fw_cell_list, bw_cell_list, inputdata, # sequence_length=CFG.ARCH.SEQ_LENGTH * np.ones(CFG.TRAIN.BATCH_SIZE), dtype=tf.float32) # stack_lstm_layer = self.dropout(inputdata=stack_lstm_layer, keep_prob=0.5,\ # is_training=self._is_training, name='sequence_drop_out') [batch_s, _, hidden_nums] = inputdata.get_shape().as_list() # [batch, width, 2*n_hidden] shape = tf.shape(stack_lstm_layer) rnn_reshaped = tf.reshape(stack_lstm_layer, [shape[0] * shape[1], shape[2]]) w = tf.get_variable(name='w', shape=[hidden_nums, self._num_classes], \ initializer=tf.truncated_normal_initializer(stddev=0.02), trainable=True) # Doing the affine projection logits = tf.matmul(rnn_reshaped, w, name='logits') logits = tf.reshape(logits, [shape[0], shape[1], self._num_classes], name='logits_reshape') raw_pred = tf.argmax(tf.nn.softmax(logits), axis=2, name='raw_prediction') # Swap batch and batch axis rnn_out = tf.transpose(logits, [1, 0, 2], name='transpose_time_major') # [width, batch, n_classes] return rnn_out, raw_pred
def StackedBRNN(input_rnn, input_size, hidden_size, num_layers, dropout_input=0, dropout_output=0, scope_stack=None): cells_fw = [] cells_bw = [] for i in range(num_layers): cell_fw = BasicLSTMCell(hidden_size, state_is_tuple=True) cell_bw = BasicLSTMCell(hidden_size, state_is_tuple=True) d_cell_fw = tf.contrib.rnn.DropoutWrapper( cell_fw, output_keep_prob=dropout_output, input_keep_prob=dropout_input) d_cell_bw = tf.contrib.rnn.DropoutWrapper( cell_bw, output_keep_prob=dropout_output, input_keep_prob=dropout_input) cells_fw.append(d_cell_fw) cells_bw.append(d_cell_bw) outputs, _, _ = stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, input_rnn, dtype=tf.float32, sequence_length=input_size, scope=scope_stack) return outputs
def stack_bi_gru_layer(self, embedded_input, sequence_length): def gru_cell(): if not self.is_training: self.keep_prob = 1.0 with tf.name_scope('gru_cell'): cell = rnn.GRUCell(self.settings.bi_gru_hidden_dim, reuse=tf.get_variable_scope().reuse) return rnn.DropoutWrapper(cell, output_keep_prob=self.keep_prob) cells_fw = [gru_cell() for _ in range(self.settings.bi_gru_layer_num)] cells_bw = [gru_cell() for _ in range(self.settings.bi_gru_layer_num)] initial_states_fw = [ cell_fw.zero_state(self.settings.batch_size, tf.float32) for cell_fw in cells_fw ] initial_states_bw = [ cell_bw.zero_state(self.settings.batch_size, tf.float32) for cell_bw in cells_bw ] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn( cells_fw, cells_bw, embedded_input, sequence_length=sequence_length, initial_states_fw=initial_states_fw, initial_states_bw=initial_states_bw, dtype=tf.float32) return outputs # [batch_size, max_time, layers_output]
def bi_rnn(inputs, extract_size, reuse=None): with tf.variable_scope("rnn", reuse=reuse): """express output of CNN on time step dimension.""" fw_cells = list() bw_cells = list() rnn_layer_num = 2 keep_prob = 0.5 for _ in range(rnn_layer_num): # Define cells fw_cell = rnn.BasicLSTMCell(extract_size, forget_bias=1.0) bw_cell = rnn.BasicLSTMCell(extract_size, forget_bias=1.0) # Drop out in case of over-fitting. fw_cell = rnn.DropoutWrapper(fw_cell, input_keep_prob=keep_prob, output_keep_prob=keep_prob) bw_cell = rnn.DropoutWrapper(bw_cell, input_keep_prob=keep_prob, output_keep_prob=keep_prob) # Stack same cells. fw_cells.append(fw_cell) bw_cells.append(bw_cell) output, _, _ = rnn.stack_bidirectional_dynamic_rnn(fw_cells, bw_cells, inputs, dtype=tf.float32) output = tf.transpose(output, [1, 0, 2]) seq_len = tf.shape(output)[0] - 1 output = tf.gather_nd(output, [seq_len]) # TODO express output of CNN on time step dimension. return output
def bi_gru(self, X_inputs): """build the bi-GRU network. Return the encoder represented vector. X_inputs: [batch_size, n_step] n_step: 句子的词数量;或者文档的句子数。 outputs: [fw_state, embeddings, bw_state], shape=[batch_size, hidden_size+embedding_size+hidden_size] """ inputs = tf.nn.embedding_lookup( self.embedding, X_inputs) # [batch_size, n_step, embedding_size] cells_fw = [self.gru_cell() for _ in range(self.n_layer)] cells_bw = [self.gru_cell() for _ in range(self.n_layer)] initial_states_fw = [ cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw ] initial_states_bw = [ cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw ] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn( cells_fw, cells_bw, inputs, initial_states_fw=initial_states_fw, initial_states_bw=initial_states_bw, dtype=tf.float32) hidden_outputs = tf.concat([outputs, inputs], axis=2) return hidden_outputs # shape =[seg_num, n_steps, hidden_size*2+embedding_size]
def bi_gru(self, inputs): print('================== bi_gru ==================') """build the Bi-GRU network. 返回个所有层的隐含状态。""" cells_fw = [self.gru_cell() for _ in range(self.n_layer)] cells_bw = [self.gru_cell() for _ in range(self.n_layer)] print('inputs: ', inputs.shape) # (?, 200, 256) print('cells_fw: ', type(cells_fw)) print('cells_bw: ', type(cells_bw)) initial_states_fw = [ cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw ] initial_states_bw = [ cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw ] # print('initial_states_bw: ', np.asarray(initial_states_bw).shape) outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn( cells_fw, cells_bw, inputs, initial_states_fw=initial_states_fw, initial_states_bw=initial_states_bw, dtype=tf.float32) outputs = tf.concat([outputs, inputs], axis=2) print('outputs: ', outputs.shape) # (?, 200, self.hidden_size * 2) return outputs
def DNNModel(self): with tf.name_scope('DNNModel'): lstm_fw_cells = list() lstm_bw_cells = list() for _ in range(self.layerNum): # Define LSTM cells with tensorflow fw_cell = rnn.BasicLSTMCell(self.hiddenSize, forget_bias=1.0) bw_cell = rnn.BasicLSTMCell(self.hiddenSize, forget_bias=1.0) # Drop out in case of over-fitting. fw_cell = rnn.DropoutWrapper(fw_cell, input_keep_prob=self.keep_prob, output_keep_prob=self.keep_prob) bw_cell = rnn.DropoutWrapper(bw_cell, input_keep_prob=self.keep_prob, output_keep_prob=self.keep_prob) # Stack same LSTM cells. lstm_fw_cells.append(fw_cell) lstm_bw_cells.append(bw_cell) pass outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn( lstm_fw_cells, lstm_bw_cells, self.x, dtype=tf.float32 ) self.logits = tf.contrib.layers.fully_connected(outputs, self.classNum, activation_fn=None) self.variableSummaries(self.logits) pass pass
def __init__(self, training, batch_size, name=''): super(FeatureNet, self).__init__() self.training = training # scalar self.batch_size = batch_size # [ΣK, 35/45, 7] self.feature = tf.placeholder( tf.float32, [None, cfg.VOXEL_POINT_COUNT, 7], name='feature') # [ΣK] self.number = tf.placeholder(tf.int64, [None], name='number') # [ΣK, 4], each row stores (batch, d, h, w) self.coordinate = tf.placeholder( tf.int64, [None, 4], name='coordinate') # with tf.variable_scope(name, reuse=tf.AUTO_REUSE) as scope: # self.vfe1 = VFELayer(32, 'VFE-1') # self.vfe2 = VFELayer(128, 'VFE-2') # # # boolean mask [K, T, 2 * units] # mask = tf.not_equal(tf.reduce_max( # self.feature, axis=2, keep_dims=True), 0) # x = self.vfe1.apply(self.feature, mask, self.training) # x = self.vfe2.apply(x, mask, self.training) # print("x.get_shape()", x.get_shape()) # # [ΣK, 128] # voxelwise = tf.reduce_max(x, axis=1) self.__hidden_nums=64 self.__layers_nums=2 fw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in [self.__hidden_nums] * self.__layers_nums] # Backward direction cells bw_cell_list = [rnn.BasicLSTMCell(nh, forget_bias=1.0) for nh in [self.__hidden_nums] * self.__layers_nums] stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn(fw_cell_list, bw_cell_list, self.feature, dtype=tf.float32) print("stack_lstm_layer.get_shape()",stack_lstm_layer.get_shape()) voxelwise = tf.reduce_max(stack_lstm_layer, axis=1) print("voxelwise.get_shape()", voxelwise.get_shape()) # car: [batch_size * 400 * 352 * 128] # pedestrian/cyclist: [N * 10 * 200 * 240 * 128] self.outputs = tf.scatter_nd( self.coordinate, voxelwise, [self.batch_size, 10, cfg.INPUT_HEIGHT, cfg.INPUT_WIDTH, 128])
def _sequence_label(self, inputdata, name): """ Implements the sequence label part of the network :param inputdata: :param name: :return: """ with tf.variable_scope(name_or_scope=name): # construct stack lstm rcnn layer # forward lstm cell fw_cell_list = [ tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self._hidden_nums] * self._layers_nums ] # Backward direction cells bw_cell_list = [ tf.nn.rnn_cell.LSTMCell(nh, forget_bias=1.0) for nh in [self._hidden_nums] * self._layers_nums ] stack_lstm_layer, _, _ = rnn.stack_bidirectional_dynamic_rnn( fw_cell_list, bw_cell_list, inputdata, dtype=tf.float32) stack_lstm_layer = self.dropout(inputdata=stack_lstm_layer, keep_prob=0.5, is_training=self._is_training, name='sequence_drop_out') [batch_s, _, hidden_nums ] = inputdata.get_shape().as_list() # [batch, width, 2*n_hidden] shape = tf.shape(stack_lstm_layer) rnn_reshaped = tf.reshape(stack_lstm_layer, [shape[0] * shape[1], shape[2]]) w = tf.get_variable( name='w', shape=[hidden_nums, self._num_classes], initializer=tf.truncated_normal_initializer(stddev=0.02), trainable=True) # Doing the affine projection logits = tf.matmul(rnn_reshaped, w, name='logits') logits = tf.reshape(logits, [shape[0], shape[1], self._num_classes], name='logits_reshape') raw_pred = tf.argmax(tf.nn.softmax(logits), axis=2, name='raw_prediction') # Swap batch and batch axis rnn_out = tf.transpose( logits, [1, 0, 2], name='transpose_time_major') # [width, batch, n_classes] return rnn_out, raw_pred
def feed_forward(inputs, seq_length): # multiple RNN layers stacked_rnn_fw, stacked_rnn_bw = [], [] for _ in range(num_layers - 1): stacked_rnn_fw.append(rnn.BasicLSTMCell(rnn_size, state_is_tuple=True)) stacked_rnn_bw.append(rnn.BasicLSTMCell(rnn_size, state_is_tuple=True)) # adding attention at last LSTM layer stacked_rnn_fw.append( tf.contrib.rnn.AttentionCellWrapper(rnn.BasicLSTMCell( rnn_size, state_is_tuple=True), attn_length=attn_length, state_is_tuple=True)) stacked_rnn_bw.append( tf.contrib.rnn.AttentionCellWrapper(rnn.BasicLSTMCell( rnn_size, state_is_tuple=True), attn_length=attn_length, state_is_tuple=True)) # stacking LSTM layers cell_fw = rnn.MultiRNNCell(cells=stacked_rnn_fw, state_is_tuple=True) cell_bw = rnn.MultiRNNCell(cells=stacked_rnn_bw, state_is_tuple=True) # applying bidirectional wrapper rnn_outputs, final_state_fw, final_state_bw = stack_bidirectional_dynamic_rnn( [cell_fw], [cell_bw], inputs, dtype=tf.float32, sequence_length=seq_length) print('rnn_outputs shape:', rnn_outputs.shape) # applying batch normalization if phase == True: # while training batch_mean, batch_var = tf.nn.moments(rnn_outputs, [0]) train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay)) train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay)) with tf.control_dependencies([train_mean, train_var]): rnn_outputs = tf.nn.batch_normalization(rnn_outputs, batch_mean, batch_var, beta, scale, epsilon) else: # while validation rnn_outputs = tf.nn.batch_normalization(rnn_outputs, pop_mean, pop_var, beta, scale, epsilon) rnn_outputs = tf.nn.relu(rnn_outputs) rnn_outputs = tf.nn.dropout(rnn_outputs, keep_prob) # stacking the outputs of all time frames last_rnn_output = tf.gather_nd( rnn_outputs, tf.stack([tf.range(batch_size), seq_length - 1], axis=1)) # output layer output = tf.add(tf.matmul(last_rnn_output, layer['weight']), layer['bias']) print('Output shape', output.get_shape()) return output
def bi_gru(self, inputs): """build the bi-GRU network. 返回个所有层的隐含状态。""" cells_fw = [self.gru_cell() for _ in range(self.n_layer)] cells_bw = [self.gru_cell() for _ in range(self.n_layer)] initial_states_fw = [cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw] initial_states_bw = [cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs, initial_states_fw=initial_states_fw, initial_states_bw=initial_states_bw, dtype=tf.float32) return outputs
def bi_lstm(self, inputs): """build the bi-LSTM network. 返回个所有层的隐含状态。""" cells_fw = [self.lstm_cell() for _ in range(self.n_layer)] cells_bw = [self.lstm_cell() for _ in range(self.n_layer)] initial_states_fw = [cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw] initial_states_bw = [cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs, initial_states_fw=initial_states_fw, initial_states_bw=initial_states_bw, dtype=tf.float32) return outputs
def __init__(self, learning_rate, num_classes, hidden_units): # Initialize data and variables self.weights = tf.Variable( tf.random_uniform([hidden_units * 2, num_classes], minval=-0.5, maxval=0.5)) self.biases = tf.Variable(tf.random_uniform([num_classes])) self.x = tf.placeholder("float", [None, 700, 44]) self.y = tf.placeholder("float", [None, 700, num_classes]) # Do the prediction self.fw_rnn_cell1 = rnn.LSTMCell(hidden_units, forget_bias=1.0) self.fw_rnn_cell2 = rnn.LSTMCell(hidden_units, forget_bias=1.0) self.fw_rnn_cell3 = rnn.LSTMCell(hidden_units, forget_bias=1.0) self.bw_rnn_cell1 = rnn.LSTMCell(hidden_units, forget_bias=1.0) self.bw_rnn_cell2 = rnn.LSTMCell(hidden_units, forget_bias=1.0) self.bw_rnn_cell3 = rnn.LSTMCell(hidden_units, forget_bias=1.0) self.fw_rnn_cells = [ self.fw_rnn_cell1, self.fw_rnn_cell2, self.fw_rnn_cell3 ] self.bw_rnn_cells = [ self.bw_rnn_cell1, self.bw_rnn_cell2, self.bw_rnn_cell3 ] self.outputs, self.states_fw, self.states_bw = rnn.stack_bidirectional_dynamic_rnn( self.fw_rnn_cells, self.bw_rnn_cells, self.x, dtype=tf.float32) # self.output.shape is (?, 700, 600) self.outputs_reshaped = tf.reshape(self.outputs, [-1, 2 * hidden_units]) self.y_reshaped = tf.reshape(self.y, [-1, num_classes]) # check importantFunctions.py : line-40 to see how it works # reference link is : # https://stackoverflow.com/questions/38051143/no-broadcasting-for-tf-matmul-in-tensorflow # self.y_predicted = tf.nn.softmax(tf.matmul(self.outputs_reshaped, self.weights) + self.biases) self.y_predicted = tf.matmul(self.outputs_reshaped, self.weights) + self.biases # Define the loss function self.loss = tf.nn.softmax_cross_entropy_with_logits( logits=self.y_predicted, labels=self.y_reshaped) # Define the trainer and optimizer self.optimizer = tf.train.GradientDescentOptimizer(learning_rate) self.trainer = self.optimizer.minimize(self.loss) # creating session and initializing variables self.sess = tf.Session() self.init = tf.global_variables_initializer() self.sess.run(self.init) # get accuracy self.get_equal = tf.equal(tf.argmax(self.y_reshaped, 1), tf.argmax(self.y_predicted, 1)) self.accuracy = tf.reduce_mean(tf.cast(self.get_equal, tf.float32))
def bi_gru(self, inputs, seg_num): """build the bi-GRU network. Return the encoder represented vector. n_step: 句子的词数量;或者文档的句子数。 seg_num: 序列的数量,原本应该为 batch_size, 但是这里将 batch_size 个 doc展开成很多个句子。 """ cells_fw = [self.gru_cell() for _ in range(self.n_layer)] cells_bw = [self.gru_cell() for _ in range(self.n_layer)] initial_states_fw = [cell_fw.zero_state(seg_num, tf.float32) for cell_fw in cells_fw] initial_states_bw = [cell_bw.zero_state(seg_num, tf.float32) for cell_bw in cells_bw] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs, initial_states_fw = initial_states_fw, initial_states_bw = initial_states_bw, dtype=tf.float32) # outputs: Output Tensor shaped: seg_num, max_time, layers_output],其中layers_output=hidden_size * 2 在这里。 return outputs
def discriminator_rnn(x, labels, df_dim, number_classes, kernel=(3, 3), strides=(2, 2), dilations=(1, 1), pooling='avg', update_collection=None, act=tf.nn.relu, scope_name='Discriminator', reuse=False): num_layers = 3 num_nodes = [int(8 / 2), df_dim, df_dim] x = tf.transpose(tf.squeeze(x), perm=[0, 2, 1]) with tf.variable_scope(scope_name) as scope: if reuse: scope.reuse_variables() # Define LSTM cells enc_fw_cells = [ LSTMCell(num_nodes[layer], name="fw_" + str(layer)) for layer in range(num_layers) ] enc_bw_cells = [ LSTMCell(num_nodes[layer], name="bw_" + str(layer)) for layer in range(num_layers) ] # Connect LSTM cells bidirectionally and stack (all_states, fw_state, bw_state) = stack_bidirectional_dynamic_rnn(cells_fw=enc_fw_cells, cells_bw=enc_bw_cells, inputs=x, dtype=tf.float32) # Concatenate results for k in range(num_layers): if k == 0: con_c = tf.concat((fw_state[k].c, bw_state[k].c), 1) con_h = tf.concat((fw_state[k].h, bw_state[k].h), 1) else: con_c = tf.concat((con_c, fw_state[k].c, bw_state[k].c), 1) con_h = tf.concat((con_h, fw_state[k].h, bw_state[k].h), 1) output = all_states[:, x.get_shape.as_list()[2]] output = ops.snlinear(output, 1, update_collection=update_collection, name='d_sn_linear') return output, tf.concat((fw_state[2].c, bw_state[2].c), 1)
def lstm(seq, seq_len, num_class, is_training): # bid-lstm with tf.variable_scope('bid-lstm'): lstm_fw_cell = tf.nn.rnn_cell.LSTMCell(256, forget_bias=1.0, state_is_tuple=True) lstm_bw_cell = tf.nn.rnn_cell.LSTMCell(256, forget_bias=1.0, state_is_tuple=True) cell_fw = [lstm_fw_cell] * 2 cell_bw = [lstm_bw_cell] * 2 outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cell_fw, cell_bw, seq, dtype=tf.float32) # cell = tf.contrib.rnn.LSTMCell(512, state_is_tuple=True) # stack = tf.contrib.rnn.MultiRNNCell([cell] * 1, state_is_tuple=True) # outputs, _ = tf.nn.dynamic_rnn(cell, seq, dtype=tf.float32) outputs = tf.cond( pred=is_training, true_fn=lambda: tf.nn.dropout(outputs, keep_prob=0.4), false_fn=lambda: outputs, name='lstm_dropout') batch_size = 100 output = tf.reshape(outputs, [-1, 256 * 2]) W = tf.get_variable(name='weights', shape=[256 * 2, num_class], initializer=tf.truncated_normal_initializer( stddev=5e-2, dtype=tf.float16)) b = tf.get_variable(name='biases', shape=[num_class], initializer=tf.constant_initializer(0.0)) logits = tf.matmul(output, W) + b # logits = tf.cond(pred=is_training, # true_fn=lambda: tf.nn.dropout(logits, keep_prob=0.8), # false_fn=lambda: logits, # name='lstm_dropout_logits') #[batch_size,max_timesteps,num_classes] logits = tf.reshape(logits, [batch_size, -1, num_class]) raw_pred = tf.argmax(tf.nn.softmax(logits), axis=2, name='raw_prediction') #转置矩阵,第0和第1列互换位置=>[max_timesteps,batch_size,num_classes] logits = tf.transpose(logits, (1, 0, 2)) return logits, raw_pred, seq_len
def bi_gru(self, X_inputs): """build the bi-GRU network. Return the encoder represented vector. X_inputs: [batch_size, n_step] n_step: 句子的词数量;或者文档的句子数。 outputs: [fw_state, embeddings, bw_state], shape=[batch_size, hidden_size+embedding_size+hidden_size] """ inputs = tf.nn.embedding_lookup(self.embedding, X_inputs) # [batch_size, n_step, embedding_size] cells_fw = [self.gru_cell() for _ in range(self.n_layer)] cells_bw = [self.gru_cell() for _ in range(self.n_layer)] initial_states_fw = [cell_fw.zero_state(self.batch_size, tf.float32) for cell_fw in cells_fw] initial_states_bw = [cell_bw.zero_state(self.batch_size, tf.float32) for cell_bw in cells_bw] outputs, _, _ = rnn.stack_bidirectional_dynamic_rnn(cells_fw, cells_bw, inputs, initial_states_fw = initial_states_fw, initial_states_bw = initial_states_bw, dtype=tf.float32) hidden_outputs = tf.concat([outputs, inputs], axis=2) return hidden_outputs # shape =[seg_num, n_steps, hidden_size*2+embedding_size]