def BiRNN(x, weights, biases): # Prepare data shape to match `bidirectional_rnn` function requirements # Current data input shape: (batch_size, n_steps, n_input) # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.unstack(x, n_steps, 1) # Define lstm cells with tensorflow # Forward direction cell lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) # Get lstm cell output try: outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) # Linear activation, using rnn inner loop last output return tf.matmul(outputs[-1], weights['out']) + biases['out']
def inference(x, n_in=None, n_time=None, n_hidden=None, n_out=None): def weight_variable(shape): initial = tf.truncated_normal(shape, stddev=0.01) return tf.Variable(initial) def bias_variable(shape): initial = tf.zeros(shape, dtype=tf.float32) return tf.Variable(initial) # 시계열 데이터의 형식을 API의 사양에 맞추기 위해 최종적으로 # (미니배치 크기, 입력 차원수)가 시간 길이 만큼 있는 형태로 변형한다 x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_in]) x = tf.split(x, n_time, 0) cell_forward = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) cell_backward = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = \ rnn.static_bidirectional_rnn(cell_forward, cell_backward, x, dtype=tf.float32) W = weight_variable([n_hidden * 2, n_out]) b = bias_variable([n_out]) y = tf.nn.softmax(tf.matmul(outputs[-1], W) + b) return y
def summery_BiRNN(self, x, weights, biases, Lstmcell, sen_len): x = tf.unstack(x, self.sentence_length, 1) with tf.variable_scope('summery_nn'): outputs, _, _ = rnn.static_bidirectional_rnn(Lstmcell['fw_lstm'], Lstmcell['bw_lstm'], x, dtype=tf.float32) sum_len = tf.reduce_sum(sen_len, 0) return tf.matmul(outputs[-1], weights['summery']) + biases['summery']
def BiRnn(x,n_hidden): # Unstack to get a list of 'n_steps' tensors of shape (batch_size, embedding_size) x = tf.unstack(x, n_steps, 1) lstm_fw_cells = rnn.MultiRNNCell([attn_cell(n_hidden) for _ in range(num_layers)] , state_is_tuple=True) lstm_bw_cells = rnn.MultiRNNCell([attn_cell(n_hidden) for _ in range(num_layers)] , state_is_tuple=True) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cells, lstm_bw_cells, x,dtype=tf.float32) outputs = tf.transpose(tf.stack(outputs), perm=[1, 0, 2]) outputs = tf.reshape(outputs,[-1,2*n_hidden]) return outputs
def intent_BiRNN(x, weights, biases): x = tf.unstack(x, n_words, 1) lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return tf.matmul(outputs[-1], weights['intent_out']) + biases['intent_out']
def BiRNN(x, weights, biases): x = tf.unstack(x, n_steps, 1) lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=0.5) # Backward direction cell lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=0.5) # Get lstm cell output try: outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) # Linear activation, using rnn inner loop last output return tf.matmul(outputs[-1], weights['out']) + biases['out']
def RNN(x, weights, biases): # Prepare data shape to match `rnn` function requirements # Current data input shape: (batch_size, n_steps, n_input) # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.unstack(x, n_steps, 1) # Define a lstm cell with tensorflow #lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) #lstm_cell = rnn.GRUCell(n_hidden) lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) lstm_cell_bk = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) # make the deep rnn no_of_layers = 8 # layer number of drnn stacked_lstm = rnn.MultiRNNCell([ #rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple = True, reuse = tf.get_variable_scope().reuse) rnn.LSTMCell(n_hidden, use_peepholes=True, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) for _ in range(no_of_layers) ]) stacked_lstm_bk = rnn.MultiRNNCell([ #rnn.BasicLSTMCell(n_hidden, forget_bias=1.0, state_is_tuple = True, reuse = tf.get_variable_scope().reuse) rnn.LSTMCell(n_hidden, use_peepholes=True, forget_bias=1.0, state_is_tuple=True, reuse=tf.get_variable_scope().reuse) for _ in range(no_of_layers) ]) # providing the dropout for rnn #lstm_cell = rnn.DropoutWrapper(lstm_cell, output_keep_prob=0.5) # for rnn stacked_lstm = rnn.DropoutWrapper(stacked_lstm, output_keep_prob=0.5) # for deep rnn # Get lstm cell output #outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32) # single layer rnn #outputs, states = rnn.static_rnn(stacked_lstm,, x, dtype=tf.float32) # deep rnn #outputs, states, states_bk = rnn.static_bidirectional_rnn(lstm_cell, lstm_cell_bk, x, dtype=tf.float32) # single layer dirnn outputs, states, states_bk = rnn.static_bidirectional_rnn( stacked_lstm, stacked_lstm_bk, x, dtype=tf.float32) # deep dirnn # Linear activation, using rnn inner loop last output return tf.matmul(outputs[-1], weights['out']) + biases['out']
def _build_net(self): self.x = tf.placeholder(tf.float32, [None, time_step, unit_num]) self.y = tf.placeholder(tf.int32, [None, time_step]) seq_x = tf.reshape(self.x, [-1, time_step * unit_num]) seq_x = tf.split(seq_x, time_step, axis=1) cell_forward = tf.contrib.rnn.BasicLSTMCell(unit_num) cell_backward = tf.contrib.rnn.BasicLSTMCell(unit_num) cell_forward = DropoutWrapper(cell_forward, input_keep_prob=1.0, output_keep_prob=DROPOUT_RATE) cell_backward = DropoutWrapper(cell_backward, input_keep_prob=1.0, output_keep_prob=DROPOUT_RATE) # outputs:(time_step, batch_size, 2 * num_units) outputs, output_state_fw, output_state_bw = \ static_bidirectional_rnn(cell_forward, cell_backward, seq_x, dtype=tf.float32) # Convert (time_step, batch_size, 2 * num_units) to (batch_size, time_step, 2 * num_units) rnn_features = tf.transpose(outputs, [1, 0, 2]) rnn_features = tf.reshape(rnn_features, [-1, 2 * unit_num]) # CNN # You could use more advanced kernel, which is introduced in # https://github.com/tensorflow/models/blob/master/tutorials/image/cifar10/cifar10.py filter_deep = 2 # 推荐2 或 1 # 值越大,CNN特征占的比例就越多。 如果是2表示CNN的特征和bi-rnn的特征数量一样。 cnn_W = tf.get_variable("cnn_w", shape=[EMBEDDING_SIZE, 3, 1, filter_deep]) cnn_b = tf.get_variable("cnn_b", shape=[filter_deep]) # it is better to make the units number equal to the RNN unit number # cnn_input : (batch_size, time_step, unit_num, 1) cnn_input = tf.expand_dims(self.x, axis=3) # conv_features : (batch_size, time_step, unit_num, 2) conv_features = tf.nn.conv2d( cnn_input, cnn_W, strides=[1, 1, 1, 1], padding='SAME') + cnn_b conv_features = tf.nn.dropout(conv_features, keep_prob=DROPOUT_RATE) conv_features = tf.reshape(conv_features, [-1, unit_num * filter_deep]) all_feature = tf.concat([rnn_features, conv_features], axis=1) # projection: W = tf.get_variable( "projection_w", [(filter_deep + 2) * unit_num, TAGS_NUM]) #这里的2是指bi-rnn,所以是个常量 b = tf.get_variable("projection_b", [TAGS_NUM]) projection = tf.matmul(all_feature, W) + b self.outputs = tf.reshape(projection, [-1, time_step, TAGS_NUM]) # self.outputs = tf.transpose(output, [1,0,2]) #BATCH_SIZE * time_step * TAGS_NUM self.log_likelihood, self.transition_params = tf.contrib.crf.crf_log_likelihood( self.outputs, self.y, sequence_lengths) # Add a training op to tune the parameters. self.loss = tf.reduce_mean(-self.log_likelihood) self.train_op = tf.train.AdamOptimizer().minimize(self.loss)
def BiRNN(x, weights, biases): x = tf.unstack(x, n_steps, 1) #lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_fw_cell = rnn.GRUCell(FLAGS.hidden_size) #lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn.GRUCell(FLAGS.hidden_size) try: outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return tf.nn.softmax( tf.matmul(outputs[-1], weights['out']) + biases['out'])
def bilstm_model(inputs, targets, config): """ use BasicLSTMCell,MultiRNNCell method to build lstm model retun logits,cost and others """ batch_size = config.batch_size num_steps = config.num_steps num_layers = config.num_layers hidden_size = config.hidden_size vocab_size = config.vocab_size target_num = config.target_num # target output number lstm_fw_cell = rnn.BasicLSTMCell(hidden_size, forget_bias=0.0, state_is_tuple=True) lstm_bw_cell = rnn.BasicLSTMCell(hidden_size, forget_bias=0.0, state_is_tuple=True) cell_fw = rnn.MultiRNNCell([lstm_fw_cell] * num_layers, state_is_tuple=True) cell_bw = rnn.MultiRNNCell([lstm_bw_cell] * num_layers, state_is_tuple=True) initial_state_fw = cell_fw.zero_state(batch_size, tf.float32) initial_state_bw = cell_bw.zero_state(batch_size, tf.float32) #split to get a list of n_steps tensors of shape (batch_size,n_input) inputs_list = [ tf.squeeze(s, axis=1) for s in tf.split(value=inputs, num_or_size_splits=num_steps, axis=1) ] with tf.variable_scope('pos_bilstm'): outputs, state_fw, state_bw = rnn.static_bidirectional_rnn( cell_fw, cell_bw, inputs_list, initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw) # outputs is a length T list of output vectors, which is [batch_size, 2*hidden_size] output = tf.reshape(tf.concat(outputs, 1), [-1, hidden_size * 2]) softmax_w = tf.get_variable('softmax_w', [hidden_size * 2, target_num], dtype=tf.float32) softmax_b = tf.get_variable('softmax_b', [target_num], dtype=tf.float32) logits = tf.matmul(output, softmax_w) + softmax_b # add extra statistics to monitor correct_prediction = tf.equal(tf.cast(tf.argmax(logits, 1), tf.int32), tf.reshape(targets, [-1])) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.reshape( targets, [-1]), logits=logits) cost = tf.reduce_sum(loss) / batch_size return cost, logits
def BdRNN(x, weight, bias, n_steps, n_hidden, n_input, name="output", reshape = False): # Unstack to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.unstack(x, n_steps, 1) lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) try: outputs, SL, SR = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: #The older versions of TF do not produce the two output states outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) y = tf.add(tf.matmul(outputs, weight), bias, name="rnn_output") return y, SL, SR
def BiRNN(x, num_hidden, keep_prob): # Define weights weights = { # Hidden layer weights => 2*n_hidden because of forward + backward cells 'out': tf.Variable(tf.random_normal([2 * num_hidden, num_classes])) } biases = {'out': tf.Variable(tf.random_normal([num_classes]))} # Prepare data shape to match `rnn` function requirements current data input shape: (batch_size, timesteps, n_input) #batch size = number of sample points, time step = number of sample points to be considered at each instance of time #num_input = number of features # Required shape: 'timesteps' tensors list of shape (batch_size, num_input) x = tf.unstack( x, timesteps, 1 ) # Unstack to get a list of 'timesteps' tensors of shape (batch_size, num_input) # Define lstm cells with tensorflow lstm_fw_cell = tf.nn.rnn_cell.DropoutWrapper( rnn.BasicLSTMCell(num_hidden, forget_bias=0.4, reuse=tf.AUTO_REUSE), input_keep_prob=keep_prob, output_keep_prob=keep_prob, variational_recurrent=False) # Forward direction cell lstm_bw_cell = tf.nn.rnn_cell.DropoutWrapper( rnn.BasicLSTMCell(num_hidden, forget_bias=0.4, reuse=tf.AUTO_REUSE), input_keep_prob=keep_prob, output_keep_prob=keep_prob, variational_recurrent=False) # Backward direction cell # Get lstm cell output try: outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return (tf.matmul(outputs[-1], weights['out']) + biases['out'] ) # Linear activation, using rnn inner loop last output
def rnn_unit(self, input, name='LSTM'): # check, then set the right name assert self.rnn_type in ['LSTM', 'GRU'], \ 'rnn_type has to be either LSTM or GRU' name = 'LSTM' if self.rnn_type == 'LSTM' else 'GRU' if self.bidirection: name += '_bidir' with tf.name_scope(name): input = tf.nn.embedding_lookup(self.embedding_matrix, input) # reshaping # Permuting batch_size and n_steps input = tf.transpose(input, [1, 0, 2]) # Reshaping to (n_steps*batch_size, n_input) input = tf.reshape(input, [-1, self.char_embed_dim]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) rnn_inputs = tf.split(input, self.seq_len, 0) # setting the correct RNN cell type (LSTM of GRU) rnn_cell = rnn.BasicLSTMCell if self.rnn_type == 'LSTM' \ else rnn.GRUCell # setting the args (forget_bias applies only to LSTM) rnn_cell_args = { 'num_units': self.hidden_state_size, 'activation': self.activation_function } if 'LSTMCell' in str(rnn_cell.__call__): rnn_cell_args['forget_bias'] = 1.0 rnn_cell(**rnn_cell_args) cell_fw = rnn_cell(**rnn_cell_args) cell_fw = rnn.DropoutWrapper(cell_fw, output_keep_prob=self.keep_prob, seed=self.seed) if self.bidirection: # add another cell for backwards direction and a dropout wrapper cell_bw = rnn_cell(**rnn_cell_args) cell_bw = rnn.DropoutWrapper(cell_bw, output_keep_prob=self.keep_prob, seed=self.seed) outputs, _, _ = rnn.static_bidirectional_rnn(cell_fw, cell_bw, rnn_inputs, dtype=tf.float32, scope=name) else: outputs, _ = rnn.static_rnn(cell_fw, rnn_inputs, dtype=tf.float32, scope=name) if not self.target_rep: # take only last output (list for structure consistency) outputs = [outputs[-1]] if self.verbose_summary: tf.summary.histogram('outputs', outputs, collections=['train']) return outputs
def _init_model(self): # Create multiple forward lstm cell cell_fw = rnn.MultiRNNCell([ rnn.BasicLSTMCell(self._config['hidden_size']) for _ in range(self._config['num_layers']) ]) # Create multiple backward lstm cell cell_bw = rnn.MultiRNNCell([ rnn.BasicLSTMCell(self._config['hidden_size']) for _ in range(self._config['num_layers']) ]) inputs = self._input.input_data # Add dropout layer to the input data if self._is_training and self._config['keep_prob'] < 1: intpus = [ tf.nn.dropout(single_input, self._config['keep_prob']) for single_input in inputs ] self._outputs, _, _ = rnn.static_bidirectional_rnn(cell_fw, cell_bw, inputs, dtype=tf.float32) # Hidden layer weights => 2*hidden_size because of forward + backward cells softmax_w = tf.get_variable( "softmax_w", [2 * self._config['hidden_size'], self._config['num_classes']]) softmax_b = tf.get_variable("softmax_b", [self._config['num_classes']]) # Linear activation, using rnn inner loop last output # logit shape: [batch_size, num_classes] self._logits = tf.matmul(self._outputs[-1], softmax_w) + softmax_b # Define loss # Required targets shape: [batch_size, num_classes] (one hot vector) self._cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits( logits=self._logits, labels=self._input.targets)) # Evaluate model self._correct_pred = tf.equal(tf.argmax(self._logits, 1), tf.argmax(self._input.targets, 1)) self.accuracy = tf.reduce_mean(tf.cast(self._correct_pred, tf.float32)) # Define optimizer self._lr = tf.Variable(0.0, trainable=False) self._train_op = tf.train.AdamOptimizer( learning_rate=self._lr).minimize(self._cost) self._new_lr = tf.placeholder(tf.float32, shape=[], name="new_learning_rate") self._lr_update = tf.assign(self._lr, self._new_lr)
def model(inputs, variables): # Define lstm cells with tensorflow inputs = tf.unstack(inputs, FRAME_WINDOW, 1) with tf.variable_scope('layer1') as scope: # Forward direction cell lstm_fw_cell = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, inputs, dtype=tf.float32, scope=scope) with tf.variable_scope('layer2') as scope: # Forward direction cell lstm_fw_cell_1 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0) # Backward direction cell lstm_bw_cell_1 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0) outputs_1, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell_1, lstm_bw_cell_1, outputs, dtype=tf.float32, scope=scope) with tf.variable_scope('layer3') as scope: # Forward direction cell lstm_fw_cell_2 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0) # Backward direction cell lstm_bw_cell_2 = rnn.BasicLSTMCell(HIDDEN_DIMENSION, forget_bias=1.0) outputs_2, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell_2, lstm_bw_cell_2, outputs_1, dtype=tf.float32, scope=scope) return tf.nn.sigmoid( tf.matmul(outputs_2[-1], variables['w1_lstm']) + variables['b1_lstm'])
def BiRNN(x, weights, biases): x = tf.unstack(x, n_steps, 1) lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) # Get lstm cell output try: outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return tf.matmul(outputs[-1], weights['out']) + biases['out']
def forward(self, user_metapath, item_metapath, is_training, rnn_cell=rnn.LSTMCell): user_item_embedding_table = \ tf.get_variable(name="user_embedding", shape=[self.user_num+self.item_num, self.embedding_size], initializer=tf.contrib.layers.xavier_initializer()) user_metapath_input = tf.nn.embedding_lookup(user_item_embedding_table, user_metapath) user_forward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="user_fw_%d" % idx) for idx in range(self.lstm_layer_num)] user_backward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="user_bw_%d" % idx) for idx in range(self.lstm_layer_num)] user_multilayer_forward_rnn_cell = rnn.MultiRNNCell(user_forward_rnn_cell_list) user_multilayer_backward_rnn_cell = rnn.MultiRNNCell(user_backward_rnn_cell_list) user_output, _, _ = rnn.static_bidirectional_rnn(cell_fw=user_multilayer_forward_rnn_cell, cell_bw=user_multilayer_backward_rnn_cell, inputs=tf.unstack(user_metapath_input, num=3, axis=1), dtype=tf.float32) item_metapath_input = tf.nn.embedding_lookup(user_item_embedding_table, item_metapath) item_forward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="item_fw_%d" % idx) for idx in range(self.lstm_layer_num)] item_backward_rnn_cell_list = [rnn_cell(num_units=self.lstm_hidden_state_size, name="item_bw_%d" % idx) for idx in range(self.lstm_layer_num)] item_multilayer_forward_rnn_cell = rnn.MultiRNNCell(item_forward_rnn_cell_list) item_multilayer_backward_rnn_cell = rnn.MultiRNNCell(item_backward_rnn_cell_list) item_output, _, _ = rnn.static_bidirectional_rnn(cell_fw=item_multilayer_forward_rnn_cell, cell_bw=item_multilayer_backward_rnn_cell, inputs=tf.unstack(item_metapath_input, num=3, axis=1), dtype=tf.float32) output = tf.concat([user_output[0], item_output[0]], axis=1) for mlp_num_unit in self.mlp_hidden_size_list: output = tf.keras.layers.Dense(units=mlp_num_unit, activation=tf.nn.relu, kernel_initializer=tf.contrib.layers.xavier_initializer())(output) output = tf.layers.dropout(output, training=is_training) self.saver = tf.train.Saver(max_to_keep=100) return output
def prediction(self): data_unstacked = tf.unstack(self.data, n_steps, 1) lstm_forward_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_backward_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_forward_cell, lstm_backward_cell, data_unstacked, dtype=tf.float32) return tf.matmul(outputs[-1], self.weights['out']) + self.biases['out']
def bi_rnn_model(x, weight, bias): x = tf.unstack(x, timestep, axis=1) #print(tf.shape(x)) lstm_cell_forward = rnn.BasicLSTMCell(rnn_hidden, forget_bias=1.0) lstm_cell_backward = rnn.BasicLSTMCell(rnn_hidden, forget_bias=1.0) output, _, _ = rnn.static_bidirectional_rnn(lstm_cell_forward, lstm_cell_backward, x, dtype=tf.float32) return tf.matmul(output[-1], weight['out']) + bias['out']
def discriminator(self, inputs, reuse=False): with tf.variable_scope('discriminator', reuse=reuse) as scope: # build RNN here lstm_ = rnn.LSTMCell(self.Config.D_HIDDEN_NEURONS) lstm_ = rnn.DropoutWrapper( lstm_, input_keep_prob=self.Config.DROPOUT_IN, output_keep_prob=self.Config.DROPOUT_OUT) stacked_lstm = rnn.MultiRNNCell( [lstm_ for _ in range(self.Config.D_NUM_LAYERS)]) # initialize initial state to 0 init_state = stacked_lstm.zero_state(self.batchsize, tf.float32) # bidirectional RNN outputs, _, _ = rnn.static_bidirectional_rnn( stacked_lstm, stacked_lstm, tf.unstack(tf.transpose(inputs, perm=[1, 0, 2])), initial_state_fw=init_state, initial_state_bw=init_state, scope=scope) outputs = tf.contrib.layers.fully_connected( outputs, self.Config.AE_DENSE_NEURONS, reuse=reuse, scope=scope, activation_fn=None) # specify activation function here outputs = self.leakyrelu(outputs) # dropout outputs = tf.nn.dropout(outputs, keep_prob=self.Config.DROPOUT_OUT) outputs = tf.transpose(outputs, perm=[1, 0, 2]) # use only the last hidden state as input to dense layer outputs = tf.slice( outputs, [0, self.Config.TIMESTEPS - 1, 0], [self.batchsize, 1, self.Config.AE_DENSE_NEURONS]) # use last hidden state as input to a dense layer outputs = tf.layers.dense(outputs, int(self.Config.AE_DENSE_NEURONS / 2), name='discriminator/pre_output') # specify activation function here outputs = self.leakyrelu(outputs) # dropout outputs = tf.nn.dropout(outputs, keep_prob=self.Config.DROPOUT_OUT) # final output of discriminator outputs = tf.layers.dense(outputs, 1, name='discriminator/output') return outputs
def biRNN(x,weights,biase): x = tf.transpose(x,[1,0,2]) x = tf.reshape(x,[-1,n_input]) x = tf.split(x,n_step,0) lstm_fw_cell = rnn.BasicLSTMCell(n_hidden) lstm_bw_cell = rnn.BasicLSTMCell(n_hidden) outs,_,_ = rnn.static_bidirectional_rnn(lstm_fw_cell,lstm_bw_cell,x,dtype = tf.float32) return tf.matmul(outs[-1],weights['out'])+biase['out']
def bi_lstm_layer(self,inputs): if self.hidden_layer_num >1: lstm_fw = rnn.MultiRNNCell([self.lstm_cell() for _ in range(self.hidden_layer_num)]) lstm_bw = rnn.MultiRNNCell([self.lstm_cell() for _ in range(self.hidden_layer_num)]) else: lstm_fw = self.lstm_cell() lstm_bw = self.lstm_cell() outpus,_,_ = rnn.static_bidirectional_rnn(lstm_fw,lstm_bw,inputs,sequence_length=self.lengths,dtype=tf.float32) features = tf.reshape(outpus,[-1,self.num_hidden *2]) return features
def BiRNN(x, weights, biases): x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(x, n_steps) lstm_fw_cell = BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return tf.matmul(outputs[-1], weights) + biases
def brnn(x, config, is_training): """ input args a. num_layers b. timesteps c. cell_fn d. activation_fn e. batch_size f. num_units """ # params cell_fn = config.cell_fn activation_fn = config.activation_fn initializer_fn = config.initializer_fn dropout_keep = config.dropout_keep num_units = config.num_units num_layers = config.num_layers # get shape and reshape batchsize, n_dim, n_step = get_shape(x) x = tf.reshape(x, [-1, n_step, n_dim]) # transform to list x = tf.unstack(x, n_step, axis=1) # sequence_length sequence_length = [n_dim for _ in range(batchsize)] fw_cells = [] bw_cells = [] for idx in range(num_layers): # define fw_cell = cell_fn(num_units[idx], activation=activation_fn) bw_cell = cell_fn(num_units[idx], activation=activation_fn) # define dropout if dropout_keep is not None and is_training: fw_cell = rnn.DropoutWrapper(fw_cell, output_keep_prob=dropout_keep) bw_cell = rnn.DropoutWrapper(bw_cell, output_keep_prob=dropout_keep) fw_cells.append(fw_cell) bw_cells.append(bw_cell) # stacks multi-layer fw_cells = rnn.MultiRNNCell(fw_cells, state_is_tuple=True) bw_cells = rnn.MultiRNNCell(bw_cells, state_is_tuple=True) # output outputs, _, _ = rnn.static_bidirectional_rnn( fw_cells, bw_cells, x, dtype=tf.float32, sequence_length=sequence_length) logit = inner_product(outputs[-1], 1) return logit, outputs
def generate_source_target(self, x, scope=None): with tf.variable_scope(scope): weights_st = weight_variable([2 * self.hidden_rnn * time_steps, 2 * self.hidden_rnn]) biases_st = bias_variable([2 * self.hidden_rnn]) if cell_type == 'lstm': if num_layers > 1: # define rnn-cell with tensor_flow # forward direction cell fw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.LSTMCell(self.hidden_rnn) for _ in range(num_layers)]) # backward direction cell bw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.LSTMCell(self.hidden_rnn) for _ in range(num_layers)]) else: fw_cell_st = rnn.LSTMCell(self.hidden_rnn) # backward direction cell bw_cell_st = rnn.LSTMCell(self.hidden_rnn) elif cell_type == 'gru': if num_layers > 1: fw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.GRUCell(self.hidden_rnn) for _ in range(num_layers)]) # backward direction cell bw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.GRUCell(self.hidden_rnn) for _ in range(num_layers)]) else: fw_cell_st = rnn.GRUCell(self.hidden_rnn) # backward direction cell bw_cell_st = rnn.GRUCell(self.hidden_rnn) else: if num_layers > 1: fw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.BasicRNNCell(self.hidden_rnn) for _ in range(num_layers)]) # backward direction cell bw_cell_st = tf.contrib.rnn.MultiRNNCell([rnn.BasicRNNCell(self.hidden_rnn) for _ in range(num_layers)]) else: fw_cell_st = rnn.BasicRNNCell(self.hidden_rnn) # backward direction cell bw_cell_st = rnn.BasicRNNCell(self.hidden_rnn) # get rnn-cell outputs l_outputs_st, a_st, b_st = rnn.static_bidirectional_rnn(fw_cell_st, bw_cell_st, x, dtype=tf.float32) l_outputs_st = tf.transpose(tf.stack(l_outputs_st, axis=0), perm=[1, 0, 2]) l_outputs_st = tf.reshape(l_outputs_st, [-1, 2 * self.hidden_rnn * time_steps]) outputs_st = tf.nn.tanh(tf.matmul(l_outputs_st, weights_st) + biases_st) lo_gits_st = tf.reshape(outputs_st, [-1, 2 * self.hidden_rnn]) ab_st = tf.concat((a_st[1], b_st[1]), axis=1) return lo_gits_st, ab_st
def Bi_RNN(X, W, B, nsteps, name): X = tf.transpose(X, [1, 0, 2]) X = tf.reshape(X, [-1, diminput]) H_1 = tf.matmul(X, W["w1"]) + b["b1"] H_1 = tf.split(H_1, nsteps, 0) lstm_fw_cell = tf.nn.rnn_cell.BasicLSTMCell(dimhidden, forget_bias=1.0) lstm_bw_cell = tf.nn.rnn_cell.BasicLSTMCell(dimhidden, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, H_1, dtype=tf.float32) return tf.matmul(outputs[-1], W["w2"]) + b["b2"]
def BiRNN(x, weights, biases): x = tf.reshape(x, [-1, N_INPUT]) x = tf.split(x, N_INPUT, 1) rnn_cell_fw = rnn.MultiRNNCell([getCell(N_HIDDEN) for _ in range(2)]) # rnn_cell_fw = getCell(N_HIDDEN) rnn_cell_bw = rnn.MultiRNNCell([getCell(N_HIDDEN) for _ in range(2)]) # rnn_cell_bw = getCell(N_HIDDEN) outputs, _, _ = rnn.static_bidirectional_rnn(rnn_cell_fw, rnn_cell_bw, x, dtype=tf.float32) return tf.matmul(outputs[-2], weights['out']) + biases['out']
def generate_rnn_output(self): """ Generate RNN state outputs with word embeddings as inputs """ with tf.variable_scope("generate_seq_output"): if self.bidirectional_rnn: embedding = tf.get_variable( "embedding", [self.source_vocab_size, self.word_embedding_size], initializer=initializers.xavier_initializer()) encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input) \ for encoder_input in self.encoder_inputs] rnn_outputs = static_bidirectional_rnn( self.cell_fw, self.cell_bw, encoder_emb_inputs, sequence_length=self.sequence_length, dtype=tf.float32) # tf.keras.layers.Bidirectional(tf.keras.layers.RNN(self.cell_fw, unroll=True)) encoder_outputs, encoder_state_fw, encoder_state_bw = rnn_outputs # with state_is_tuple = True, if num_layers > 1, # here we simply use the state from last layer as the encoder state state_fw = encoder_state_fw[-1] state_bw = encoder_state_bw[-1] encoder_state = tf.concat( [tf.concat(state_fw, 1), tf.concat(state_bw, 1)], 1) top_states = [tf.reshape(e, [-1, 1, self.cell_fw.output_size \ + self.cell_bw.output_size]) for e in encoder_outputs] attention_states = tf.concat(top_states, 1) else: embedding = tf.get_variable( "embedding", [self.source_vocab_size, self.word_embedding_size]) encoder_emb_inputs = list() encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input) \ for encoder_input in self.encoder_inputs] rnn_outputs = static_rnn(self.cell_fw, encoder_emb_inputs, sequence_length=self.sequence_length, dtype=tf.float32) encoder_outputs, encoder_state = rnn_outputs # with state_is_tuple = True, if num_layers > 1, # here we use the state from last layer as the encoder state state = encoder_state[-1] encoder_state = tf.concat(state, 1) top_states = [ tf.reshape(e, [-1, 1, self.cell_fw.output_size]) for e in encoder_outputs ] attention_states = tf.concat(top_states, 1) return encoder_outputs, encoder_state, attention_states
def BiRNN(x, weights, biases): x = tf.unstack(x, num=timesteps, axis=1) lstm_fw_cell = rnn.BasicLSTMCell(feature_hidden, forget_bias=1.0) lstm_bw_cell = rnn.BasicLSTMCell(feature_hidden, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return tf.matmul(outputs[-1], weights['out']) + biases['out']
def BiRNN(x, seqlen): inputs = tf.unstack(x, num=dataset.max_seqlen, axis=1) lstm_fw_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) lstm_bw_cell = rnn.BasicLSTMCell(num_hidden, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, inputs, dtype=tf.float32, sequence_length=seqlen) outputs = tf.transpose(tf.stack(outputs), perm=[1, 0, 2]) outputs = tf.reduce_max(outputs, axis=1) return outputs
def BiRnn(x): # Unstack to get a list of 'n_steps' tensors of shape (batch_size, embedding_size) x = tf.unstack(x, n_steps, 1) lstm_fw_cells = rnn.MultiRNNCell([attn_cell() for _ in range(num_layers)], state_is_tuple=True) lstm_bw_cells = rnn.MultiRNNCell([attn_cell() for _ in range(num_layers)], state_is_tuple=True) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cells, lstm_bw_cells, x, dtype=tf.float32) return outputs
def BiRNN(x, weights, biases): x = tf.unstack(x, n_steps, 1) lstm_fw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return tf.add(tf.matmul(outputs[-1], weights["out"]), biases["out"])
def generate_rnn_output(self): """ Generate RNN state outputs with word embeddings as inputs """ with tf.variable_scope("generate_seq_output"): if self.bidirectional_rnn: embedding = tf.get_variable("embedding", [self.source_vocab_size, self.word_embedding_size]) encoder_emb_inputs = list() encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input)\ for encoder_input in self.encoder_inputs] rnn_outputs = static_bidirectional_rnn(self.cell_fw, self.cell_bw, encoder_emb_inputs, sequence_length=self.sequence_length, dtype=tf.float32) encoder_outputs, encoder_state_fw, encoder_state_bw = rnn_outputs # with state_is_tuple = True, if num_layers > 1, # here we simply use the state from last layer as the encoder state state_fw = encoder_state_fw[-1] state_bw = encoder_state_bw[-1] encoder_state = tf.concat([tf.concat(state_fw, 1), tf.concat(state_bw, 1)], 1) top_states = [tf.reshape(e, [-1, 1, self.cell_fw.output_size \ + self.cell_bw.output_size]) for e in encoder_outputs] attention_states = tf.concat(top_states, 1) else: embedding = tf.get_variable("embedding", [self.source_vocab_size, self.word_embedding_size]) encoder_emb_inputs = list() encoder_emb_inputs = [tf.nn.embedding_lookup(embedding, encoder_input)\ for encoder_input in self.encoder_inputs] rnn_outputs = static_rnn(self.cell_fw, encoder_emb_inputs, sequence_length=self.sequence_length, dtype=tf.float32) encoder_outputs, encoder_state = rnn_outputs # with state_is_tuple = True, if num_layers > 1, # here we use the state from last layer as the encoder state state = encoder_state[-1] encoder_state = tf.concat(state, 1) top_states = [tf.reshape(e, [-1, 1, self.cell_fw.output_size]) for e in encoder_outputs] attention_states = tf.concat(top_states, 1) return encoder_outputs, encoder_state, attention_states
def generate_embedding_RNN_output(encoder_inputs, cell, num_encoder_symbols, word_embedding_size, num_heads=1, dtype=dtypes.float32, scope=None, initial_state_attention=False, sequence_length=None, bidirectional_rnn=False): """ Generate RNN state outputs with word embeddings as inputs - Note that this example code does not include output label dependency modeling. One may add a loop function as in the rnn_decoder function in tf seq2seq.py example to feed emitted label embedding back to RNN state. """ with variable_scope.variable_scope(scope or "generate_embedding_RNN_output"): if bidirectional_rnn: encoder_cell_fw = cell encoder_cell_bw = cell embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size]) encoder_embedded_inputs = list() encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs] encoder_outputs, encoder_state_fw, encoder_state_bw = rnn.static_bidirectional_rnn( encoder_cell_fw, encoder_cell_bw, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype) encoder_state = array_ops.concat(axis=1, values=[array_ops.concat(axis=1, values=encoder_state_fw), array_ops.concat(axis=1, values=encoder_state_bw)]) top_states = [array_ops.reshape(e, [-1, 1, cell.output_size*2]) for e in encoder_outputs] attention_states = array_ops.concat(axis=1, values=top_states) else: encoder_cell = cell embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size]) encoder_embedded_inputs = list() encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs] encoder_outputs, encoder_state = rnn.rnn( encoder_cell, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype) encoder_state = array_ops.concat(1, encoder_state) top_states = [array_ops.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs] attention_states = array_ops.concat(1, top_states) return encoder_outputs, encoder_state, attention_states