def BiRNN(x, weights, biases): # Prepare data shape to match `bidirectional_rnn` function requirements # Current data input shape: (batch_size, n_steps, n_input) # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) # Permuting batch_size and n_steps x = tf.transpose(x, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) x = tf.reshape(x, [-1, n_input]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.split(axis=0, num_or_size_splits=n_steps, value=x) # Define lstm cells with tensorflow # Forward direction cell lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Get lstm cell output try: outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) # Linear activation, using rnn inner loop last output return tf.matmul(outputs[-1], weights['out']) + biases['out']
def RNN(x, weights, biases): x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(0, n_steps, x) ''' lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32) return tf.matmul(outputs[-1], weights['out']) + biases['out'] ''' lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) try: outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) # Linear activation, using rnn inner loop last output return tf.matmul(outputs[-1], weights['out']) + biases['out']
def inference(images, eval=False): NUM_HIDDEN = 128 W = _variable_with_weight_decay('weight', [2 * NUM_HIDDEN, NUM_CLASSES], 0.001, 0.01) b = _variable_with_weight_decay('bias', [NUM_CLASSES], 0.001, 0) s = images.get_shape() n_batches, n_steps, n_features = int(s[0]), int(s[1]), int(s[2]) #print(n_batches, n_steps, n_features) inputs = tf.reshape(images, [-1, n_features]) inputs = tf.split(0, n_steps, inputs) fw_cell = rnn_cell.LSTMCell(NUM_HIDDEN, forget_bias=1.0, state_is_tuple=True) bw_cell = rnn_cell.LSTMCell(NUM_HIDDEN, forget_bias=1.0, state_is_tuple=True) try: outputs, _, _ = rnn.bidirectional_rnn(fw_cell, bw_cell, inputs, dtype=tf.float32) except Exception: outputs = rnn.bidirectional_rnn(fw_cell, bw_cell, x, dtype=tf.float32) logits = tf.matmul(outputs[-1], W) + b return logits
def BiRNN(x, weights, biases): # Prepare data shape to match `bidirectional_rnn` function requirements # Current data input shape: (batch_size, n_steps, n_input) # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) # Permuting batch_size and n_steps x = tf.transpose(x, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) x = tf.reshape(x, [-1, n_input]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.split(0, n_steps, x) # Define lstm cells with tensorflow # Forward direction cell lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Get lstm cell output try: outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) # Linear activation, using rnn inner loop last output return tf.matmul(outputs[-1], weights['out']) + biases['out']
def init_all(self, wr): self.word_reader = wr # initializer = tf.constant_initializer(value=wr.word_vectors, dtype=tf.float32) # self.word_dict = tf.get_variable('word_dict', shape=[len(wr.word_vectors), config.embedding_size], # initializer=initializer, trainable=config.trainable) self.word_dict = tf.get_variable('haha', shape=None, dtype=tf.float32, initializer=tf.constant(wr.word_vectors), trainable=False) # paraphrase sentences self.x1_index = tf.placeholder(tf.int32, [None, config.max_sentence_len]) self.x1 = tf.nn.embedding_lookup(self.word_dict, self.x1_index) # Permuting batch_size and n_steps self.x1 = tf.transpose(self.x1, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) self.x1 = tf.reshape(self.x1, [-1, config.embedding_size]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) self.x1 = tf.split(0, config.max_sentence_len, self.x1) self.x2_index = tf.placeholder(tf.int32, [None, config.max_sentence_len]) self.x2 = tf.nn.embedding_lookup(self.word_dict, self.x2_index) # Permuting batch_size and n_steps self.x2 = tf.transpose(self.x2, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) self.x2 = tf.reshape(self.x2, [-1, config.embedding_size]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) self.x2 = tf.split(0, config.max_sentence_len, self.x2) self.y = tf.placeholder("int64", [None]) # Forward direction cell lstm_fw_cell_x1 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True) # Backward direction cell lstm_bw_cell_x1 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True) # Forward direction cell lstm_fw_cell_x2 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True) # Backward direction cell lstm_bw_cell_x2 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True) # Get lstm cell output outputs_x1, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_x1, lstm_bw_cell_x1, self.x1, dtype=tf.float32, scope="RNN1") outputs_x2, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_x2, lstm_bw_cell_x2, self.x2, dtype=tf.float32, scope="RNN2") outputs = tf.concat(1, [outputs_x1[-1], outputs_x2[-1]]) self.weights = tf.Variable(tf.random_uniform([config.hidden_size * 4, config.classes], dtype=tf.float32)) # self.weights = tf.Variable(tf.random_uniform([config.embedding_size * 2, config.classes], # dtype=tf.float32)) self.b = tf.Variable(tf.random_uniform([config.classes]), dtype=tf.float32) mid = tf.matmul(outputs, self.weights) + self.b self.results = tf.nn.softmax(mid) self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(self.results, self.y) self.cost = tf.reduce_sum(self.loss) init = tf.initialize_all_variables() vars = tf.trainable_variables() opt=tf.train.GradientDescentOptimizer(learning_rate=config.learning_rate) self.optimizer = opt.minimize(self.cost, var_list=vars) self.sess = tf.Session() self.sess.run(init) return
def inference(inputs, n_input, n_steps, n_hidden, n_classes): W = tf.Variable(tf.random_normal([2*n_hidden, n_classes])) b = tf.Variable(tf.random_normal([n_classes])) inputs = tf.reshape(inputs, [-1, n_input]) inputs = tf.split(0, n_steps, inputs) fw_cell = rnn_cell.LSTMCell(n_hidden, forget_bias = 1.0, state_is_tuple = True) bw_cell = rnn_cell.LSTMCell(n_hidden, forget_bias = 1.0, state_is_tuple = True) try: outputs,_,_ = rnn.bidirectional_rnn(fw_cell, bw_cell, inputs, dtype = tf.float32) except Exception: outputs = rnn.bidirectional_rnn(fw_cell, bw_cell, x, dtype = tf.float32) return tf.matmul(outputs[-1], W) + b
def build_similarity_calculator(self): question = tf.placeholder(tf.int32, [1, self.len_question]) question_sequence_length = tf.constant([1], dtype=tf.int64) answer = tf.placeholder(tf.int32, [1, self.len_answer]) answer_sequence_length = tf.constant([1], dtype=tf.int64) lstm_state_fw_Q = tf.zeros([1, self.lstm_fw_Q.state_size]) lstm_state_bw_Q = tf.zeros([1, self.lstm_bw_Q.state_size]) lstm_state_fw_A = tf.zeros([1, self.lstm_fw_A.state_size]) lstm_state_bw_A = tf.zeros([1, self.lstm_bw_A.state_size]) with tf.device("/cpu:0"): embedded_question = tf.nn.embedding_lookup(self.Wemb, tf.transpose(question)) embedded_answer = tf.nn.embedding_lookup(self.Wemb, tf.transpose(answer)) question_brnn_output = rnn.bidirectional_rnn( self.lstm_fw_Q, self.lstm_bw_Q, tf.unpack(embedded_question), sequence_length=question_sequence_length, initial_state_fw=lstm_state_fw_Q, initial_state_bw=lstm_state_bw_Q, ) question_pooled_output = tf.reduce_mean(tf.pack(question_brnn_output), 0) question_final_emb = tf.nn.xw_plus_b(question_pooled_output, self.W_Q_emb, self.b_Q_emb) question_final_emb = tf.nn.l2_normalize(question_final_emb, dim=1) answer_brnn_output = rnn.bidirectional_rnn( self.lstm_fw_A, self.lstm_bw_A, tf.unpack(embedded_answer), sequence_length=answer_sequence_length, initial_state_fw=lstm_state_fw_A, initial_state_bw=lstm_state_bw_A) answer_brnn_pooled_output = tf.reduce_mean(tf.pack(answer_brnn_output), 0) answer_final_emb = tf.nn.xw_plus_b(answer_brnn_pooled_output, self.W_A_emb, self.b_A_emb) answer_final_emb = tf.nn.l2_normalize(answer_final_emb, dim=1) similarity = tf.reduce_mean( tf.matmul(question_final_emb, tf.transpose(answer_final_emb))) return question, question_sequence_length, answer, answer_sequence_length, similarity
def bidirectional_lstm_inference(x): RNN_HIDDEN_UNITS = 128 # x was [BATCH_SIZE, 32, 32, 3] # x changes to [32, BATCH_SIZE, 32, 3] x = tf.transpose(x, [1, 0, 2, 3]) # x changes to [32 * BATCH_SIZE, 32 * 3] x = tf.reshape(x, [-1, IMAGE_SIZE * RGB_CHANNEL_SIZE]) # x changes to array of 32 * [BATCH_SIZE, 32 * 3] x = tf.split(0, IMAGE_SIZE, x) weights = tf.Variable( tf.random_normal([2 * RNN_HIDDEN_UNITS, LABEL_SIZE])) biases = tf.Variable(tf.random_normal([LABEL_SIZE])) # output size is 128, state size is (c=128, h=128) fw_lstm_cell = rnn_cell.BasicLSTMCell(RNN_HIDDEN_UNITS, forget_bias=1.0) bw_lstm_cell = rnn_cell.BasicLSTMCell(RNN_HIDDEN_UNITS, forget_bias=1.0) # outputs is array of 32 * [BATCH_SIZE, 128] outputs, _, _ = rnn.bidirectional_rnn(fw_lstm_cell, bw_lstm_cell, x, dtype=tf.float32) # outputs[-1] is [BATCH_SIZE, 128] return tf.matmul(outputs[-1], weights) + biases
def BILSTM(x, hidden_size): # biLSTM: # 功能:添加bidirectional_lstm操作 # 参数: # x: [batch, height, width] / [batch, step, embedding_size] # hidden_size: lstm隐藏层节点个数 # 输出: # output: [batch, height, 2*hidden_size] / [batch, step, 2*hidden_size] # input transformation input_x = tf.transpose(x, [1, 0, 2]) # input_x = tf.reshape(input_x, [-1, w]) # input_x = tf.split(0, h, input_x) input_x = tf.unpack(input_x) # define the forward and backward lstm cells gru_fw_cell = rnn_cell.GRUCell(hidden_size) gru_bw_cell = rnn_cell.GRUCell(hidden_size) output, _, _ = rnn.bidirectional_rnn(gru_fw_cell, gru_bw_cell, input_x, dtype=tf.float32) # output transformation to the original tensor type output = tf.pack(output) output = tf.transpose(output, [1, 0, 2]) return output
def BiRNN(x, f_cell, b_cell): x = tf.unstack(x, n_words, 2) outputs, _, _ = rnn.bidirectional_rnn(f_cell, b_cell, x, dtype=tf.float32) return outputs[-1]
def bidir_ltsm(x): with tf.name_scope('Weights'): # Permuting batch_size and n_steps #x = tf.transpose(x, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) x = tf.reshape(x, [-1, model.num_features]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.split(0, model.max_timesteps, x) weights_out1 = tf.Variable(tf.truncated_normal([2, num_hidden], stddev=np.sqrt(1./num_hidden)), name='weights') biases_out1 = tf.Variable(tf.zeros(num_hidden), name='biases') weights_out2 = tf.Variable(tf.truncated_normal([2, num_hidden], stddev=np.sqrt(1./num_hidden)), name='weights') biases_out2 = tf.Variable(tf.zeros(num_hidden), name='biases') with tf.name_scope('LTSM'): forward = rnn_cell.LSTMCell(num_hidden, use_peepholes=True, forget_bias=1.0) backward = rnn_cell.LSTMCell(num_hidden, use_peepholes=True, forget_bias=1.0) with tf.name_scope('Bidirectionrnn'): bidirectional_h1, _, _ = bidirectional_rnn(forward, backward, x, dtype=tf.float32) bd_h1s = [tf.reshape(t, [batch_size, 2, num_hidden]) for t in bidirectional_h1] with tf.name_scope('logits'): weights_class = tf.Variable(tf.truncated_normal([num_hidden, model.num_classes], stddev=np.sqrt(1./num_hidden)), name='weights') biases_class = tf.Variable(tf.zeros([model.num_classes])) out_h1 = [tf.reduce_sum(tf.mul(t, weights_out1), reduction_indices=1) + biases_out1 for t in bd_h1s] logits = [tf.matmul(t, weights_class) + biases_class for t in out_h1] logits3d = tf.pack(logits) return logits3d
def embedding_encoder(encoder_inputs, cell, embedding, num_symbols, embedding_size, bidirectional=False, dtype=None, weight_initializer=None, scope=None): with variable_scope.variable_scope(scope or "embedding_encoder", dtype=dtype) as scope: dtype = scope.dtype # Encoder. if not embedding: embedding = variable_scope.get_variable( "embedding", [num_symbols, embedding_size], initializer=weight_initializer()) emb_inp = [ embedding_ops.embedding_lookup(embedding, i) for i in encoder_inputs ] if bidirectional: _, output_state_fw, output_state_bw = rnn.bidirectional_rnn( cell, cell, emb_inp, dtype=dtype) encoder_state = tf.concat(1, [output_state_fw, output_state_bw]) else: _, encoder_state = rnn.rnn(cell, emb_inp, dtype=dtype) return encoder_state
def _tf_enc_embedding_attention_seq2seq(self, encoder_inputs, cell, num_encoder_symbols, embedding_size, num_heads=1, dtype=dtypes.float32, scope=None, encoder="reverse", sequence_length=None, bucket_length=None, init_backward=False, bow_emb_size=None, single_src_embedding=False): """Embedding sequence-to-sequence model with attention. """ with tf.variable_scope(scope or "embedding_attention_seq2seq", reuse=True): # Encoder. if encoder == "bidirectional": encoder_cell_fw = rnn_cell.EmbeddingWrapper( cell.get_fw_cell(), embedding_classes=num_encoder_symbols, embedding_size=embedding_size) embed_scope = None if single_src_embedding: logging.info("Reuse forward src embedding for backward encoder") with variable_scope.variable_scope("BiRNN/FW/EmbeddingWrapper") as es: embed_scope = es encoder_cell_bw = rnn_cell.EmbeddingWrapper( cell.get_bw_cell(), embedding_classes=num_encoder_symbols, embedding_size=embedding_size, embed_scope=embed_scope) encoder_outputs, encoder_state, encoder_state_bw = rnn.bidirectional_rnn(encoder_cell_fw, encoder_cell_bw, encoder_inputs, dtype=dtype, sequence_length=sequence_length, bucket_length=bucket_length) logging.debug("Bidirectional state size=%d" % cell.state_size) # this shows double the size for lstms elif encoder == "reverse": encoder_cell = rnn_cell.EmbeddingWrapper( cell, embedding_classes=num_encoder_symbols, embedding_size=embedding_size) encoder_outputs, encoder_state = rnn.rnn( encoder_cell, encoder_inputs, dtype=dtype, sequence_length=sequence_length, bucket_length=bucket_length, reverse=True) logging.debug("Unidirectional state size=%d" % cell.state_size) elif encoder == "bow": encoder_outputs, encoder_state = cell.embed(rnn_cell.Embedder, num_encoder_symbols, bow_emb_size, encoder_inputs, dtype=dtype) # First calculate a concatenation of encoder outputs to put attention on. if encoder == "bow": top_states = [array_ops.reshape(e, [-1, 1, bow_emb_size]) for e in encoder_outputs] else: top_states = [array_ops.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs] attention_states = array_ops.concat(1, top_states) initial_state = encoder_state if encoder == "bidirectional" and init_backward: initial_state = encoder_state_bw return self._tf_enc_embedding_attention_decoder( attention_states, initial_state, cell, num_heads=num_heads)
def compute_states(self,emb): def unpack_sequence(tensor): return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2])) with tf.variable_scope("Composition",initializer= tf.contrib.layers.xavier_initializer(),regularizer= tf.contrib.layers.l2_regularizer(self.reg)): cell_fw = rnn_cell.LSTMCell(self.hidden_dim) cell_bw = rnn_cell.LSTMCell(self.hidden_dim) #tf.cond(tf.less(self.dropout #if tf.less(self.dropout, tf.constant(1.0)): cell_fw = rnn_cell.DropoutWrapper(cell_fw, output_keep_prob=self.dropout,input_keep_prob=self.dropout) cell_bw=rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=self.dropout,input_keep_prob=self.dropout) #output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32) outputs,_,_=rnn.bidirectional_rnn(cell_fw,cell_bw,unpack_sequence(emb),sequence_length=self.lngths,dtype=tf.float32) #output = pack_sequence(outputs) sum_out=tf.reduce_sum(tf.pack(outputs),[0]) sent_rep = tf.div(sum_out,tf.expand_dims(tf.to_float(self.lngths),1)) final_state=sent_rep return final_state
def LSTM_Network(input_, weightsOutH1, weightsClasses, biasesOutH1, biasesClasses): ####Network forwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True) backwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True) fbH1, _, _ = bidirectional_rnn(forwardH1, backwardH1, inputList, dtype=tf.float32, scope='BDLSTM_H1') fbH1rs = [tf.reshape(t, [batchSize, 2, nHidden]) for t in fbH1] outH1 = [ tf.reduce_sum(tf.mul(t, weightsOutH1), reduction_indices=1) + biasesOutH1 for t in fbH1rs ] logits = [tf.matmul(t, weightsClasses) + biasesClasses for t in outH1] ####Optimizing logits3d = tf.pack(logits) return logits3d
def compute_states(self, emb): def unpack_sequence(tensor): return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2])) with tf.variable_scope( "Composition", initializer=tf.contrib.layers.xavier_initializer(), regularizer=tf.contrib.layers.l2_regularizer(self.reg)): cell_fw = rnn_cell.LSTMCell(self.hidden_dim) cell_bw = rnn_cell.LSTMCell(self.hidden_dim) #tf.cond(tf.less(self.dropout #if tf.less(self.dropout, tf.constant(1.0)): cell_fw = rnn_cell.DropoutWrapper(cell_fw, output_keep_prob=self.dropout, input_keep_prob=self.dropout) cell_bw = rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=self.dropout, input_keep_prob=self.dropout) #output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32) outputs, _, _ = rnn.bidirectional_rnn(cell_fw, cell_bw, unpack_sequence(emb), sequence_length=self.lngths, dtype=tf.float32) #output = pack_sequence(outputs) sum_out = tf.reduce_sum(tf.pack(outputs), [0]) sent_rep = tf.div(sum_out, tf.expand_dims(tf.to_float(self.lngths), 1)) final_state = sent_rep return final_state
def get_similarity(one_answer, one_answer_sequence): tf.get_variable_scope().reuse_variables() with tf.device("/cpu:0"): embedded_answer = tf.nn.embedding_lookup( self.Wemb, tf.transpose(one_answer)) answer_output = rnn.bidirectional_rnn( self.lstm_fw_A, self.lstm_bw_A, tf.unpack(embedded_answer), sequence_length=one_answer_sequence, initial_state_fw=lstm_state_fw_A, initial_state_bw=lstm_state_bw_A) answer_pooled_output = tf.reduce_sum(tf.pack(answer_output), 0) answer_pooled_output = answer_pooled_output / tf.expand_dims( tf.to_float(one_answer_sequence) + 1e-6, 1) answer_final_emb = tf.nn.xw_plus_b(answer_pooled_output, self.W_A_emb, self.b_A_emb) answer_final_emb = tf.nn.l2_normalize(answer_final_emb, dim=1, epsilon=1e-7) similarity_matrix = tf.matmul(question_final_emb, tf.transpose(answer_final_emb)) one_diagonal = tf.diag([1.] * self.batch_size) similarity = tf.reduce_sum(similarity_matrix * one_diagonal) return similarity
def BiRNN(x, f_cell, b_cell): x = tf.unstack(x, n_words, 2) outputs, _, _ = rnn.bidirectional_rnn(f_cell, b_cell, x, dtype=tf.float32) return tf.pack(tf.transpose(outputs, [1, 0, 2]))
def embedding_encoder(encoder_inputs, cell, embedding, num_symbols, embedding_size, bidirectional=False, dtype=None, weight_initializer=None, scope=None): with variable_scope.variable_scope( scope or "embedding_encoder", dtype=dtype) as scope: dtype = scope.dtype # Encoder. if not embedding: embedding = variable_scope.get_variable("embedding", [num_symbols, embedding_size], initializer=weight_initializer()) emb_inp = [embedding_ops.embedding_lookup(embedding, i) for i in encoder_inputs] if bidirectional: _, output_state_fw, output_state_bw = rnn.bidirectional_rnn(cell, cell, emb_inp, dtype=dtype) encoder_state = tf.concat(1, [output_state_fw, output_state_bw]) else: _, encoder_state = rnn.rnn( cell, emb_inp, dtype=dtype) return encoder_state
def BiRNN(x, weights, biases): # shape of x will be (batch_size, n_steps, n_channels) # need to transform to (n_steps, batch_size, n_channels) to comply with tf bi_rnn x = tf.transpose(x, [1, 0, 2]) x = tf.unpack(x, axis = 0) # Forward direction cell lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) try: outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32) except Exception: outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32) return tf.matmul(outputs[-1], weights['out']) + biases['out']
def BiRNN(x, weights, biases): # Prepare data shape to match `bidirectional_rnn` function requirements # Current data input shape: (batch_size, n_steps, n_input) # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) # Permuting batch_size and n_steps x = tf.transpose(x, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) x = tf.reshape(x, [-1, n_input]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.split(0, n_steps, x) # Define lstm cells with tensorflow with tf.variable_scope("lstm1") as scope1: lstm_fw_cell_1 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell_1 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs_1, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_1, lstm_bw_cell_1, x, dtype=tf.float32) with tf.variable_scope("lstm2") as scope2: lstm_fw_cell_2 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell_2 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs_2, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_2, lstm_bw_cell_2, outputs_1, dtype=tf.float32) with tf.variable_scope("lstm3") as scope3: lstm_fw_cell_3 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell_3 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs_3, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_3, lstm_bw_cell_3, outputs_2, dtype=tf.float32) outputs = outputs_3 outputs = tf.reshape(tf.concat(0, outputs), [MAX_LEN * BATCH_SIZE, n_hidden * 2]) # Linear activation, using rnn inner loop last output return tf.matmul(outputs, weights['out']) + biases['out']
def multiRNN(x, weights, biases, num_layers): # Permuting batch_size and n_steps x = tf.transpose(x, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) x = tf.reshape(x, [-1, n_input]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.split(0, n_steps, x) # # x = tf.transpose(tf.pack(x), perm=[1,0,2]) # print(x) initializer = tf.random_uniform_initializer(-1, 1) cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_hidden) cell = tf.nn.rnn_cell.DropoutWrapper(cell=cell, output_keep_prob=0.5) cell = tf.nn.rnn_cell.MultiRNNCell(cells=[cell] * num_layers) cell2 = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_hidden) cell2 = tf.nn.rnn_cell.DropoutWrapper(cell=cell2, output_keep_prob=0.5) cell2 = tf.nn.rnn_cell.MultiRNNCell(cells=[cell2] * num_layers) # init_state = multicell.zero_state(batch_size, tf.float32) # # Get lstm cell output # try: # outputs, _, _ = rnn.bidirectional_rnn(cell, cell, x, # dtype=tf.float32) # except Exception: # Old TensorFlow version only returns outputs not states # outputs = rnn.bidirectional_rnn(cell, cell, x, # dtype=tf.float32) # try: # outputs, _, _ = rnn.rnn(multicell, x, dtype=tf.float32) # except Exception: # Old TensorFlow version only returns outputs not states # outputs = rnn.rnn(multicell, x, dtype=tf.float32) output, _, _ = rnn.bidirectional_rnn(cell, cell2, x, dtype=tf.float32) # output = tf.transpose(output, [1, 0, 2]) # Linear activation, using rnn inner loop last output # return tf.matmul(rnn_outputs[-1], weights['out']) + biases['out'] # return tf.matmul(rnn_outputs[-1], W) + b # output = tf.transpose(output, [1, 0, 2]) # last = tf.gather(output, int(output.get_shape()[0]) - 1) # # Softmax layer. # weight = tf.Variable(tf.truncated_normal([n_hidden, y.get_shape()[1]], stddev=0.01)) # bias = tf.Variable(tf.constant(0.1, shape=[y.get_shape()[1]])) # prediction = tf.nn.softmax(tf.matmul(last, weight) + bias) pred = tf.matmul(output[-1], weights['out']) + biases['out'] return pred
def RNN(x, weights, biases): x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(0, n_steps, x) lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) outputs_all = tf.concat(0, outputs) # (N*batch) * 2*n_hidden M = tanh(tf.matmul(outputs_all, W_h)) # (N*batch) * 2*hidden M_2 = tanh(tf.matmul(outputs_all, W_2_h)) a = tf.nn.softmax(tf.matmul(M, w)) # (N*batch) * 1 a = tf.reshape(a, [n_steps, -1, 1]) # N*batch*1 a = tf.transpose(a, [1, 2, 0]) # batch*1*N a_2 = tf.nn.softmax(tf.matmul(M_2, w_2)) # (N*batch) * 1 a_2 = tf.reshape(a_2, [n_steps, -1, 1]) # N*batch*1 a_2 = tf.transpose(a_2, [1, 2, 0]) # batch*1*N outputs_all = tf.reshape(outputs_all, [n_steps, -1, 2 * n_hidden]) # N*batch*d outputs_all = tf.transpose(outputs_all, [1, 0, 2]) # batch*N*d batch_s = 32 a = tf.split(0, batch_s, a) a_2 = tf.split(0, batch_s, a_2) outputs_all = tf.split(0, batch_s, outputs_all) r = [] r_2 = [] for i in range(batch_s): a_temp = a[i][0:1, :, :] a_2_temp = a_2[i][0:1, :, :] o_temp = outputs_all[i][0:1, :, :] o_2_temp = outputs_all[i][0:1, :, :] att = tf.reshape(a_temp, [1, n_steps]) out = tf.reshape(o_temp, [n_steps, 2 * n_hidden]) att_2 = tf.reshape(a_2_temp, [1, n_steps]) out_2 = tf.reshape(o_2_temp, [n_steps, 2 * n_hidden]) r.append(tf.matmul(att, out)) r_2.append(tf.matmul(att_2, out_2)) r = tf.concat(0, r) # batch*d r_2 = tf.concat(0, r_2) # batch*d #attention_4 = tf.matmul(r,a) _h = tanh(W_p * r + W_2_p * r_2 + W_x * outputs[-1]) predict = tf.matmul(_h, weights['out']) + biases['out'] return predict, outputs
def BIGRU(x, hidden_size): input_x = tf.transpose(x, [1, 0, 2]) input_x = tf.unstack(input_x) # define the forward and backward lstm cells gru_fw_cell = rnn_cell.GRUCell(hidden_size) gru_bw_cell = rnn_cell.GRUCell(hidden_size) output, _, _ = rnn.bidirectional_rnn(gru_fw_cell, gru_bw_cell, input_x, dtype=tf.float32) # output transformation to the original tensor type output = tf.stack(output) output = tf.transpose(output, [1, 0, 2]) return output
def create_lstm_seq(input, length_labels, units_size, batch_size=32, num_rnn_layers=None, out_activation_func=None): """ LSTM with input of (batch_size, width as n_steps, and height*feature size) """ batch_size_d, height_d, width_d, feature_size_d = input.get_shape() height, width, feature_size = height_d.value, width_d.value, feature_size_d.value new_feature_size = feature_size*height n_steps = width input = tf.transpose(input, [0, 2, 1, 3]) input = tf.reshape(input, [-1, n_steps, new_feature_size]) input = tf.transpose(input, [1, 0, 2]) input = tf.reshape(input, [-1, new_feature_size]) inputs = tf.split(0, n_steps, input) print 'New input shape', inputs[0].get_shape() print 'number of steps', len(inputs) lstm_fw_cell = rnn_cell.BasicLSTMCell(units_size, state_is_tuple=True) lstm_bw_cell = rnn_cell.BasicLSTMCell(units_size, state_is_tuple=True) if num_rnn_layers is not None: lstm_fw_cell = rnn_cell.MultiRNNCell([lstm_fw_cell] * num_rnn_layers, state_is_tuple=True) lstm_bw_cell = rnn_cell.MultiRNNCell([lstm_bw_cell] * num_rnn_layers, state_is_tuple=True) try: outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, inputs, sequence_length=None, dtype=tf.float32) except Exception: # Old TensorFlow version only returns outputs not states outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, inputs, dtype=tf.float32) if out_activation_func is not None: outputs = [out_activation_func(o) for o in outputs] print 'New output shape', tf.pack(outputs).get_shape() return locals()
def RNN(x, weights, biases): x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(0, n_steps, x) lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) predict = tf.matmul(outputs[-1], weights['out']) + biases['out'] return predict, outputs
def RNN(x, weights, biases): x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(0, n_steps, x) lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) outputs_all = tf.concat(0, outputs) # (N*batch) * 2*n_hidden M = tanh(tf.matmul(outputs_all, W_h)) # (N*batch) * 2*hidden a = tf.nn.softmax(tf.matmul(M, w)) # (N*batch) * 1 a = tf.reshape(a, [n_steps, -1, 1]) # N*batch*1 a = tf.transpose(a, [1, 2, 0]) # batch*1*N #attention_2 = tf.squeeze(attention_2) # batch*N #attention_2 = tf.reshape(attention_2, [-1,n_steps]) # batch*N outputs_all = tf.reshape(outputs_all, [n_steps, -1, 2 * n_hidden]) # N*batch*d outputs_all = tf.transpose(outputs_all, [1, 0, 2]) # batch*N*d #batch_s = x[0].get_shape()[0] #batch_s = array_ops.shape(x[0])[0] #batch_s = tf.int32(tf.size(x[0])) #batch_s = tf.cast(batch_s,int) #print(batch_s) batch_s = 100 a = tf.split(0, batch_s, a) outputs_all = tf.split(0, batch_s, outputs_all) ''' o_2 = outputs_all[128][0:1,:,:] return o_2 ''' r = [] for i in range(batch_s): a_2 = a[i][0:1, :, :] o_2 = outputs_all[i][0:1, :, :] att = tf.reshape(a_2, [1, n_steps]) out = tf.reshape(o_2, [n_steps, 2 * n_hidden]) r.append(tf.matmul(att, out)) r = tf.concat(0, r) # batch*d #attention_4 = tf.matmul(r,a) _h = tanh(W_p * r + W_x * outputs[-1]) predict = tf.matmul(_h, weights['out']) + biases['out'] return predict, outputs
def utterance_encoder(self, enc_inputs): # for the utterance level encoder: enc_inputs is of dimension (max_utter, batch_size, cell_size+max_images*image_embedding_size) utterance_states = None with tf.variable_scope(self.enc_scope_utter) as scope: # init_state = self.enc_cells_utter.zero_state(self.batch_size, tf.float32) # enc_inputs is of dimension (max_utter, batch_size, cell_size+max_images*image_embedding_size) # _, states = rnn.rnn(self.enc_cells_utter, enc_inputs, scope=scope, dtype=tf.float32) # _, states, _ = rnn.bidirectional_dynamic_rnn(self.enc_cells_utter[0], self.enc_cells_utter[1], enc_inputs, dtype=tf.float32, scope=scope) _, states, _ = rnn.bidirectional_rnn(self.enc_cells_utter[0], self.enc_cells_utter[1], enc_inputs, dtype=tf.float32, scope=scope) # rnn.rnn takes a max_utter sized list of tensors of dimension (batch_size * cell_size+(max_images*image_embedding_size)) utterance_states = states # utterance_states is of dimension (batch_size, cell_size) # self.tf_print(utterance_states) return utterance_states
def sentence_embedding(self, inputs, keep_prob, w): with tf.device('/cpu:0'): embedding_layer = tf.nn.embedding_lookup(w['word_embedding_w'],inputs) # batch_size x max_len x word_embedding cell_input = tf.transpose(embedding_layer,[1,0,2]) cell_input = tf.reshape(cell_input,[-1,self.hiddensize]) cell_input = tf.split(0,self.max_len,cell_input) with tf.variable_scope('forward'): lstm_fw_cell = rnn_cell.DropoutWrapper(rnn_cell.BasicLSTMCell(self.rnnsize,forget_bias=1.0,state_is_tuple=True),input_keep_prob=keep_prob,output_keep_prob=keep_prob) with tf.variable_scope('backward'): lstm_bw_cell = rnn_cell.DropoutWrapper(rnn_cell.BasicLSTMCell(self.rnnsize,forget_bias=1.0,state_is_tuple=True),input_keep_prob=keep_prob,output_keep_prob=keep_prob) outputs,_,_ = rnn.bidirectional_rnn(lstm_fw_cell,lstm_bw_cell,cell_input,dtype=tf.float32) # outputs shape: seq_len x [batch_size x (fw_cell_size + bw_cell_size)] att = self.attention_layer(outputs,w) return att
def BiRNN(x, weights, biases): #具体过程参照test.py x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(0, n_steps, x) #lstm cells #forward direction cell lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) #backward direction cell lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) #get lstm cell output try: outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) except Exception as e: outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, dtype=tf.float32) return tf.matmul(outputs[-1], weights['out']) + biases['out']
def hierarchical_encoder(self): '''Acts as encoder for word and utterance level. Return - enc_states which is final output of utterance level encoder RNN which is of size batch_size*self.cell_size.''' enc_states = self.encoder1_inputs + self.encoder2_inputs #Merging self.encoder1_inputs and self.encoder2_inputs list. n_steps = self.max_len #Word level encoder steps is same as self.max_len which is 150. #Calling self.encoder function, for word level encoder. enc_states = self.encoder(enc_states, self.enc_cells[0], self.enc_scopes[0], n_steps) #Using bidirectional RNN for utterance level encoder. _, utterance_state, _ = rnn.bidirectional_rnn(self.enc_cells[1][0], self.enc_cells[1][1], enc_states, dtype=tf.float32) return utterance_state
def bilstm(cell, x, scope=None): with tf.variable_scope(scope or "birnn") as scope: # input transformation input_x = tf.transpose(x, [1, 0, 2]) input_x = tf.unpack(input_x) output, _, _ = rnn.bidirectional_rnn(cell, cell, input_x, dtype=tf.float32) # output transformation to the original tensor type output = tf.pack(output) output = tf.transpose(output, [1, 0, 2]) return output
def RNN(x, weights, biases): x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, n_input]) x = tf.split(0, n_steps, x) gru_fw_cell = rnn_cell.GRUCell(n_hidden) gru_fw_cell = tf.nn.rnn_cell.DropoutWrapper(gru_fw_cell, output_keep_prob=0.7) gru_bw_cell = rnn_cell.GRUCell(n_hidden) gru_bw_cell = tf.nn.rnn_cell.DropoutWrapper(gru_bw_cell, output_keep_prob=0.7) outputs, _, _ = rnn.bidirectional_rnn(gru_fw_cell, gru_bw_cell, x, dtype=tf.float32) outputs[-1] = tf.nn.dropout(outputs[-1], keep_prob=0.25) predict = tf.matmul(outputs[-1], weights['out']) + biases['out'] return predict, outputs
def BiRNN(_X, _istate_fw, _istate_bw, _weights, _biases): _X = tf.transpose(_X, [1, 0, 2]) _X = tf.reshape(_X, [-1, n_input]) fw_cell_1 = rnn_cell.BasicRNNCell(n_hidden) bw_cell_1 = rnn_cell.BasicRNNCell(n_hidden) fw_cell=rnn_cell.MultiRNNCell([fw_cell_1]*num_layers) bw_cell=rnn_cell.MultiRNNCell([bw_cell_1]*num_layers) _X = tf.split(0, n_steps, _X) seq=np.int32(np.ones(batch_size)*Truncated) outputs, statefw,statebw = rnn.bidirectional_rnn(fw_cell, bw_cell, _X, initial_state_fw=_istate_fw, initial_state_bw=_istate_bw, sequence_length=seq) return tf.matmul(outputs[-1], _weights['out']) + _biases['out']
def BiRNN(x, rnn_size, embed_size, n_steps, reuse=False): with tf.variable_scope("lstm"): if reuse: tf.get_variable_scope().reuse_variables() x = tf.transpose(x, [1, 0, 2]) x = tf.reshape(x, [-1, embed_size]) x = tf.split(0, n_steps, x) lstm_fw_cell = rnn_cell.BasicLSTMCell(rnn_size, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(rnn_size, forget_bias=1.0) outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) res = outputs[-1] return res
def generate_embedding_RNN_output(encoder_inputs, cell, num_encoder_symbols, word_embedding_size, num_heads=1, dtype=dtypes.float32, scope=None, initial_state_attention=False, sequence_length=None, bidirectional_rnn=False): """ Generate RNN state outputs with word embeddings as inputs - Note that this example code does not include output label dependency modeling. One may add a loop function as in the rnn_decoder function in tf seq2seq.py example to feed emitted label embedding back to RNN state. """ with variable_scope.variable_scope(scope or "generate_embedding_RNN_output"): if bidirectional_rnn: encoder_cell_fw = cell encoder_cell_bw = cell embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size]) encoder_embedded_inputs = list() encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs] encoder_outputs, encoder_state_fw, encoder_state_bw = rnn.bidirectional_rnn( encoder_cell_fw, encoder_cell_bw, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype) encoder_state = array_ops.concat(1, [encoder_state_fw, encoder_state_bw]) top_states = [array_ops.reshape(e, [-1, 1, cell.output_size*2]) for e in encoder_outputs] attention_states = array_ops.concat(1, top_states) else: encoder_cell = cell embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size]) encoder_embedded_inputs = list() encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs] encoder_outputs, encoder_state = rnn.rnn( encoder_cell, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype) top_states = [array_ops.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs] attention_states = array_ops.concat(1, top_states) return encoder_outputs, encoder_state, attention_states
def __init__(self, embedding, max_length, cell, num_symbols, dtype, **kwargs): self.embedding = embedding self.cell = cell # account for _GO and _EOS max_length += 2 self.lengths = kwargs.get( 'lengths', tf.placeholder(tf.int32, shape=[None], name="encoder_lengths")) self.inputs = kwargs.get('inputs', [ tf.placeholder( tf.int32, shape=[None], name="encoder_input{0}".format(i)) for i in xrange(max_length) ]) self.weights = kwargs.get('weights', [ tf.placeholder( tf.float32, shape=[None], name="encoder_weight{0}".format(i)) for i in xrange(max_length) ]) inputs = [ embedding_ops.embedding_lookup(embedding, i) for i in self.inputs ] self.outputs, self.state_fw, self.state_bw = rnn.bidirectional_rnn( self.cell, self.cell, inputs, sequence_length=self.lengths, dtype=dtype) #self.outputs = [tf.add(*tf.split(1, 2, o)) for o in self.outputs] self.state = self.state_fw + self.state_bw # aggregate fw+bw states #self.state = tf.concat(1, [self.state_fw, self.state_bw]) # concatenate fw+bw states top_states = [ array_ops.reshape(e, [-1, 1, cell.output_size]) for e in self.outputs ] self.attention_states = array_ops.concat(1, top_states)
def BiRNN(x, n_input, n_steps, n_hidden): # Prepare data shape to match `bidirectional_rnn` function requirements # Current data input shape: (batch_size, n_steps, n_input) # Required shape: 'n_steps' tensors list of shape (batch_size, n_input) # Permuting batch_size and n_steps x = tf.transpose(x, [1, 0, 2]) # Reshape to (n_steps*batch_size, n_input) x = tf.reshape(x, [-1, n_input]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_input) x = tf.split(0, n_steps, x) # Define lstm cells with tensorflow # Forward direction cell lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Backward direction cell lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0) # Get lstm cell output outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype=tf.float32) return outputs
def runCTC(batch): INPUT_PATH = '../TRAIN/All/mfcc/' #directory of MFCC nFeatures x nFrames 2-D array .npy files TARGET_PATH = '../TRAIN/All/phone_y/' #directory of nPhonemes 1-D array .npy files ####Learning Parameters learningRate = 0.001 momentum = 0.9 nEpochs = 300 batchSize = batch.shape[1] ####Network Parameters nFeatures = 39 #12 MFCC coefficients + energy, and derivatives nHidden = 256 nClasses = 30 #39 phonemes, plus the "blank" for CTC ####Load data print('Loading data') with open('TIMIT_data_prepared_for_CTC.pkl','rb') as f: data= pickle.load(f) input_list = batch charmap = data['chars'] print(charmap) charmap.append('_') #batchedData, maxTimeSteps = data_lists_to_batches(input_list, target_list, batchSize) maxTimeSteps = 776 totalN = len(input_list) ####Define graph print('Defining graph') graph = tf.Graph() with graph.as_default(): ####NOTE: try variable-steps inputs and dynamic bidirectional rnn, when it's implemented in tensorflow ####Graph input inputX = tf.placeholder(tf.float32, shape=(maxTimeSteps, batchSize, nFeatures)) #Prep input data to fit requirements of rnn.bidirectional_rnn # Reshape to 2-D tensor (nTimeSteps*batchSize, nfeatures) inputXrs = tf.reshape(inputX, [-1, nFeatures]) # Split to get a list of 'n_steps' tensors of shape (batch_size, n_hidden) inputList = tf.split(0, maxTimeSteps, inputXrs) targetIxs = tf.placeholder(tf.int64) targetVals = tf.placeholder(tf.int32) targetShape = tf.placeholder(tf.int64) targetY = tf.SparseTensor(targetIxs, targetVals, targetShape) seqLengths = tf.placeholder(tf.int32, shape=(batchSize)) ####Weights & biases weightsOutH1 = tf.Variable(tf.truncated_normal([2, nHidden], stddev=np.sqrt(2.0 / (2*nHidden)))) biasesOutH1 = tf.Variable(tf.zeros([nHidden])) weightsOutH2 = tf.Variable(tf.truncated_normal([2, nHidden], stddev=np.sqrt(2.0 / (2*nHidden)))) biasesOutH2 = tf.Variable(tf.zeros([nHidden])) weightsClasses = tf.Variable(tf.truncated_normal([nHidden, nClasses], stddev=np.sqrt(2.0 / nHidden))) biasesClasses = tf.Variable(tf.zeros([nClasses])) ####Network forwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True) backwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True) fbH1, _, _ = bidirectional_rnn(forwardH1, backwardH1, inputList, dtype=tf.float32, scope='BDLSTM_H1') fbH1rs = [tf.reshape(t, [batchSize, 2, nHidden]) for t in fbH1] outH1 = [tf.reduce_sum(tf.mul(t, weightsOutH1), reduction_indices=1) + biasesOutH1 for t in fbH1rs] logits = [tf.matmul(t, weightsClasses) + biasesClasses for t in outH1] ####Optimizing logits3d = tf.pack(logits) loss = tf.reduce_mean(ctc.ctc_loss(logits3d, targetY, seqLengths)) optimizer = tf.train.MomentumOptimizer(learningRate, momentum).minimize(loss) ####Evaluating logitsMaxTest = tf.slice(tf.argmax(logits3d,2), [0, 0], [seqLengths[0], 1]) predictions = tf.to_int32(ctc.ctc_beam_search_decoder(logits3d, seqLengths)[0][0]) errorRate = tf.reduce_sum(tf.edit_distance(predictions, targetY, normalize=False)) / \ tf.to_float(tf.size(targetY.values)) ####Run session with tf.Session(graph=graph) as session: print('Initializing') saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state('/users/TeamASR/models') if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path): print("Reading model parameters from %s" % ckpt.model_checkpoint_path) saver.restore(session, ckpt.model_checkpoint_path) else: print("Created model with fresh parameters.") session.run(tf.initialize_all_variables()) feedDict = {inputX: batch, seqLengths: (np.ones([batchSize])*776)} logit = session.run([logits3d], feed_dict=feedDict) return logit
####Weights & biases stddev = np.sqrt(2.0 / (2 * nHidden)) truncated_normal = tf.truncated_normal([2, nHidden], stddev=stddev) weightsOutH1 = tf.Variable(truncated_normal) biasesOutH1 = tf.Variable(tf.zeros([nHidden])) weightsOutH2 = tf.Variable(truncated_normal) biasesOutH2 = tf.Variable(tf.zeros([nHidden])) half_normal = tf.truncated_normal([nHidden, nClasses], stddev=np.sqrt(2.0 / nHidden)) weightsClasses = tf.Variable(half_normal) biasesClasses = tf.Variable(tf.zeros([nClasses])) ####Network forwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True) backwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True) print("building bidirectional_rnn ... SLOW!!!") fbH1, _, _ = bidirectional_rnn(forwardH1, backwardH1, inputList, dtype=tf.float32, scope='BDLSTM_H1') print("done building rnn") print("building fbH1rs ") fbH1rs = [tf.reshape(t, [Size, 2, nHidden]) for t in fbH1] print("building outH1 ") outH1 = [tf.reduce_sum(tf.multiply(t, weightsOutH1), axis=1) + biasesOutH1 for t in fbH1rs] print("building logits ") logits = [tf.matmul(t, weightsClasses) + biasesClasses for t in outH1] print("len(outH1) %d"% len(outH1)) ####Optimizing print("building loss") logits3d = tf.stack(logits) loss = tf.reduce_mean(ctc.ctc_loss(logits3d, targetY, seqLengths)) out = tf.identity(loss, 'ctc_loss_mean') optimizer = tf.train.MomentumOptimizer(learningRate, momentum).minimize(loss)
targ_ids = tf.placeholder(tf.int64) targ_vals = tf.placeholder(tf.int32) targ_shape = tf.placeholder(tf.int64) targets = tf.SparseTensor(targ_ids,targ_vals, targ_shape) l_reshape1 = tf.reshape(l_in,(-1,input_size) ) W1 = weight_variable([input_size,hidden_size]) b1 = bias_variable([hidden_size]) h_1 = tf.nn.relu(tf.matmul(l_reshape1,W1) + b1) l_reshape2 = tf.reshape(h_1, [-1, n_time_steps,hidden_size]) rnn_input_tr = tf.transpose(l_reshape2,[1,0,2]) rnn_input_re = tf.reshape(rnn_input_tr,[-1,hidden_size]) rnn_input = tf.split(0, n_time_steps, rnn_input_re) lstm_fw_cell = rnn_cell.BasicLSTMCell(hidden_size, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(hidden_size, forget_bias=1.0) # Get lstm cell output lstm_outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, rnn_input, dtype=tf.float32) lstm_outputs_re=tf.reshape(lstm_outputs, [n_time_steps, -1, 2*hidden_size]) lstm_output_tr=tf.transpose(lstm_outputs_re, [1,0,2]) W2 = weight_variable([2*hidden_size,output_size]) b2 = bias_variable([output_size]) #n_batch, n_time_steps, n_features = l_in.input_var.shape #Unnecessary in this version. Just collecting the info so that we can reshape the output back to the original shape l_reshape3 = tf.reshape(lstm_output_tr,[-1,2*hidden_size] ) h_2 = tf.matmul(l_reshape3,W2) + b2 l_reshape4 = tf.reshape(h_2,[-1,output_size]) l_soft = tf.nn.softmax(l_reshape4) l_soft_reshaped = tf.reshape(l_soft,[-1,n_time_steps,output_size]) l_soft_tr = tf.transpose(l_soft_reshaped, [1,0,2]) loss = tf.reduce_mean(tf.nn.ctc_loss(l_soft_tr, targets,seqLengths)) optimizer = tf.train.AdamOptimizer(learningRate).minimize(loss)
seq5 = tf.constant([[1,2,3,4,5,6], [1,2,3,4,5,6]]) # batch_size = 2 unpacked_seqs_embeds = [None,] seqs_embeds = [None,] for seq in [seq1,seq2,seq3,seq4,seq5]: seqs_embed = tf.nn.embedding_lookup(embed,seq) seqs_embeds.append(seqs_embed) unpacked_seqs_embeds.append(tf.unpack(tf.transpose(seqs_embed,perm=[1, 0, 2]))) with tf.variable_scope("naive",initializer=tf.truncated_normal_initializer(seed=1)) as scope: if FLAGS.rnn_type == "fw": n_output1,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[1],dtype=tf.float32,sequence_length=None ) elif FLAGS.rnn_type == "bi": n_output1,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[1],dtype=tf.float32,sequence_length=None ) with tf.variable_scope("naive2",initializer=tf.truncated_normal_initializer(seed=1)) as scope: if FLAGS.rnn_type == "fw": n_output2,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[2],dtype=tf.float32,sequence_length=None ) elif FLAGS.rnn_type == "bi": n_output2,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[2],dtype=tf.float32,sequence_length=None ) with tf.variable_scope("batch1",initializer=tf.truncated_normal_initializer(seed=1)) as scope: if FLAGS.rnn_type == "fw": n_output3,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[3],dtype=tf.float32,sequence_length=None ) elif FLAGS.rnn_type == "bi": n_output3,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[3],dtype=tf.float32,sequence_length=None ) with tf.variable_scope("batch2",initializer=tf.truncated_normal_initializer(seed=1)) as scope: if FLAGS.rnn_type == "fw": n_output4,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[4],dtype=tf.float32,sequence_length=None ) elif FLAGS.rnn_type == "bi": n_output4,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[4],dtype=tf.float32,sequence_length=None )
rnn_dim = 1024 max_len = 100 label = tf.constant(np.random.rand(rnn_dim),dtype=tf.float32) embed = tf.Variable(tf.constant(np.random.rand(vocab_size,embed_dim),dtype=tf.float32),trainable=False) seq_len= tf.placeholder(tf.int32,[batch_size]) seq = tf.placeholder(tf.int32,[batch_size,max_len]) seqs_embed = tf.nn.embedding_lookup(embed,seq) unpacked_seqs_embed = tf.unpack(tf.transpose(seqs_embed,perm=[1, 0, 2])) with tf.variable_scope("naive",initializer=tf.truncated_normal_initializer(seed=1)) as scope: if FLAGS.rnn_type == "fw": n_output1,_ = rnn.rnn(rnn_cell.BasicRNNCell(rnn_dim),unpacked_seqs_embed,dtype=tf.float32,sequence_length=seq_len ) elif FLAGS.rnn_type == "bi": n_output1,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(rnn_dim/2),rnn_cell.BasicRNNCell(rnn_dim/2),unpacked_seqs_embeds,dtype=tf.float32,sequence_length=seq_len ) with tf.variable_scope("dynamic",initializer=tf.truncated_normal_initializer(seed=1)) as scope: if FLAGS.rnn_type == "fw": n_output2,_ = rnn.dynamic_rnn(rnn_cell.BasicRNNCell(rnn_dim),seqs_embed,dtype=tf.float32,time_major=False,sequence_length=seq_len ) elif FLAGS.rnn_type == "bi" and tf.__version__.startswith("0.10"): n_output2,_ = rnn.bidirectional_dynamic_rnn(rnn_cell.BasicRNNCell(rnn_dim/2),rnn_cell.BasicRNNCell(rnn_dim/2),seqs_embed,dtype=tf.float32,time_major=False,sequence_length=seq_len ) avgLosses = [] # average pooling loss if FLAGS.rnn_type == "fw" or (FLAGS.rnn_type == "bi" and tf.__version__.startswith("0.10")): outputs = [n_output1,n_output2] else: outputs = [n_output1] for i,n_output in enumerate(outputs):
def __init__(self, sequence_length, num_classes, vocab_size, embedding_size, num_filter, filter_lengths, pool_lengths, dropout, hidden_size, l2_reg_lambda): """ sequence_length: the length of our sentences (paddled) = 336 num_classes: 2 vocab_size: 111 embedding_size: 8 num_filter: 128 filter_lengths: [5, 3] pool_lengths: [2, 2] dropout: 0.5 hidden_size: 128 l2_reg_lambda: 5 * 10 ^ -4 """ assert len(filter_lengths) == len(pool_lengths) # Placeholders for input, output and initial state of bi-lstm input_x = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_x') input_y = tf.placeholder(tf.float32, shape=[None, num_classes], name='input_y') istate_fw = tf.placeholder("float", [None, 2 * hidden_size]) istate_bw = tf.placeholder("float", [None, 2 * hidden_size]) # Keeping track of l2 regularization loss (optional) l2_loss = tf.constant(0.0) # Embedding layer with tf.name_scope('embedding'): W_embed = tf.Variable( tf.random_uniform(shape=[vocab_size, embedding_size], minval=-1, maxval=1), name='W_embed') embedded_chars = tf.nn.embedding_lookup(W_embed, input_x) embedded_chars_expanded = tf.expand_dims(embedded_chars, -1) # Convolution + maxpool 0 num_conv = 0 with tf.name_scope('conv-maxpool-%d' % num_conv): # Conv0 filter_shape0 = [filter_lengths[num_conv], embedding_size, 1, num_filter] W_conv0 = tf.Variable(tf.truncated_normal(shape=filter_shape0, stddev=0.1), name='W0') b_conv0 = tf.Variable(tf.constant(value=0.1, shape=[num_filter]), name='b0') conv0 = tf.nn.conv2d( input=embedded_chars_expanded, filter=W_conv0, strides=[1, 1, 1, 1], padding="VALID", name='conv0' ) # Apply nonlinearity h0 = tf.nn.relu(tf.nn.bias_add(conv0, b_conv0), name='relu0') # Max pooling pool0 = tf.nn.max_pool( value=tf.expand_dims(tf.squeeze(h0, [2]), -1), # [?, conv, 1, num_filter] -> [?, conv, num_filter, 1] ksize=[1, pool_lengths[num_conv], 1, 1], strides=[1, pool_lengths[num_conv], 1, 1], padding='VALID', name='pool0' ) num_conv += 1 # Convolution + maxpool 1 with tf.name_scope('conv-maxpool-%d' % num_conv): # Conv1 filter_shape1 = [filter_lengths[num_conv], num_filter, 1, num_filter] W_conv1 = tf.Variable(tf.truncated_normal(shape=filter_shape1, stddev=0.1), name='W1') b_conv1 = tf.Variable(tf.constant(value=0.1, shape=[num_filter]), name='b1') conv1 = tf.nn.conv2d( input=pool0, filter=W_conv1, strides=[1, 1, 1, 1], padding="VALID", name='conv1') # Apply nonlinearity h1 = tf.nn.relu(tf.nn.bias_add(conv1, b_conv1), name='relu1') # Max pooling pool1 = tf.nn.max_pool( value=tf.expand_dims(tf.squeeze(h1, [2]), -1), ksize=[1, pool_lengths[num_conv], 1, 1], strides=[1, pool_lengths[num_conv], 1, 1], padding='VALID', name='pool1') # Add dropout 0 with tf.name_scope('drop_conv'): drop_conv = tf.nn.dropout(pool1, keep_prob=dropout) # Reshape tensor with tf.name_scope('reshape_conv_rnn'): _input = tf.squeeze(drop_conv, [3]) _input = tf.transpose(_input, [1, 0, 2]) _input = tf.reshape(_input, [-1, hidden_size]) _input = tf.split(0, 82, _input) # TODO: 82 대신 자동화 되서 나온 숫자. # Bidirectional with tf.name_scope('Bi-LSTM'): lstm_fw_cell = rnn_cell.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0) lstm_bw_cell = rnn_cell.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0) output, _, _ = rnn.bidirectional_rnn( cell_fw=lstm_fw_cell, cell_bw=lstm_bw_cell, inputs=_input, initial_state_fw=istate_fw, initial_state_bw=istate_bw) # Add dropout1 with tf.name_scope('drop_rnn'): drop_rnn = tf.nn.dropout(output[-1], keep_prob=0.5) # Final (unnormalized) scores and predictions with tf.name_scope('softmax'): W_soft = tf.get_variable( 'W', shape=[2*hidden_size, num_classes], initializer=tf.contrib.layers.xavier_initializer()) # W_soft = tf.Variable(tf.truncated_normal(shape=[2*hidden_size, num_classes])) b_soft = tf.Variable(tf.constant(value=0.1, shape=[num_classes])) l2_loss += tf.nn.l2_loss(W_soft) l2_loss += tf.nn.l2_loss(b_soft) # scores = tf.matmul(drop_rnn, W_soft) + b_soft scores = tf.nn.xw_plus_b(x=drop_rnn, weights=W_soft, biases=b_soft, name='scores') predictions = tf.argmax(scores, 1, name='predictions') # Calculate loss function with tf.name_scope('loss'): losses = tf.nn.softmax_cross_entropy_with_logits(scores, input_y) self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss / 2 # Accuracy with tf.name_scope("accuracy"): correct_predictions = tf.equal(predictions, tf.argmax(input_y, 1)) self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def stack_bidirectional_rnn(cells_fw, cells_bw, inputs, initial_states_fw=None, initial_states_bw=None, dtype=None, sequence_length=None, scope=None): """Creates a bidirectional recurrent neural network. Stacks several bidirectional rnn layers. The combined forward and backward layer outputs are used as input of the next layer. tf.bidirectional_rnn does not allow to share forward and backward information between layers. The input_size of the first forward and backward cells must match. The initial state for both directions is zero and no intermediate states are returned. As described in https://arxiv.org/abs/1303.5778 Args: cells_fw: List of instances of RNNCell, one per layer, to be used for forward direction. cells_bw: List of instances of RNNCell, one per layer, to be used for backward direction. inputs: A length T list of inputs, each a tensor of shape [batch_size, input_size], or a nested tuple of such elements. initial_states_fw: (optional) A list of the initial states (one per layer) for the forward RNN. Each tensor must has an appropriate type and shape `[batch_size, cell_fw.state_size]`. initial_states_bw: (optional) Same as for `initial_states_fw`, but using the corresponding properties of `cells_bw`. dtype: (optional) The data type for the initial state. Required if either of the initial states are not provided. sequence_length: (optional) An int32/int64 vector, size `[batch_size]`, containing the actual lengths for each of the sequences. scope: VariableScope for the created subgraph; defaults to None. Returns: A tuple (outputs, output_state_fw, output_state_bw) where: outputs is a length `T` list of outputs (one for each input), which are depth-concatenated forward and backward outputs. output_states_fw is the final states, one tensor per layer, of the forward rnn. output_states_bw is the final states, one tensor per layer, of the backward rnn. Raises: TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`. ValueError: If inputs is None, not a list or an empty list. """ if not cells_fw: raise ValueError("Must specify at least one fw cell for BidirectionalRNN.") if not cells_bw: raise ValueError("Must specify at least one bw cell for BidirectionalRNN.") if not isinstance(cells_fw, list): raise ValueError("cells_fw must be a list of RNNCells (one per layer).") if not isinstance(cells_bw, list): raise ValueError("cells_bw must be a list of RNNCells (one per layer).") if len(cells_fw) != len(cells_bw): raise ValueError("Forward and Backward cells must have the same depth.") if initial_states_fw is not None and (not isinstance(cells_fw, list) or len(cells_fw) != len(cells_fw)): raise ValueError( "initial_states_fw must be a list of state tensors (one per layer).") if initial_states_bw is not None and (not isinstance(cells_bw, list) or len(cells_bw) != len(cells_bw)): raise ValueError( "initial_states_bw must be a list of state tensors (one per layer).") states_fw = [] states_bw = [] prev_layer = inputs with vs.variable_scope(scope or "stack_bidirectional_rnn"): for i, (cell_fw, cell_bw) in enumerate(zip(cells_fw, cells_bw)): initial_state_fw = None initial_state_bw = None if initial_states_fw: initial_state_fw = initial_states_fw[i] if initial_states_bw: initial_state_bw = initial_states_bw[i] with vs.variable_scope("cell_%d" % i) as cell_scope: prev_layer, state_fw, state_bw = rnn.bidirectional_rnn( cell_fw, cell_bw, prev_layer, initial_state_fw=initial_state_fw, initial_state_bw=initial_state_bw, sequence_length=sequence_length, dtype=dtype, scope=cell_scope) states_fw.append(state_fw) states_bw.append(state_bw) return prev_layer, tuple(states_fw), tuple(states_bw)