def BiRNN(x, weights, biases):
# Prepare data shape to match `bidirectional_rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, n_input])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(axis=0, num_or_size_splits=n_steps, value=x)
# Define lstm cells with tensorflow
# Forward direction cell
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Get lstm cell output
try:
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def RNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(0, n_steps, x)
'''
lstm_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
'''
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
try:
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def inference(images, eval=False):
NUM_HIDDEN = 128
W = _variable_with_weight_decay('weight', [2 * NUM_HIDDEN, NUM_CLASSES],
0.001, 0.01)
b = _variable_with_weight_decay('bias', [NUM_CLASSES], 0.001, 0)
s = images.get_shape()
n_batches, n_steps, n_features = int(s[0]), int(s[1]), int(s[2])
#print(n_batches, n_steps, n_features)
inputs = tf.reshape(images, [-1, n_features])
inputs = tf.split(0, n_steps, inputs)
fw_cell = rnn_cell.LSTMCell(NUM_HIDDEN,
forget_bias=1.0,
state_is_tuple=True)
bw_cell = rnn_cell.LSTMCell(NUM_HIDDEN,
forget_bias=1.0,
state_is_tuple=True)
try:
outputs, _, _ = rnn.bidirectional_rnn(fw_cell,
bw_cell,
inputs,
dtype=tf.float32)
except Exception:
outputs = rnn.bidirectional_rnn(fw_cell, bw_cell, x, dtype=tf.float32)
logits = tf.matmul(outputs[-1], W) + b
return logits
def BiRNN(x, weights, biases):
# Prepare data shape to match `bidirectional_rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, n_input])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, n_steps, x)
# Define lstm cells with tensorflow
# Forward direction cell
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Get lstm cell output
try:
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x,
dtype=tf.float32)
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def init_all(self, wr):
self.word_reader = wr
# initializer = tf.constant_initializer(value=wr.word_vectors, dtype=tf.float32)
# self.word_dict = tf.get_variable('word_dict', shape=[len(wr.word_vectors), config.embedding_size],
# initializer=initializer, trainable=config.trainable)
self.word_dict = tf.get_variable('haha', shape=None, dtype=tf.float32,
initializer=tf.constant(wr.word_vectors),
trainable=False)
# paraphrase sentences
self.x1_index = tf.placeholder(tf.int32, [None, config.max_sentence_len])
self.x1 = tf.nn.embedding_lookup(self.word_dict, self.x1_index)
# Permuting batch_size and n_steps
self.x1 = tf.transpose(self.x1, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
self.x1 = tf.reshape(self.x1, [-1, config.embedding_size])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
self.x1 = tf.split(0, config.max_sentence_len, self.x1)
self.x2_index = tf.placeholder(tf.int32, [None, config.max_sentence_len])
self.x2 = tf.nn.embedding_lookup(self.word_dict, self.x2_index)
# Permuting batch_size and n_steps
self.x2 = tf.transpose(self.x2, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
self.x2 = tf.reshape(self.x2, [-1, config.embedding_size])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
self.x2 = tf.split(0, config.max_sentence_len, self.x2)
self.y = tf.placeholder("int64", [None])
# Forward direction cell
lstm_fw_cell_x1 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True)
# Backward direction cell
lstm_bw_cell_x1 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True)
# Forward direction cell
lstm_fw_cell_x2 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True)
# Backward direction cell
lstm_bw_cell_x2 = rnn_cell.LSTMCell(config.hidden_size, forget_bias=1.0, state_is_tuple=True)
# Get lstm cell output
outputs_x1, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_x1, lstm_bw_cell_x1,
self.x1, dtype=tf.float32, scope="RNN1")
outputs_x2, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_x2, lstm_bw_cell_x2,
self.x2, dtype=tf.float32, scope="RNN2")
outputs = tf.concat(1, [outputs_x1[-1], outputs_x2[-1]])
self.weights = tf.Variable(tf.random_uniform([config.hidden_size * 4, config.classes],
dtype=tf.float32))
# self.weights = tf.Variable(tf.random_uniform([config.embedding_size * 2, config.classes],
# dtype=tf.float32))
self.b = tf.Variable(tf.random_uniform([config.classes]), dtype=tf.float32)
mid = tf.matmul(outputs, self.weights) + self.b
self.results = tf.nn.softmax(mid)
self.loss = tf.nn.sparse_softmax_cross_entropy_with_logits(self.results, self.y)
self.cost = tf.reduce_sum(self.loss)
init = tf.initialize_all_variables()
vars = tf.trainable_variables()
opt=tf.train.GradientDescentOptimizer(learning_rate=config.learning_rate)
self.optimizer = opt.minimize(self.cost, var_list=vars)
self.sess = tf.Session()
self.sess.run(init)
return
def inference(inputs, n_input, n_steps, n_hidden, n_classes): W = tf.Variable(tf.random_normal([2*n_hidden, n_classes])) b = tf.Variable(tf.random_normal([n_classes])) inputs = tf.reshape(inputs, [-1, n_input]) inputs = tf.split(0, n_steps, inputs) fw_cell = rnn_cell.LSTMCell(n_hidden, forget_bias = 1.0, state_is_tuple = True) bw_cell = rnn_cell.LSTMCell(n_hidden, forget_bias = 1.0, state_is_tuple = True) try: outputs,_,_ = rnn.bidirectional_rnn(fw_cell, bw_cell, inputs, dtype = tf.float32) except Exception: outputs = rnn.bidirectional_rnn(fw_cell, bw_cell, x, dtype = tf.float32) return tf.matmul(outputs[-1], W) + b
def build_similarity_calculator(self):
question = tf.placeholder(tf.int32, [1, self.len_question])
question_sequence_length = tf.constant([1], dtype=tf.int64)
answer = tf.placeholder(tf.int32, [1, self.len_answer])
answer_sequence_length = tf.constant([1], dtype=tf.int64)
lstm_state_fw_Q = tf.zeros([1, self.lstm_fw_Q.state_size])
lstm_state_bw_Q = tf.zeros([1, self.lstm_bw_Q.state_size])
lstm_state_fw_A = tf.zeros([1, self.lstm_fw_A.state_size])
lstm_state_bw_A = tf.zeros([1, self.lstm_bw_A.state_size])
with tf.device("/cpu:0"):
embedded_question = tf.nn.embedding_lookup(self.Wemb,
tf.transpose(question))
embedded_answer = tf.nn.embedding_lookup(self.Wemb,
tf.transpose(answer))
question_brnn_output = rnn.bidirectional_rnn(
self.lstm_fw_Q,
self.lstm_bw_Q,
tf.unpack(embedded_question),
sequence_length=question_sequence_length,
initial_state_fw=lstm_state_fw_Q,
initial_state_bw=lstm_state_bw_Q,
)
question_pooled_output = tf.reduce_mean(tf.pack(question_brnn_output),
0)
question_final_emb = tf.nn.xw_plus_b(question_pooled_output,
self.W_Q_emb, self.b_Q_emb)
question_final_emb = tf.nn.l2_normalize(question_final_emb, dim=1)
answer_brnn_output = rnn.bidirectional_rnn(
self.lstm_fw_A,
self.lstm_bw_A,
tf.unpack(embedded_answer),
sequence_length=answer_sequence_length,
initial_state_fw=lstm_state_fw_A,
initial_state_bw=lstm_state_bw_A)
answer_brnn_pooled_output = tf.reduce_mean(tf.pack(answer_brnn_output),
0)
answer_final_emb = tf.nn.xw_plus_b(answer_brnn_pooled_output,
self.W_A_emb, self.b_A_emb)
answer_final_emb = tf.nn.l2_normalize(answer_final_emb, dim=1)
similarity = tf.reduce_mean(
tf.matmul(question_final_emb, tf.transpose(answer_final_emb)))
return question, question_sequence_length, answer, answer_sequence_length, similarity
def bidirectional_lstm_inference(x):
RNN_HIDDEN_UNITS = 128
# x was [BATCH_SIZE, 32, 32, 3]
# x changes to [32, BATCH_SIZE, 32, 3]
x = tf.transpose(x, [1, 0, 2, 3])
# x changes to [32 * BATCH_SIZE, 32 * 3]
x = tf.reshape(x, [-1, IMAGE_SIZE * RGB_CHANNEL_SIZE])
# x changes to array of 32 * [BATCH_SIZE, 32 * 3]
x = tf.split(0, IMAGE_SIZE, x)
weights = tf.Variable(
tf.random_normal([2 * RNN_HIDDEN_UNITS, LABEL_SIZE]))
biases = tf.Variable(tf.random_normal([LABEL_SIZE]))
# output size is 128, state size is (c=128, h=128)
fw_lstm_cell = rnn_cell.BasicLSTMCell(RNN_HIDDEN_UNITS,
forget_bias=1.0)
bw_lstm_cell = rnn_cell.BasicLSTMCell(RNN_HIDDEN_UNITS,
forget_bias=1.0)
# outputs is array of 32 * [BATCH_SIZE, 128]
outputs, _, _ = rnn.bidirectional_rnn(fw_lstm_cell,
bw_lstm_cell,
x,
dtype=tf.float32)
# outputs[-1] is [BATCH_SIZE, 128]
return tf.matmul(outputs[-1], weights) + biases
def BILSTM(x, hidden_size):
# biLSTM：
# 功能：添加bidirectional_lstm操作
# 参数：
# x: [batch, height, width] / [batch, step, embedding_size]
# hidden_size: lstm隐藏层节点个数
# 输出：
# output: [batch, height, 2*hidden_size] / [batch, step, 2*hidden_size]
# input transformation
input_x = tf.transpose(x, [1, 0, 2])
# input_x = tf.reshape(input_x, [-1, w])
# input_x = tf.split(0, h, input_x)
input_x = tf.unpack(input_x)
# define the forward and backward lstm cells
gru_fw_cell = rnn_cell.GRUCell(hidden_size)
gru_bw_cell = rnn_cell.GRUCell(hidden_size)
output, _, _ = rnn.bidirectional_rnn(gru_fw_cell,
gru_bw_cell,
input_x,
dtype=tf.float32)
# output transformation to the original tensor type
output = tf.pack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def BiRNN(x, f_cell, b_cell):
x = tf.unstack(x, n_words, 2)
outputs, _, _ = rnn.bidirectional_rnn(f_cell, b_cell, x, dtype=tf.float32)
return outputs[-1]
def bidir_ltsm(x):
with tf.name_scope('Weights'):
# Permuting batch_size and n_steps
#x = tf.transpose(x, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, model.num_features])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, model.max_timesteps, x)
weights_out1 = tf.Variable(tf.truncated_normal([2, num_hidden], stddev=np.sqrt(1./num_hidden)), name='weights')
biases_out1 = tf.Variable(tf.zeros(num_hidden), name='biases')
weights_out2 = tf.Variable(tf.truncated_normal([2, num_hidden], stddev=np.sqrt(1./num_hidden)), name='weights')
biases_out2 = tf.Variable(tf.zeros(num_hidden), name='biases')
with tf.name_scope('LTSM'):
forward = rnn_cell.LSTMCell(num_hidden, use_peepholes=True, forget_bias=1.0)
backward = rnn_cell.LSTMCell(num_hidden, use_peepholes=True, forget_bias=1.0)
with tf.name_scope('Bidirectionrnn'):
bidirectional_h1, _, _ = bidirectional_rnn(forward, backward, x, dtype=tf.float32)
bd_h1s = [tf.reshape(t, [batch_size, 2, num_hidden]) for t in bidirectional_h1]
with tf.name_scope('logits'):
weights_class = tf.Variable(tf.truncated_normal([num_hidden, model.num_classes], stddev=np.sqrt(1./num_hidden)), name='weights')
biases_class = tf.Variable(tf.zeros([model.num_classes]))
out_h1 = [tf.reduce_sum(tf.mul(t, weights_out1), reduction_indices=1) + biases_out1 for t in bd_h1s]
logits = [tf.matmul(t, weights_class) + biases_class for t in out_h1]
logits3d = tf.pack(logits)
return logits3d
def embedding_encoder(encoder_inputs,
cell,
embedding,
num_symbols,
embedding_size,
bidirectional=False,
dtype=None,
weight_initializer=None,
scope=None):
with variable_scope.variable_scope(scope or "embedding_encoder",
dtype=dtype) as scope:
dtype = scope.dtype
# Encoder.
if not embedding:
embedding = variable_scope.get_variable(
"embedding", [num_symbols, embedding_size],
initializer=weight_initializer())
emb_inp = [
embedding_ops.embedding_lookup(embedding, i)
for i in encoder_inputs
]
if bidirectional:
_, output_state_fw, output_state_bw = rnn.bidirectional_rnn(
cell, cell, emb_inp, dtype=dtype)
encoder_state = tf.concat(1, [output_state_fw, output_state_bw])
else:
_, encoder_state = rnn.rnn(cell, emb_inp, dtype=dtype)
return encoder_state
def _tf_enc_embedding_attention_seq2seq(self, encoder_inputs, cell,
num_encoder_symbols,
embedding_size,
num_heads=1,
dtype=dtypes.float32,
scope=None,
encoder="reverse",
sequence_length=None,
bucket_length=None,
init_backward=False,
bow_emb_size=None,
single_src_embedding=False):
"""Embedding sequence-to-sequence model with attention.
"""
with tf.variable_scope(scope or "embedding_attention_seq2seq", reuse=True):
# Encoder.
if encoder == "bidirectional":
encoder_cell_fw = rnn_cell.EmbeddingWrapper(
cell.get_fw_cell(), embedding_classes=num_encoder_symbols,
embedding_size=embedding_size)
embed_scope = None
if single_src_embedding:
logging.info("Reuse forward src embedding for backward encoder")
with variable_scope.variable_scope("BiRNN/FW/EmbeddingWrapper") as es:
embed_scope = es
encoder_cell_bw = rnn_cell.EmbeddingWrapper(
cell.get_bw_cell(), embedding_classes=num_encoder_symbols,
embedding_size=embedding_size, embed_scope=embed_scope)
encoder_outputs, encoder_state, encoder_state_bw = rnn.bidirectional_rnn(encoder_cell_fw, encoder_cell_bw,
encoder_inputs, dtype=dtype,
sequence_length=sequence_length,
bucket_length=bucket_length)
logging.debug("Bidirectional state size=%d" % cell.state_size) # this shows double the size for lstms
elif encoder == "reverse":
encoder_cell = rnn_cell.EmbeddingWrapper(
cell, embedding_classes=num_encoder_symbols,
embedding_size=embedding_size)
encoder_outputs, encoder_state = rnn.rnn(
encoder_cell, encoder_inputs, dtype=dtype, sequence_length=sequence_length, bucket_length=bucket_length, reverse=True)
logging.debug("Unidirectional state size=%d" % cell.state_size)
elif encoder == "bow":
encoder_outputs, encoder_state = cell.embed(rnn_cell.Embedder, num_encoder_symbols,
bow_emb_size, encoder_inputs, dtype=dtype)
# First calculate a concatenation of encoder outputs to put attention on.
if encoder == "bow":
top_states = [array_ops.reshape(e, [-1, 1, bow_emb_size])
for e in encoder_outputs]
else:
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size])
for e in encoder_outputs]
attention_states = array_ops.concat(1, top_states)
initial_state = encoder_state
if encoder == "bidirectional" and init_backward:
initial_state = encoder_state_bw
return self._tf_enc_embedding_attention_decoder(
attention_states, initial_state, cell, num_heads=num_heads)
def compute_states(self,emb):
def unpack_sequence(tensor):
return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2]))
with tf.variable_scope("Composition",initializer=
tf.contrib.layers.xavier_initializer(),regularizer=
tf.contrib.layers.l2_regularizer(self.reg)):
cell_fw = rnn_cell.LSTMCell(self.hidden_dim)
cell_bw = rnn_cell.LSTMCell(self.hidden_dim)
#tf.cond(tf.less(self.dropout
#if tf.less(self.dropout, tf.constant(1.0)):
cell_fw = rnn_cell.DropoutWrapper(cell_fw,
output_keep_prob=self.dropout,input_keep_prob=self.dropout)
cell_bw=rnn_cell.DropoutWrapper(cell_bw, output_keep_prob=self.dropout,input_keep_prob=self.dropout)
#output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32)
outputs,_,_=rnn.bidirectional_rnn(cell_fw,cell_bw,unpack_sequence(emb),sequence_length=self.lngths,dtype=tf.float32)
#output = pack_sequence(outputs)
sum_out=tf.reduce_sum(tf.pack(outputs),[0])
sent_rep = tf.div(sum_out,tf.expand_dims(tf.to_float(self.lngths),1))
final_state=sent_rep
return final_state
def LSTM_Network(input_, weightsOutH1, weightsClasses, biasesOutH1,
biasesClasses):
####Network
forwardH1 = rnn_cell.LSTMCell(nHidden,
use_peepholes=True,
state_is_tuple=True)
backwardH1 = rnn_cell.LSTMCell(nHidden,
use_peepholes=True,
state_is_tuple=True)
fbH1, _, _ = bidirectional_rnn(forwardH1,
backwardH1,
inputList,
dtype=tf.float32,
scope='BDLSTM_H1')
fbH1rs = [tf.reshape(t, [batchSize, 2, nHidden]) for t in fbH1]
outH1 = [
tf.reduce_sum(tf.mul(t, weightsOutH1), reduction_indices=1) +
biasesOutH1 for t in fbH1rs
]
logits = [tf.matmul(t, weightsClasses) + biasesClasses for t in outH1]
####Optimizing
logits3d = tf.pack(logits)
return logits3d
def compute_states(self, emb):
def unpack_sequence(tensor):
return tf.unpack(tf.transpose(tensor, perm=[1, 0, 2]))
with tf.variable_scope(
"Composition",
initializer=tf.contrib.layers.xavier_initializer(),
regularizer=tf.contrib.layers.l2_regularizer(self.reg)):
cell_fw = rnn_cell.LSTMCell(self.hidden_dim)
cell_bw = rnn_cell.LSTMCell(self.hidden_dim)
#tf.cond(tf.less(self.dropout
#if tf.less(self.dropout, tf.constant(1.0)):
cell_fw = rnn_cell.DropoutWrapper(cell_fw,
output_keep_prob=self.dropout,
input_keep_prob=self.dropout)
cell_bw = rnn_cell.DropoutWrapper(cell_bw,
output_keep_prob=self.dropout,
input_keep_prob=self.dropout)
#output, state = rnn.dynamic_rnn(cell,emb,sequence_length=self.lngths,dtype=tf.float32)
outputs, _, _ = rnn.bidirectional_rnn(cell_fw,
cell_bw,
unpack_sequence(emb),
sequence_length=self.lngths,
dtype=tf.float32)
#output = pack_sequence(outputs)
sum_out = tf.reduce_sum(tf.pack(outputs), [0])
sent_rep = tf.div(sum_out, tf.expand_dims(tf.to_float(self.lngths), 1))
final_state = sent_rep
return final_state
def get_similarity(one_answer, one_answer_sequence):
tf.get_variable_scope().reuse_variables()
with tf.device("/cpu:0"):
embedded_answer = tf.nn.embedding_lookup(
self.Wemb, tf.transpose(one_answer))
answer_output = rnn.bidirectional_rnn(
self.lstm_fw_A,
self.lstm_bw_A,
tf.unpack(embedded_answer),
sequence_length=one_answer_sequence,
initial_state_fw=lstm_state_fw_A,
initial_state_bw=lstm_state_bw_A)
answer_pooled_output = tf.reduce_sum(tf.pack(answer_output), 0)
answer_pooled_output = answer_pooled_output / tf.expand_dims(
tf.to_float(one_answer_sequence) + 1e-6, 1)
answer_final_emb = tf.nn.xw_plus_b(answer_pooled_output,
self.W_A_emb, self.b_A_emb)
answer_final_emb = tf.nn.l2_normalize(answer_final_emb,
dim=1,
epsilon=1e-7)
similarity_matrix = tf.matmul(question_final_emb,
tf.transpose(answer_final_emb))
one_diagonal = tf.diag([1.] * self.batch_size)
similarity = tf.reduce_sum(similarity_matrix * one_diagonal)
return similarity
def BiRNN(x, f_cell, b_cell):
x = tf.unstack(x, n_words, 2)
outputs, _, _ = rnn.bidirectional_rnn(f_cell, b_cell, x, dtype=tf.float32)
return tf.pack(tf.transpose(outputs, [1, 0, 2]))
def embedding_encoder(encoder_inputs,
cell,
embedding,
num_symbols,
embedding_size,
bidirectional=False,
dtype=None,
weight_initializer=None,
scope=None):
with variable_scope.variable_scope(
scope or "embedding_encoder", dtype=dtype) as scope:
dtype = scope.dtype
# Encoder.
if not embedding:
embedding = variable_scope.get_variable("embedding", [num_symbols, embedding_size],
initializer=weight_initializer())
emb_inp = [embedding_ops.embedding_lookup(embedding, i) for i in encoder_inputs]
if bidirectional:
_, output_state_fw, output_state_bw = rnn.bidirectional_rnn(cell, cell, emb_inp,
dtype=dtype)
encoder_state = tf.concat(1, [output_state_fw, output_state_bw])
else:
_, encoder_state = rnn.rnn(
cell, emb_inp, dtype=dtype)
return encoder_state
def BiRNN(x, weights, biases):
# shape of x will be (batch_size, n_steps, n_channels)
# need to transform to (n_steps, batch_size, n_channels) to comply with tf bi_rnn
x = tf.transpose(x, [1, 0, 2])
x = tf.unpack(x, axis = 0)
# Forward direction cell
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
try:
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32)
except Exception:
outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, x, dtype = tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def BiRNN(x, weights, biases):
# Prepare data shape to match `bidirectional_rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, n_input])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, n_steps, x)
# Define lstm cells with tensorflow
with tf.variable_scope("lstm1") as scope1:
lstm_fw_cell_1 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell_1 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs_1, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_1,
lstm_bw_cell_1,
x,
dtype=tf.float32)
with tf.variable_scope("lstm2") as scope2:
lstm_fw_cell_2 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell_2 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs_2, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_2,
lstm_bw_cell_2,
outputs_1,
dtype=tf.float32)
with tf.variable_scope("lstm3") as scope3:
lstm_fw_cell_3 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell_3 = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs_3, _, _ = rnn.bidirectional_rnn(lstm_fw_cell_3,
lstm_bw_cell_3,
outputs_2,
dtype=tf.float32)
outputs = outputs_3
outputs = tf.reshape(tf.concat(0, outputs),
[MAX_LEN * BATCH_SIZE, n_hidden * 2])
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs, weights['out']) + biases['out']
def multiRNN(x, weights, biases, num_layers):
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, n_input])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, n_steps, x)
# # x = tf.transpose(tf.pack(x), perm=[1,0,2])
# print(x)
initializer = tf.random_uniform_initializer(-1, 1)
cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_hidden)
cell = tf.nn.rnn_cell.DropoutWrapper(cell=cell, output_keep_prob=0.5)
cell = tf.nn.rnn_cell.MultiRNNCell(cells=[cell] * num_layers)
cell2 = tf.nn.rnn_cell.BasicLSTMCell(num_units=n_hidden)
cell2 = tf.nn.rnn_cell.DropoutWrapper(cell=cell2, output_keep_prob=0.5)
cell2 = tf.nn.rnn_cell.MultiRNNCell(cells=[cell2] * num_layers)
# init_state = multicell.zero_state(batch_size, tf.float32)
# # Get lstm cell output
# try:
# outputs, _, _ = rnn.bidirectional_rnn(cell, cell, x,
# dtype=tf.float32)
# except Exception: # Old TensorFlow version only returns outputs not states
# outputs = rnn.bidirectional_rnn(cell, cell, x,
# dtype=tf.float32)
# try:
# outputs, _, _ = rnn.rnn(multicell, x, dtype=tf.float32)
# except Exception: # Old TensorFlow version only returns outputs not states
# outputs = rnn.rnn(multicell, x, dtype=tf.float32)
output, _, _ = rnn.bidirectional_rnn(cell, cell2, x, dtype=tf.float32)
# output = tf.transpose(output, [1, 0, 2])
# Linear activation, using rnn inner loop last output
# return tf.matmul(rnn_outputs[-1], weights['out']) + biases['out']
# return tf.matmul(rnn_outputs[-1], W) + b
# output = tf.transpose(output, [1, 0, 2])
# last = tf.gather(output, int(output.get_shape()[0]) - 1)
# # Softmax layer.
# weight = tf.Variable(tf.truncated_normal([n_hidden, y.get_shape()[1]], stddev=0.01))
# bias = tf.Variable(tf.constant(0.1, shape=[y.get_shape()[1]]))
# prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
pred = tf.matmul(output[-1], weights['out']) + biases['out']
return pred
def RNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(0, n_steps, x)
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
outputs_all = tf.concat(0, outputs) # (N*batch) * 2*n_hidden
M = tanh(tf.matmul(outputs_all, W_h)) # (N*batch) * 2*hidden
M_2 = tanh(tf.matmul(outputs_all, W_2_h))
a = tf.nn.softmax(tf.matmul(M, w)) # (N*batch) * 1
a = tf.reshape(a, [n_steps, -1, 1]) # N*batch*1
a = tf.transpose(a, [1, 2, 0]) # batch*1*N
a_2 = tf.nn.softmax(tf.matmul(M_2, w_2)) # (N*batch) * 1
a_2 = tf.reshape(a_2, [n_steps, -1, 1]) # N*batch*1
a_2 = tf.transpose(a_2, [1, 2, 0]) # batch*1*N
outputs_all = tf.reshape(outputs_all,
[n_steps, -1, 2 * n_hidden]) # N*batch*d
outputs_all = tf.transpose(outputs_all, [1, 0, 2]) # batch*N*d
batch_s = 32
a = tf.split(0, batch_s, a)
a_2 = tf.split(0, batch_s, a_2)
outputs_all = tf.split(0, batch_s, outputs_all)
r = []
r_2 = []
for i in range(batch_s):
a_temp = a[i][0:1, :, :]
a_2_temp = a_2[i][0:1, :, :]
o_temp = outputs_all[i][0:1, :, :]
o_2_temp = outputs_all[i][0:1, :, :]
att = tf.reshape(a_temp, [1, n_steps])
out = tf.reshape(o_temp, [n_steps, 2 * n_hidden])
att_2 = tf.reshape(a_2_temp, [1, n_steps])
out_2 = tf.reshape(o_2_temp, [n_steps, 2 * n_hidden])
r.append(tf.matmul(att, out))
r_2.append(tf.matmul(att_2, out_2))
r = tf.concat(0, r) # batch*d
r_2 = tf.concat(0, r_2) # batch*d
#attention_4 = tf.matmul(r,a)
_h = tanh(W_p * r + W_2_p * r_2 + W_x * outputs[-1])
predict = tf.matmul(_h, weights['out']) + biases['out']
return predict, outputs
def BIGRU(x, hidden_size):
input_x = tf.transpose(x, [1, 0, 2])
input_x = tf.unstack(input_x)
# define the forward and backward lstm cells
gru_fw_cell = rnn_cell.GRUCell(hidden_size)
gru_bw_cell = rnn_cell.GRUCell(hidden_size)
output, _, _ = rnn.bidirectional_rnn(gru_fw_cell, gru_bw_cell, input_x, dtype=tf.float32)
# output transformation to the original tensor type
output = tf.stack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def create_lstm_seq(input, length_labels, units_size, batch_size=32, num_rnn_layers=None, out_activation_func=None):
"""
LSTM with input of (batch_size, width as n_steps, and height*feature size)
"""
batch_size_d, height_d, width_d, feature_size_d = input.get_shape()
height, width, feature_size = height_d.value, width_d.value, feature_size_d.value
new_feature_size = feature_size*height
n_steps = width
input = tf.transpose(input, [0, 2, 1, 3])
input = tf.reshape(input, [-1, n_steps, new_feature_size])
input = tf.transpose(input, [1, 0, 2])
input = tf.reshape(input, [-1, new_feature_size])
inputs = tf.split(0, n_steps, input)
print 'New input shape', inputs[0].get_shape()
print 'number of steps', len(inputs)
lstm_fw_cell = rnn_cell.BasicLSTMCell(units_size, state_is_tuple=True)
lstm_bw_cell = rnn_cell.BasicLSTMCell(units_size, state_is_tuple=True)
if num_rnn_layers is not None:
lstm_fw_cell = rnn_cell.MultiRNNCell([lstm_fw_cell] * num_rnn_layers, state_is_tuple=True)
lstm_bw_cell = rnn_cell.MultiRNNCell([lstm_bw_cell] * num_rnn_layers, state_is_tuple=True)
try:
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, inputs, sequence_length=None,
dtype=tf.float32)
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, inputs,
dtype=tf.float32)
if out_activation_func is not None:
outputs = [out_activation_func(o) for o in outputs]
print 'New output shape', tf.pack(outputs).get_shape()
return locals()
def RNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(0, n_steps, x)
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
predict = tf.matmul(outputs[-1], weights['out']) + biases['out']
return predict, outputs
def RNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(0, n_steps, x)
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
outputs_all = tf.concat(0, outputs) # (N*batch) * 2*n_hidden
M = tanh(tf.matmul(outputs_all, W_h)) # (N*batch) * 2*hidden
a = tf.nn.softmax(tf.matmul(M, w)) # (N*batch) * 1
a = tf.reshape(a, [n_steps, -1, 1]) # N*batch*1
a = tf.transpose(a, [1, 2, 0]) # batch*1*N
#attention_2 = tf.squeeze(attention_2) # batch*N
#attention_2 = tf.reshape(attention_2, [-1,n_steps]) # batch*N
outputs_all = tf.reshape(outputs_all,
[n_steps, -1, 2 * n_hidden]) # N*batch*d
outputs_all = tf.transpose(outputs_all, [1, 0, 2]) # batch*N*d
#batch_s = x[0].get_shape()[0]
#batch_s = array_ops.shape(x[0])[0]
#batch_s = tf.int32(tf.size(x[0]))
#batch_s = tf.cast(batch_s,int)
#print(batch_s)
batch_s = 100
a = tf.split(0, batch_s, a)
outputs_all = tf.split(0, batch_s, outputs_all)
'''
o_2 = outputs_all[128][0:1,:,:]
return o_2
'''
r = []
for i in range(batch_s):
a_2 = a[i][0:1, :, :]
o_2 = outputs_all[i][0:1, :, :]
att = tf.reshape(a_2, [1, n_steps])
out = tf.reshape(o_2, [n_steps, 2 * n_hidden])
r.append(tf.matmul(att, out))
r = tf.concat(0, r) # batch*d
#attention_4 = tf.matmul(r,a)
_h = tanh(W_p * r + W_x * outputs[-1])
predict = tf.matmul(_h, weights['out']) + biases['out']
return predict, outputs
def utterance_encoder(self, enc_inputs):
# for the utterance level encoder: enc_inputs is of dimension (max_utter, batch_size, cell_size+max_images*image_embedding_size)
utterance_states = None
with tf.variable_scope(self.enc_scope_utter) as scope:
# init_state = self.enc_cells_utter.zero_state(self.batch_size, tf.float32)
# enc_inputs is of dimension (max_utter, batch_size, cell_size+max_images*image_embedding_size)
# _, states = rnn.rnn(self.enc_cells_utter, enc_inputs, scope=scope, dtype=tf.float32)
# _, states, _ = rnn.bidirectional_dynamic_rnn(self.enc_cells_utter[0], self.enc_cells_utter[1], enc_inputs, dtype=tf.float32, scope=scope)
_, states, _ = rnn.bidirectional_rnn(self.enc_cells_utter[0], self.enc_cells_utter[1], enc_inputs,
dtype=tf.float32, scope=scope)
# rnn.rnn takes a max_utter sized list of tensors of dimension (batch_size * cell_size+(max_images*image_embedding_size))
utterance_states = states
# utterance_states is of dimension (batch_size, cell_size)
# self.tf_print(utterance_states)
return utterance_states
def sentence_embedding(self, inputs, keep_prob, w):
with tf.device('/cpu:0'):
embedding_layer = tf.nn.embedding_lookup(w['word_embedding_w'],inputs)
# batch_size x max_len x word_embedding
cell_input = tf.transpose(embedding_layer,[1,0,2])
cell_input = tf.reshape(cell_input,[-1,self.hiddensize])
cell_input = tf.split(0,self.max_len,cell_input)
with tf.variable_scope('forward'):
lstm_fw_cell = rnn_cell.DropoutWrapper(rnn_cell.BasicLSTMCell(self.rnnsize,forget_bias=1.0,state_is_tuple=True),input_keep_prob=keep_prob,output_keep_prob=keep_prob)
with tf.variable_scope('backward'):
lstm_bw_cell = rnn_cell.DropoutWrapper(rnn_cell.BasicLSTMCell(self.rnnsize,forget_bias=1.0,state_is_tuple=True),input_keep_prob=keep_prob,output_keep_prob=keep_prob)
outputs,_,_ = rnn.bidirectional_rnn(lstm_fw_cell,lstm_bw_cell,cell_input,dtype=tf.float32)
# outputs shape: seq_len x [batch_size x (fw_cell_size + bw_cell_size)]
att = self.attention_layer(outputs,w)
return att
def BiRNN(x, weights, biases):
#具体过程参照test.py
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(0, n_steps, x)
#lstm cells
#forward direction cell
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
#backward direction cell
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
#get lstm cell output
try:
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
except Exception as e:
outputs = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def hierarchical_encoder(self):
'''Acts as encoder for word and utterance level.
Return - enc_states which is final output of utterance level encoder RNN which is of size batch_size*self.cell_size.'''
enc_states = self.encoder1_inputs + self.encoder2_inputs #Merging self.encoder1_inputs and self.encoder2_inputs list.
n_steps = self.max_len #Word level encoder steps is same as self.max_len which is 150.
#Calling self.encoder function, for word level encoder.
enc_states = self.encoder(enc_states, self.enc_cells[0],
self.enc_scopes[0], n_steps)
#Using bidirectional RNN for utterance level encoder.
_, utterance_state, _ = rnn.bidirectional_rnn(self.enc_cells[1][0],
self.enc_cells[1][1],
enc_states,
dtype=tf.float32)
return utterance_state
def bilstm(cell, x, scope=None):
with tf.variable_scope(scope or "birnn") as scope:
# input transformation
input_x = tf.transpose(x, [1, 0, 2])
input_x = tf.unpack(input_x)
output, _, _ = rnn.bidirectional_rnn(cell,
cell,
input_x,
dtype=tf.float32)
# output transformation to the original tensor type
output = tf.pack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def RNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(0, n_steps, x)
gru_fw_cell = rnn_cell.GRUCell(n_hidden)
gru_fw_cell = tf.nn.rnn_cell.DropoutWrapper(gru_fw_cell,
output_keep_prob=0.7)
gru_bw_cell = rnn_cell.GRUCell(n_hidden)
gru_bw_cell = tf.nn.rnn_cell.DropoutWrapper(gru_bw_cell,
output_keep_prob=0.7)
outputs, _, _ = rnn.bidirectional_rnn(gru_fw_cell,
gru_bw_cell,
x,
dtype=tf.float32)
outputs[-1] = tf.nn.dropout(outputs[-1], keep_prob=0.25)
predict = tf.matmul(outputs[-1], weights['out']) + biases['out']
return predict, outputs
def BiRNN(_X, _istate_fw, _istate_bw, _weights, _biases):
_X = tf.transpose(_X, [1, 0, 2])
_X = tf.reshape(_X, [-1, n_input])
fw_cell_1 = rnn_cell.BasicRNNCell(n_hidden)
bw_cell_1 = rnn_cell.BasicRNNCell(n_hidden)
fw_cell=rnn_cell.MultiRNNCell([fw_cell_1]*num_layers)
bw_cell=rnn_cell.MultiRNNCell([bw_cell_1]*num_layers)
_X = tf.split(0, n_steps, _X)
seq=np.int32(np.ones(batch_size)*Truncated)
outputs, statefw,statebw = rnn.bidirectional_rnn(fw_cell, bw_cell, _X,
initial_state_fw=_istate_fw,
initial_state_bw=_istate_bw,
sequence_length=seq)
return tf.matmul(outputs[-1], _weights['out']) + _biases['out']
def BiRNN(x, rnn_size, embed_size, n_steps, reuse=False):
with tf.variable_scope("lstm"):
if reuse:
tf.get_variable_scope().reuse_variables()
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, embed_size])
x = tf.split(0, n_steps, x)
lstm_fw_cell = rnn_cell.BasicLSTMCell(rnn_size, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(rnn_size, forget_bias=1.0)
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
res = outputs[-1]
return res
def generate_embedding_RNN_output(encoder_inputs,
cell,
num_encoder_symbols,
word_embedding_size,
num_heads=1,
dtype=dtypes.float32,
scope=None,
initial_state_attention=False,
sequence_length=None,
bidirectional_rnn=False):
"""
Generate RNN state outputs with word embeddings as inputs
- Note that this example code does not include output label dependency modeling.
One may add a loop function as in the rnn_decoder function in tf seq2seq.py
example to feed emitted label embedding back to RNN state.
"""
with variable_scope.variable_scope(scope or "generate_embedding_RNN_output"):
if bidirectional_rnn:
encoder_cell_fw = cell
encoder_cell_bw = cell
embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size])
encoder_embedded_inputs = list()
encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs]
encoder_outputs, encoder_state_fw, encoder_state_bw = rnn.bidirectional_rnn(
encoder_cell_fw, encoder_cell_bw, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype)
encoder_state = array_ops.concat(1, [encoder_state_fw, encoder_state_bw])
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size*2])
for e in encoder_outputs]
attention_states = array_ops.concat(1, top_states)
else:
encoder_cell = cell
embedding = variable_scope.get_variable("embedding", [num_encoder_symbols, word_embedding_size])
encoder_embedded_inputs = list()
encoder_embedded_inputs = [embedding_ops.embedding_lookup(embedding, encoder_input) for encoder_input in encoder_inputs]
encoder_outputs, encoder_state = rnn.rnn(
encoder_cell, encoder_embedded_inputs, sequence_length=sequence_length, dtype=dtype)
top_states = [array_ops.reshape(e, [-1, 1, cell.output_size])
for e in encoder_outputs]
attention_states = array_ops.concat(1, top_states)
return encoder_outputs, encoder_state, attention_states
def __init__(self, embedding, max_length, cell, num_symbols, dtype,
**kwargs):
self.embedding = embedding
self.cell = cell
# account for _GO and _EOS
max_length += 2
self.lengths = kwargs.get(
'lengths',
tf.placeholder(tf.int32, shape=[None], name="encoder_lengths"))
self.inputs = kwargs.get('inputs', [
tf.placeholder(
tf.int32, shape=[None], name="encoder_input{0}".format(i))
for i in xrange(max_length)
])
self.weights = kwargs.get('weights', [
tf.placeholder(
tf.float32, shape=[None], name="encoder_weight{0}".format(i))
for i in xrange(max_length)
])
inputs = [
embedding_ops.embedding_lookup(embedding, i) for i in self.inputs
]
self.outputs, self.state_fw, self.state_bw = rnn.bidirectional_rnn(
self.cell,
self.cell,
inputs,
sequence_length=self.lengths,
dtype=dtype)
#self.outputs = [tf.add(*tf.split(1, 2, o)) for o in self.outputs]
self.state = self.state_fw + self.state_bw # aggregate fw+bw states
#self.state = tf.concat(1, [self.state_fw, self.state_bw]) # concatenate fw+bw states
top_states = [
array_ops.reshape(e, [-1, 1, cell.output_size])
for e in self.outputs
]
self.attention_states = array_ops.concat(1, top_states)
def BiRNN(x, n_input, n_steps, n_hidden):
# Prepare data shape to match `bidirectional_rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshape to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, n_input])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, n_steps, x)
# Define lstm cells with tensorflow
# Forward direction cell
lstm_fw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Backward direction cell
lstm_bw_cell = rnn_cell.BasicLSTMCell(n_hidden, forget_bias=1.0)
# Get lstm cell output
outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell,
lstm_bw_cell,
x,
dtype=tf.float32)
return outputs
def runCTC(batch):
INPUT_PATH = '../TRAIN/All/mfcc/' #directory of MFCC nFeatures x nFrames 2-D array .npy files
TARGET_PATH = '../TRAIN/All/phone_y/' #directory of nPhonemes 1-D array .npy files
####Learning Parameters
learningRate = 0.001
momentum = 0.9
nEpochs = 300
batchSize = batch.shape[1]
####Network Parameters
nFeatures = 39 #12 MFCC coefficients + energy, and derivatives
nHidden = 256
nClasses = 30 #39 phonemes, plus the "blank" for CTC
####Load data
print('Loading data')
with open('TIMIT_data_prepared_for_CTC.pkl','rb') as f:
data= pickle.load(f)
input_list = batch
charmap = data['chars']
print(charmap)
charmap.append('_')
#batchedData, maxTimeSteps = data_lists_to_batches(input_list, target_list, batchSize)
maxTimeSteps = 776
totalN = len(input_list)
####Define graph
print('Defining graph')
graph = tf.Graph()
with graph.as_default():
####NOTE: try variable-steps inputs and dynamic bidirectional rnn, when it's implemented in tensorflow
####Graph input
inputX = tf.placeholder(tf.float32, shape=(maxTimeSteps, batchSize, nFeatures))
#Prep input data to fit requirements of rnn.bidirectional_rnn
# Reshape to 2-D tensor (nTimeSteps*batchSize, nfeatures)
inputXrs = tf.reshape(inputX, [-1, nFeatures])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_hidden)
inputList = tf.split(0, maxTimeSteps, inputXrs)
targetIxs = tf.placeholder(tf.int64)
targetVals = tf.placeholder(tf.int32)
targetShape = tf.placeholder(tf.int64)
targetY = tf.SparseTensor(targetIxs, targetVals, targetShape)
seqLengths = tf.placeholder(tf.int32, shape=(batchSize))
####Weights & biases
weightsOutH1 = tf.Variable(tf.truncated_normal([2, nHidden],
stddev=np.sqrt(2.0 / (2*nHidden))))
biasesOutH1 = tf.Variable(tf.zeros([nHidden]))
weightsOutH2 = tf.Variable(tf.truncated_normal([2, nHidden],
stddev=np.sqrt(2.0 / (2*nHidden))))
biasesOutH2 = tf.Variable(tf.zeros([nHidden]))
weightsClasses = tf.Variable(tf.truncated_normal([nHidden, nClasses],
stddev=np.sqrt(2.0 / nHidden)))
biasesClasses = tf.Variable(tf.zeros([nClasses]))
####Network
forwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True)
backwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True)
fbH1, _, _ = bidirectional_rnn(forwardH1, backwardH1, inputList, dtype=tf.float32,
scope='BDLSTM_H1')
fbH1rs = [tf.reshape(t, [batchSize, 2, nHidden]) for t in fbH1]
outH1 = [tf.reduce_sum(tf.mul(t, weightsOutH1), reduction_indices=1) + biasesOutH1 for t in fbH1rs]
logits = [tf.matmul(t, weightsClasses) + biasesClasses for t in outH1]
####Optimizing
logits3d = tf.pack(logits)
loss = tf.reduce_mean(ctc.ctc_loss(logits3d, targetY, seqLengths))
optimizer = tf.train.MomentumOptimizer(learningRate, momentum).minimize(loss)
####Evaluating
logitsMaxTest = tf.slice(tf.argmax(logits3d,2), [0, 0], [seqLengths[0], 1])
predictions = tf.to_int32(ctc.ctc_beam_search_decoder(logits3d, seqLengths)[0][0])
errorRate = tf.reduce_sum(tf.edit_distance(predictions, targetY, normalize=False)) / \
tf.to_float(tf.size(targetY.values))
####Run session
with tf.Session(graph=graph) as session:
print('Initializing')
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state('/users/TeamASR/models')
if ckpt and tf.gfile.Exists(ckpt.model_checkpoint_path):
print("Reading model parameters from %s" % ckpt.model_checkpoint_path)
saver.restore(session, ckpt.model_checkpoint_path)
else:
print("Created model with fresh parameters.")
session.run(tf.initialize_all_variables())
feedDict = {inputX: batch, seqLengths: (np.ones([batchSize])*776)}
logit = session.run([logits3d], feed_dict=feedDict)
return logit
####Weights & biases
stddev = np.sqrt(2.0 / (2 * nHidden))
truncated_normal = tf.truncated_normal([2, nHidden], stddev=stddev)
weightsOutH1 = tf.Variable(truncated_normal)
biasesOutH1 = tf.Variable(tf.zeros([nHidden]))
weightsOutH2 = tf.Variable(truncated_normal)
biasesOutH2 = tf.Variable(tf.zeros([nHidden]))
half_normal = tf.truncated_normal([nHidden, nClasses], stddev=np.sqrt(2.0 / nHidden))
weightsClasses = tf.Variable(half_normal)
biasesClasses = tf.Variable(tf.zeros([nClasses]))
####Network
forwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True)
backwardH1 = rnn_cell.LSTMCell(nHidden, use_peepholes=True, state_is_tuple=True)
print("building bidirectional_rnn ... SLOW!!!")
fbH1, _, _ = bidirectional_rnn(forwardH1, backwardH1, inputList, dtype=tf.float32, scope='BDLSTM_H1')
print("done building rnn")
print("building fbH1rs ")
fbH1rs = [tf.reshape(t, [Size, 2, nHidden]) for t in fbH1]
print("building outH1 ")
outH1 = [tf.reduce_sum(tf.multiply(t, weightsOutH1), axis=1) + biasesOutH1 for t in fbH1rs]
print("building logits ")
logits = [tf.matmul(t, weightsClasses) + biasesClasses for t in outH1]
print("len(outH1) %d"% len(outH1))
####Optimizing
print("building loss")
logits3d = tf.stack(logits)
loss = tf.reduce_mean(ctc.ctc_loss(logits3d, targetY, seqLengths))
out = tf.identity(loss, 'ctc_loss_mean')
optimizer = tf.train.MomentumOptimizer(learningRate, momentum).minimize(loss)
targ_ids = tf.placeholder(tf.int64)
targ_vals = tf.placeholder(tf.int32)
targ_shape = tf.placeholder(tf.int64)
targets = tf.SparseTensor(targ_ids,targ_vals, targ_shape)
l_reshape1 = tf.reshape(l_in,(-1,input_size) )
W1 = weight_variable([input_size,hidden_size])
b1 = bias_variable([hidden_size])
h_1 = tf.nn.relu(tf.matmul(l_reshape1,W1) + b1)
l_reshape2 = tf.reshape(h_1, [-1, n_time_steps,hidden_size])
rnn_input_tr = tf.transpose(l_reshape2,[1,0,2])
rnn_input_re = tf.reshape(rnn_input_tr,[-1,hidden_size])
rnn_input = tf.split(0, n_time_steps, rnn_input_re)
lstm_fw_cell = rnn_cell.BasicLSTMCell(hidden_size, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(hidden_size, forget_bias=1.0)
# Get lstm cell output
lstm_outputs, _, _ = rnn.bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, rnn_input, dtype=tf.float32)
lstm_outputs_re=tf.reshape(lstm_outputs, [n_time_steps, -1, 2*hidden_size])
lstm_output_tr=tf.transpose(lstm_outputs_re, [1,0,2])
W2 = weight_variable([2*hidden_size,output_size])
b2 = bias_variable([output_size])
#n_batch, n_time_steps, n_features = l_in.input_var.shape #Unnecessary in this version. Just collecting the info so that we can reshape the output back to the original shape
l_reshape3 = tf.reshape(lstm_output_tr,[-1,2*hidden_size] )
h_2 = tf.matmul(l_reshape3,W2) + b2
l_reshape4 = tf.reshape(h_2,[-1,output_size])
l_soft = tf.nn.softmax(l_reshape4)
l_soft_reshaped = tf.reshape(l_soft,[-1,n_time_steps,output_size])
l_soft_tr = tf.transpose(l_soft_reshaped, [1,0,2])
loss = tf.reduce_mean(tf.nn.ctc_loss(l_soft_tr, targets,seqLengths))
optimizer = tf.train.AdamOptimizer(learningRate).minimize(loss)
seq5 = tf.constant([[1,2,3,4,5,6],
[1,2,3,4,5,6]]) # batch_size = 2
unpacked_seqs_embeds = [None,]
seqs_embeds = [None,]
for seq in [seq1,seq2,seq3,seq4,seq5]:
seqs_embed = tf.nn.embedding_lookup(embed,seq)
seqs_embeds.append(seqs_embed)
unpacked_seqs_embeds.append(tf.unpack(tf.transpose(seqs_embed,perm=[1, 0, 2])))
with tf.variable_scope("naive",initializer=tf.truncated_normal_initializer(seed=1)) as scope:
if FLAGS.rnn_type == "fw":
n_output1,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[1],dtype=tf.float32,sequence_length=None )
elif FLAGS.rnn_type == "bi":
n_output1,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[1],dtype=tf.float32,sequence_length=None )
with tf.variable_scope("naive2",initializer=tf.truncated_normal_initializer(seed=1)) as scope:
if FLAGS.rnn_type == "fw":
n_output2,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[2],dtype=tf.float32,sequence_length=None )
elif FLAGS.rnn_type == "bi":
n_output2,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[2],dtype=tf.float32,sequence_length=None )
with tf.variable_scope("batch1",initializer=tf.truncated_normal_initializer(seed=1)) as scope:
if FLAGS.rnn_type == "fw":
n_output3,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[3],dtype=tf.float32,sequence_length=None )
elif FLAGS.rnn_type == "bi":
n_output3,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[3],dtype=tf.float32,sequence_length=None )
with tf.variable_scope("batch2",initializer=tf.truncated_normal_initializer(seed=1)) as scope:
if FLAGS.rnn_type == "fw":
n_output4,_ = rnn.rnn(rnn_cell.BasicRNNCell(2),unpacked_seqs_embeds[4],dtype=tf.float32,sequence_length=None )
elif FLAGS.rnn_type == "bi":
n_output4,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(1),rnn_cell.BasicRNNCell(1),unpacked_seqs_embeds[4],dtype=tf.float32,sequence_length=None )
rnn_dim = 1024
max_len = 100
label = tf.constant(np.random.rand(rnn_dim),dtype=tf.float32)
embed = tf.Variable(tf.constant(np.random.rand(vocab_size,embed_dim),dtype=tf.float32),trainable=False)
seq_len= tf.placeholder(tf.int32,[batch_size])
seq = tf.placeholder(tf.int32,[batch_size,max_len])
seqs_embed = tf.nn.embedding_lookup(embed,seq)
unpacked_seqs_embed = tf.unpack(tf.transpose(seqs_embed,perm=[1, 0, 2]))
with tf.variable_scope("naive",initializer=tf.truncated_normal_initializer(seed=1)) as scope:
if FLAGS.rnn_type == "fw":
n_output1,_ = rnn.rnn(rnn_cell.BasicRNNCell(rnn_dim),unpacked_seqs_embed,dtype=tf.float32,sequence_length=seq_len )
elif FLAGS.rnn_type == "bi":
n_output1,_,_ = rnn.bidirectional_rnn(rnn_cell.BasicRNNCell(rnn_dim/2),rnn_cell.BasicRNNCell(rnn_dim/2),unpacked_seqs_embeds,dtype=tf.float32,sequence_length=seq_len )
with tf.variable_scope("dynamic",initializer=tf.truncated_normal_initializer(seed=1)) as scope:
if FLAGS.rnn_type == "fw":
n_output2,_ = rnn.dynamic_rnn(rnn_cell.BasicRNNCell(rnn_dim),seqs_embed,dtype=tf.float32,time_major=False,sequence_length=seq_len )
elif FLAGS.rnn_type == "bi" and tf.__version__.startswith("0.10"):
n_output2,_ = rnn.bidirectional_dynamic_rnn(rnn_cell.BasicRNNCell(rnn_dim/2),rnn_cell.BasicRNNCell(rnn_dim/2),seqs_embed,dtype=tf.float32,time_major=False,sequence_length=seq_len )
avgLosses = [] # average pooling loss
if FLAGS.rnn_type == "fw" or (FLAGS.rnn_type == "bi" and tf.__version__.startswith("0.10")):
outputs = [n_output1,n_output2]
else:
outputs = [n_output1]
for i,n_output in enumerate(outputs):
def __init__(self, sequence_length, num_classes, vocab_size, embedding_size,
num_filter, filter_lengths, pool_lengths, dropout, hidden_size, l2_reg_lambda):
"""
sequence_length: the length of our sentences (paddled) = 336
num_classes: 2
vocab_size: 111
embedding_size: 8
num_filter: 128
filter_lengths: [5, 3]
pool_lengths: [2, 2]
dropout: 0.5
hidden_size: 128
l2_reg_lambda: 5 * 10 ^ -4
"""
assert len(filter_lengths) == len(pool_lengths)
# Placeholders for input, output and initial state of bi-lstm
input_x = tf.placeholder(tf.int32, shape=[None, sequence_length], name='input_x')
input_y = tf.placeholder(tf.float32, shape=[None, num_classes], name='input_y')
istate_fw = tf.placeholder("float", [None, 2 * hidden_size])
istate_bw = tf.placeholder("float", [None, 2 * hidden_size])
# Keeping track of l2 regularization loss (optional)
l2_loss = tf.constant(0.0)
# Embedding layer
with tf.name_scope('embedding'):
W_embed = tf.Variable(
tf.random_uniform(shape=[vocab_size, embedding_size], minval=-1, maxval=1),
name='W_embed')
embedded_chars = tf.nn.embedding_lookup(W_embed, input_x)
embedded_chars_expanded = tf.expand_dims(embedded_chars, -1)
# Convolution + maxpool 0
num_conv = 0
with tf.name_scope('conv-maxpool-%d' % num_conv):
# Conv0
filter_shape0 = [filter_lengths[num_conv], embedding_size, 1, num_filter]
W_conv0 = tf.Variable(tf.truncated_normal(shape=filter_shape0, stddev=0.1), name='W0')
b_conv0 = tf.Variable(tf.constant(value=0.1, shape=[num_filter]), name='b0')
conv0 = tf.nn.conv2d(
input=embedded_chars_expanded,
filter=W_conv0,
strides=[1, 1, 1, 1],
padding="VALID",
name='conv0'
)
# Apply nonlinearity
h0 = tf.nn.relu(tf.nn.bias_add(conv0, b_conv0), name='relu0')
# Max pooling
pool0 = tf.nn.max_pool(
value=tf.expand_dims(tf.squeeze(h0, [2]), -1), # [?, conv, 1, num_filter] -> [?, conv, num_filter, 1]
ksize=[1, pool_lengths[num_conv], 1, 1],
strides=[1, pool_lengths[num_conv], 1, 1],
padding='VALID',
name='pool0'
)
num_conv += 1
# Convolution + maxpool 1
with tf.name_scope('conv-maxpool-%d' % num_conv):
# Conv1
filter_shape1 = [filter_lengths[num_conv], num_filter, 1, num_filter]
W_conv1 = tf.Variable(tf.truncated_normal(shape=filter_shape1, stddev=0.1), name='W1')
b_conv1 = tf.Variable(tf.constant(value=0.1, shape=[num_filter]), name='b1')
conv1 = tf.nn.conv2d(
input=pool0,
filter=W_conv1,
strides=[1, 1, 1, 1],
padding="VALID",
name='conv1')
# Apply nonlinearity
h1 = tf.nn.relu(tf.nn.bias_add(conv1, b_conv1), name='relu1')
# Max pooling
pool1 = tf.nn.max_pool(
value=tf.expand_dims(tf.squeeze(h1, [2]), -1),
ksize=[1, pool_lengths[num_conv], 1, 1],
strides=[1, pool_lengths[num_conv], 1, 1],
padding='VALID',
name='pool1')
# Add dropout 0
with tf.name_scope('drop_conv'):
drop_conv = tf.nn.dropout(pool1, keep_prob=dropout)
# Reshape tensor
with tf.name_scope('reshape_conv_rnn'):
_input = tf.squeeze(drop_conv, [3])
_input = tf.transpose(_input, [1, 0, 2])
_input = tf.reshape(_input, [-1, hidden_size])
_input = tf.split(0, 82, _input) # TODO: 82 대신 자동화 되서 나온 숫자.
# Bidirectional
with tf.name_scope('Bi-LSTM'):
lstm_fw_cell = rnn_cell.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0)
lstm_bw_cell = rnn_cell.BasicLSTMCell(num_units=hidden_size, forget_bias=1.0)
output, _, _ = rnn.bidirectional_rnn(
cell_fw=lstm_fw_cell, cell_bw=lstm_bw_cell, inputs=_input,
initial_state_fw=istate_fw, initial_state_bw=istate_bw)
# Add dropout1
with tf.name_scope('drop_rnn'):
drop_rnn = tf.nn.dropout(output[-1], keep_prob=0.5)
# Final (unnormalized) scores and predictions
with tf.name_scope('softmax'):
W_soft = tf.get_variable(
'W', shape=[2*hidden_size, num_classes], initializer=tf.contrib.layers.xavier_initializer())
# W_soft = tf.Variable(tf.truncated_normal(shape=[2*hidden_size, num_classes]))
b_soft = tf.Variable(tf.constant(value=0.1, shape=[num_classes]))
l2_loss += tf.nn.l2_loss(W_soft)
l2_loss += tf.nn.l2_loss(b_soft)
# scores = tf.matmul(drop_rnn, W_soft) + b_soft
scores = tf.nn.xw_plus_b(x=drop_rnn, weights=W_soft, biases=b_soft, name='scores')
predictions = tf.argmax(scores, 1, name='predictions')
# Calculate loss function
with tf.name_scope('loss'):
losses = tf.nn.softmax_cross_entropy_with_logits(scores, input_y)
self.loss = tf.reduce_mean(losses) + l2_reg_lambda * l2_loss / 2
# Accuracy
with tf.name_scope("accuracy"):
correct_predictions = tf.equal(predictions, tf.argmax(input_y, 1))
self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")
def stack_bidirectional_rnn(cells_fw,
cells_bw,
inputs,
initial_states_fw=None,
initial_states_bw=None,
dtype=None,
sequence_length=None,
scope=None):
"""Creates a bidirectional recurrent neural network.
Stacks several bidirectional rnn layers. The combined forward and backward
layer outputs are used as input of the next layer. tf.bidirectional_rnn
does not allow to share forward and backward information between layers.
The input_size of the first forward and backward cells must match.
The initial state for both directions is zero and no intermediate states
are returned.
As described in https://arxiv.org/abs/1303.5778
Args:
cells_fw: List of instances of RNNCell, one per layer,
to be used for forward direction.
cells_bw: List of instances of RNNCell, one per layer,
to be used for backward direction.
inputs: A length T list of inputs, each a tensor of shape
[batch_size, input_size], or a nested tuple of such elements.
initial_states_fw: (optional) A list of the initial states (one per layer)
for the forward RNN.
Each tensor must has an appropriate type and shape
`[batch_size, cell_fw.state_size]`.
initial_states_bw: (optional) Same as for `initial_states_fw`, but using
the corresponding properties of `cells_bw`.
dtype: (optional) The data type for the initial state. Required if
either of the initial states are not provided.
sequence_length: (optional) An int32/int64 vector, size `[batch_size]`,
containing the actual lengths for each of the sequences.
scope: VariableScope for the created subgraph; defaults to None.
Returns:
A tuple (outputs, output_state_fw, output_state_bw) where:
outputs is a length `T` list of outputs (one for each input), which
are depth-concatenated forward and backward outputs.
output_states_fw is the final states, one tensor per layer,
of the forward rnn.
output_states_bw is the final states, one tensor per layer,
of the backward rnn.
Raises:
TypeError: If `cell_fw` or `cell_bw` is not an instance of `RNNCell`.
ValueError: If inputs is None, not a list or an empty list.
"""
if not cells_fw:
raise ValueError("Must specify at least one fw cell for BidirectionalRNN.")
if not cells_bw:
raise ValueError("Must specify at least one bw cell for BidirectionalRNN.")
if not isinstance(cells_fw, list):
raise ValueError("cells_fw must be a list of RNNCells (one per layer).")
if not isinstance(cells_bw, list):
raise ValueError("cells_bw must be a list of RNNCells (one per layer).")
if len(cells_fw) != len(cells_bw):
raise ValueError("Forward and Backward cells must have the same depth.")
if initial_states_fw is not None and (not isinstance(cells_fw, list) or
len(cells_fw) != len(cells_fw)):
raise ValueError(
"initial_states_fw must be a list of state tensors (one per layer).")
if initial_states_bw is not None and (not isinstance(cells_bw, list) or
len(cells_bw) != len(cells_bw)):
raise ValueError(
"initial_states_bw must be a list of state tensors (one per layer).")
states_fw = []
states_bw = []
prev_layer = inputs
with vs.variable_scope(scope or "stack_bidirectional_rnn"):
for i, (cell_fw, cell_bw) in enumerate(zip(cells_fw, cells_bw)):
initial_state_fw = None
initial_state_bw = None
if initial_states_fw:
initial_state_fw = initial_states_fw[i]
if initial_states_bw:
initial_state_bw = initial_states_bw[i]
with vs.variable_scope("cell_%d" % i) as cell_scope:
prev_layer, state_fw, state_bw = rnn.bidirectional_rnn(
cell_fw,
cell_bw,
prev_layer,
initial_state_fw=initial_state_fw,
initial_state_bw=initial_state_bw,
sequence_length=sequence_length,
dtype=dtype,
scope=cell_scope)
states_fw.append(state_fw)
states_bw.append(state_bw)
return prev_layer, tuple(states_fw), tuple(states_bw)
