def biLSTM(x, hidden_size):
# biLSTM��
# ���ܣ����bidirectional_lstm����
# ������
# x: [batch, height, width] / [batch, step, embedding_size]
# hidden_size: lstm���ز�ڵ����
# �����
# output: [batch, height, 2*hidden_size] / [batch, step, 2*hidden_size]
# input transformation
input_x = tf.transpose(x, [1, 0, 2])
# input_x = tf.reshape(input_x, [-1, w])
# input_x = tf.split(0, h, input_x)
input_x = tf.unstack(input_x)
# define the forward and backward lstm cells
# lstm_fw_cell = rnn_cell.BasicLSTMCell(hidden_size, forget_bias=1.0, state_is_tuple=True)
lstm_cell = rnn.BasicLSTMCell(hidden_size,
forget_bias=1.0,
state_is_tuple=True) # 修改的
# lstm_bw_cell = rnn_cell.BasicLSTMCell(hidden_size, forget_bias=1.0, state_is_tuple=True)
output, states = rnn.static_rnn(lstm_cell, input_x, dtype=tf.float32)
# output transformation to the original tensor type
output = tf.stack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def rnn_model(x, weights, biases):
"""RNN (LSTM or GRU) model for image"""
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, n_input])
x = tf.split(x, n_steps, 0)
lstm_cell = rnn.BasicLSTMCell(n_hidden, forget_bias=1.0)
outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
return tf.matmul(outputs[-1], weights) + biases
def RNN(x, weights, biases, gru=False):
# configuring so you can get it as needed for the 28 pixels
x = tf.unstack(x, nSteps, 1)
# find which lstm to use in the documentation
lstmCell = rnn.GRUCell(nHidden) if gru else rnn.BasicLSTMCell(nHidden)
# for the rnn where to get the output and hidden state
outputs, states = tf.nn.static_rnn(lstmCell, x, dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def recurrent_neural_network(x):
x = tf.transpose(x, [1,0,2])
x = tf.reshape(x, [-1, chunk_size])
x = tf.split(x, n_chunks, 0)
l1 = tf.add(tf.matmul(x,hidden_1_layer['weight']), hidden_1_layer['bias'])
l1 = tf.nn.relu(l1)
l2 = tf.add(tf.matmul(l1,hidden_2_layer['weight']), hidden_2_layer['bias'])
l2 = tf.nn.tanh(l2)#relu(l2)
lstm_cell = rnn.BasicLSTMCell(rnn_size)
outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
output = tf.matmul(outputs[-1],output_layer['weight']) + output_layer['bias']
def recurrent_neural_network_model(x):
# number of layers i
layer = {
'weights': tf.Variable(tf.random_normal([rnn_size, n_classes])),
'bias': tf.Variable(tf.random_normal([n_classes]))
}
print(x)
x = tf.transpose(x, [1, 0, 2])
print(x)
x = tf.reshape(x, [-1, chunk_size])
print(x)
x = tf.split(x, n_chunks, 0)
print(x)
from tensorflow.contrib import rnn
lstm_cell = rnn.BasicLSTMCell(rnn_size)
outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
output = tf.matmul(outputs[-1], layer['weights']) + layer['bias']
return output
def recurrent_neural_network(x):
layer = {
'weights': tf.Variable(tf.random_normal([rnn_size, n_classes])),
'biases': tf.Variable(tf.random_normal([n_classes]))
}
# Make it [n_chunks, None, chunk_size]
x = tf.transpose(x, [1, 0, 2])
# Make it [n_chunks*n, chunk_size]
x = tf.reshape(x, [-1, time_steps])
x = tf.split(x, n_chunks, 0)
# lstm_cell = rnn_cell.BasicLSTMCell(rnn_size,state_is_tuple=True)
# outputs, states = rnn.static_rnn(lstm_cell, x, dtype=tf.float32)
lstm_layer = rnn.BasicLSTMCell(rnn_size)
outputs, states = rnn.static_rnn(lstm_layer, x, dtype="float32")
# lstm_layer2=rnn.BasicLSTMCell(rnn_size2)
# outputs,states =rnn.static_rnn(lstm_layer2,outputs,dtype="float32")
output = tf.matmul(outputs[-1], layer['weights']) + layer['biases']
return output
ytarget = np.random.rand(batch_size, hidden_size).astype(np.float32)
with tf.device('/gpu:0'):
x = [
tf.placeholder(tf.float32, [batch_size, hidden_size], name="x")
for i in range(seq_length)
]
y = tf.placeholder(tf.float32, [batch_size, hidden_size], name="y")
if network_type == 'rnn':
cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
elif network_type == 'lstm':
cell = tf.nn.rnn_cell.LSTMCell(hidden_size, hidden_size)
elif network_type == 'basic_lstm':
cell = rnn.BasicLSTMCell(hidden_size)
#cell = tf.nn.rnn_cell.BasicLSTMCell(hidden_size)
else:
raise Exception('Unknown network! ' + network_type)
print("Compiling...")
start = time.time()
output, _cell_state = rnn.static_rnn(cell, x, dtype=tf.float32)
cost = tf.reduce_sum((output[-1] - y)**2)
optim = tf.train.GradientDescentOptimizer(0.01)
train_op = optim.minimize(cost)
session = tf.Session()
session.run(tf.initialize_all_variables())
session.run(train_op, feed_dict=get_feed_dict(xinput, ytarget))
def embedding_attention_qualvec(encoder_inputs,
decoder_inputs,
for_cell,
bac_cell,
num_encoder_symbols,
num_decoder_symbols,
embedding_size,
maxout_size,
num_heads=1,
output_projection=None,
feed_previous=False,
dtype=None,
scope=None,
initial_state_attention=False):
"""Embedding sequence-to-sequence model with attention.
"""
print "inside"
if output_projection is None:
raise ValueError("Must provide output projection")
with variable_scope.variable_scope(scope or "embedding_attention_seq2seq",
dtype=dtype) as scope:
dtype = scope.dtype
# Encoder.
# encoder_outputs, encoder_state = rnn.rnn(encoder_cell,
# encoder_inputs,
# dtype=dtype)
embedding = variable_scope.get_variable(
"embedding", [num_encoder_symbols, embedding_size])
print embedding
encoder_inputs = [
embedding_ops.embedding_lookup(embedding, i)
for i in encoder_inputs
]
# Forward direction cell
for_cell = rnn.BasicLSTMCell(128, forget_bias=1.0)
# Backward direction cell
bac_cell = rnn.BasicLSTMCell(128, forget_bias=1.0)
print encoder_inputs
print for_cell
encoder_outputs, encoder_state_fw, encoder_state_bw = tf.contrib.rnn.static_bidirectional_rnn(
for_cell, bac_cell, encoder_inputs, dtype=dtype)
# encoder_outputs = tf.concat(2, outputs)
# encoder_outputs, encoder_state_fw = tf.nn.rnn(
# for_cell, encoder_inputs, dtype=dtype)
# First calculate a concatenation of encoder outputs to put attention on.
print encoder_outputs
print "rnn"
top_states = [
array_ops.reshape(e, [-1, 1, 2 * for_cell.output_size])
for e in encoder_outputs
]
# Dimension 1 are the encoder outputs. Dim2 are the
attention_states = tf.concat(top_states, 1)
# Decoder.
output_size = None
print output_size
if isinstance(feed_previous, bool):
print "iff"
return embedding_attention_decoder(
decoder_inputs,
encoder_state_fw,
encoder_state_bw,
attention_states,
for_cell,
bac_cell,
num_decoder_symbols,
embedding_size,
maxout_size=maxout_size,
num_heads=num_heads,
output_size=output_size,
output_projection=output_projection,
feed_previous=feed_previous,
initial_state_attention=initial_state_attention)
# If feed_previous is a Tensor, we construct 2 graphs and use cond.
def decoder(feed_previous_bool):
reuse = None if feed_previous_bool else True
with variable_scope.variable_scope(
variable_scope.get_variable_scope(), reuse=reuse) as scope:
outputs, state = embedding_attention_decoder(
decoder_inputs,
encoder_state_fw,
encoder_state_bw,
attention_states,
for_cell,
bac_cell,
num_decoder_symbols,
embedding_size,
maxout_size=maxout_size,
num_heads=num_heads,
output_size=output_size,
output_projection=output_projection,
feed_previous=feed_previous_bool,
update_embedding_for_previous=False,
initial_state_attention=initial_state_attention)
state_list = [state]
if nest.is_sequence(state):
state_list = nest.flatten(state)
return outputs + state_list
outputs_and_state = control_flow_ops.cond(feed_previous,
lambda: decoder(True),
lambda: decoder(False))
outputs_len = len(
decoder_inputs) # Outputs length same as decoder inputs.
state_list = outputs_and_state[outputs_len:]
state = state_list[0]
if nest.is_sequence(encoder_state):
state = nest.pack_sequence_as(structure=encoder_state,
flat_sequence=state_list)
return outputs_and_state[:outputs_len], state