def dynamic_rnn():
# load data
train_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.train.pos'))
dev_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.devel.pos'))
embedding = tf.get_variable("embedding", [FLAGS.emb_size, FLAGS.word_dim],
tf.float32)
with tf.name_scope('placeholder'):
x_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
y_ = tf.placeholder(tf.int32, [None])
mask = tf.placeholder(tf.int32, [None])
output_keep_prob = tf.placeholder(tf.float32)
# x:[batch_size,n_steps,n_input]
x = tf.nn.embedding_lookup(embedding, x_)
lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden,
state_is_tuple=True,
activation=tf.nn.relu)
lstm_cell = tf.nn.rnn_cell.DropoutWrapper(lstm_cell,
output_keep_prob=1 -
FLAGS.dropout)
# Get lstm cell output
outputs, _ = tf.nn.dynamic_rnn(lstm_cell, x, dtype=tf.float32)
outputs = tf.reshape(outputs, [-1, FLAGS.n_hidden])
# define weights and biases of logistic layer
with tf.variable_scope('linear'):
weights = tf.get_variable("weight", [FLAGS.n_hidden, FLAGS.n_classes],
tf.float32)
biases = tf.get_variable("biases", [FLAGS.n_classes], tf.float32)
logits = tf.matmul(outputs, weights) + biases
#loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits,y_) * tf.cast(mask,tf.float32))
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_))
train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
y = tf.cast(tf.nn.in_top_k(logits, y_, 1), tf.int32) * mask
correct = tf.reduce_sum(y)
with tf.Session(config=util.gpu_config()) as sess:
sess.run(tf.initialize_all_variables())
FLAGS.epoch_size = train_data.numbers() // FLAGS.batch_size
for step in range(FLAGS.epoch_size * FLAGS.epoch_step):
batch_x, batch_y, mask_feed = util.padding(
*train_data.next_batch(FLAGS.batch_size))
sess.run(train_op,
feed_dict={
x_: batch_x,
y_: batch_y,
output_keep_prob: 1 - FLAGS.dropout,
mask: mask_feed
})
if step % FLAGS.epoch_size == 0:
evalution(sess, correct, x_, y_, mask, output_keep_prob,
dev_data)
import tensorflow as tf
import os
import numpy as np
import input_data
datasetPath = "Pima-training-set.txt"
testsetPath = "Pima-prediction-set.txt"
traindata = input_data.read_dataset(datasetPath)
testdata = input_data.read_dataset(testsetPath)
x = tf.placeholder(tf.float32, [None, 8])
W = tf.Variable(tf.zeros([8, 2]))
b = tf.Variable(tf.zeros([2]))
y = tf.nn.softmax(tf.matmul(x, W) + b)
y_ = tf.placeholder(tf.float32, [None, 2])
cross_entropy = tf.reduce_mean(
-tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1]))
train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)
init = tf.initialize_all_variables()
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
sess = tf.Session()
sess.run(init)
# print traindata.next_batch(8);exit()
def bi_lstm():
tf.set_random_seed(1)
# load data
train_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.train.pos'))
dev_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.devel.pos'))
with tf.device('/cpu:0'):
embedding = tf.get_variable("embedding",
[FLAGS.emb_size, FLAGS.word_dim],
tf.float32)
with tf.name_scope('placeholder'):
x_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
y_ = tf.placeholder(tf.int32, [None])
mask = tf.placeholder(tf.int32, [None])
output_keep_prob = tf.placeholder(tf.float32)
seq_len = tf.placeholder(tf.int32, [None])
# x:[batch_size,n_steps,n_input]
x = tf.nn.embedding_lookup(embedding, x_)
with tf.device('/gpu:2'):
# lstm cell
lstm_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden)
lstm_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden)
# dropout
lstm_cell_fw = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_fw,
output_keep_prob=1 -
FLAGS.dropout)
lstm_cell_bw = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_bw,
output_keep_prob=1 -
FLAGS.dropout)
# Get lstm cell output
outputs, _ = tf.nn.bidirectional_dynamic_rnn(lstm_cell_fw,
lstm_cell_bw,
x,
sequence_length=seq_len,
dtype=tf.float32)
outputs = tf.concat(2, outputs)
outputs = tf.reshape(outputs, [-1, 2 * FLAGS.n_hidden])
# define weights and biases of logistic layer
with tf.variable_scope('linear'):
weights = tf.get_variable("weight",
[2 * FLAGS.n_hidden, FLAGS.n_classes],
tf.float32)
biases = tf.get_variable("biases", [FLAGS.n_classes], tf.float32)
logits = tf.matmul(outputs, weights) + biases
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_) *
tf.cast(mask, tf.float32))
train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
y = tf.cast(tf.nn.in_top_k(logits, y_, 1), tf.int32) * mask
correct = tf.reduce_sum(y)
with tf.Session(config=util.gpu_config()) as sess:
sess.run(tf.global_variables_initializer())
FLAGS.epoch_size = train_data.numbers() // FLAGS.batch_size
for step in range(FLAGS.epoch_size * FLAGS.epoch_step):
batch_x, batch_y, mask_feed = util.padding(
*train_data.next_batch(FLAGS.batch_size))
sequence_length = batch_x.shape[1] * np.ones(
[FLAGS.batch_size], np.int32)
sess.run(train_op,
feed_dict={
x_: batch_x,
y_: batch_y,
output_keep_prob: 1 - FLAGS.dropout,
mask: mask_feed,
seq_len: sequence_length
})
if step % 100 == 0:
evalution(sess, correct, x_, y_, mask, output_keep_prob,
seq_len, dev_data)
def bi_lstm_crf():
# load data
print 'start read dataset'
train_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.train.pos'))
dev_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.devel.pos'))
dev_data.fake_data(FLAGS.batch_size)
print 'stop read dataset'
tf.set_random_seed(1)
# 词向量放到cpu里面可以节省显存
with tf.device('/cpu:0'):
with tf.variable_scope('embedding') as scope:
random_embedding = tf.get_variable(
name="random_embedding",
shape=[FLAGS.emb_size, FLAGS.word_dim],
dtype=tf.float32)
with tf.name_scope('placeholder'):
x_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
y_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None])
output_keep_prob = tf.placeholder(tf.float32)
sequence_length = tf.reduce_sum(tf.sign(x_), reduction_indices=1)
sequence_length = tf.cast(sequence_length, tf.int32)
with tf.device('/gpu:2'):
with tf.variable_scope('input_layer'):
# x:[batch_size,n_steps,n_input]
x = tf.nn.embedding_lookup(random_embedding, x_)
# lstm cell
with tf.name_scope('bi_lstm_layer'):
lstm_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden)
lstm_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden)
# Get lstm cell output
outputs, _ = tf.nn.bidirectional_dynamic_rnn(
cell_fw=lstm_cell_fw,
cell_bw=lstm_cell_bw,
inputs=x,
sequence_length=sequence_length,
dtype=tf.float32)
outputs = tf.concat(2, outputs)
outputs = tf.reshape(outputs, [-1, 2 * FLAGS.n_hidden])
outputs = tf.nn.dropout(outputs, keep_prob=output_keep_prob)
with tf.variable_scope('Softmax'):
weights = tf.get_variable(
name="weights",
shape=[2 * FLAGS.n_hidden, FLAGS.n_classes],
dtype=tf.float32,
initializer=tf.truncated_normal_initializer(stddev=0.01))
biases = tf.get_variable(name="biases",
shape=[FLAGS.n_classes],
dtype=tf.float32)
matricized_unary_scores = tf.matmul(outputs, weights) + biases
unary_scores = tf.reshape(matricized_unary_scores,
[FLAGS.batch_size, -1, FLAGS.n_classes])
log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood(
unary_scores, y_, sequence_length)
l2_loss = tf.nn.l2_loss(weights) * FLAGS.beta
loss = tf.reduce_mean(-log_likelihood) + l2_loss
train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss)
saver = tf.train.Saver()
best_acc = 0
if FLAGS.is_training == 1:
with tf.Session(config=util.gpu_config()) as sess:
sess.run(tf.global_variables_initializer())
epoch_size = train_data.numbers() // FLAGS.batch_size
for step in range(epoch_size * FLAGS.epoch_step):
batch_x, batch_y, _ = util.padding(
*train_data.next_batch(FLAGS.batch_size))
sess.run(
[l2_loss, loss, train_op],
feed_dict={
x_: batch_x,
y_: batch_y.reshape([FLAGS.batch_size, -1]),
output_keep_prob: 1 - FLAGS.dropout
})
if step % 100 == 0:
cur_acc = evalution(sess, transition_params, dev_data, x_,
y_, output_keep_prob, unary_scores,
sequence_length)
if cur_acc > best_acc:
best_acc = cur_acc
#saver.save(sess,'best.model')
print 'best_acc: ' + str(best_acc)
else:
pass
