def dynamic_rnn(): # load data train_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.train.pos')) dev_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.devel.pos')) embedding = tf.get_variable("embedding", [FLAGS.emb_size, FLAGS.word_dim], tf.float32) with tf.name_scope('placeholder'): x_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) y_ = tf.placeholder(tf.int32, [None]) mask = tf.placeholder(tf.int32, [None]) output_keep_prob = tf.placeholder(tf.float32) # x:[batch_size,n_steps,n_input] x = tf.nn.embedding_lookup(embedding, x_) lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden, state_is_tuple=True, activation=tf.nn.relu) lstm_cell = tf.nn.rnn_cell.DropoutWrapper(lstm_cell, output_keep_prob=1 - FLAGS.dropout) # Get lstm cell output outputs, _ = tf.nn.dynamic_rnn(lstm_cell, x, dtype=tf.float32) outputs = tf.reshape(outputs, [-1, FLAGS.n_hidden]) # define weights and biases of logistic layer with tf.variable_scope('linear'): weights = tf.get_variable("weight", [FLAGS.n_hidden, FLAGS.n_classes], tf.float32) biases = tf.get_variable("biases", [FLAGS.n_classes], tf.float32) logits = tf.matmul(outputs, weights) + biases #loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits,y_) * tf.cast(mask,tf.float32)) loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_)) train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss) y = tf.cast(tf.nn.in_top_k(logits, y_, 1), tf.int32) * mask correct = tf.reduce_sum(y) with tf.Session(config=util.gpu_config()) as sess: sess.run(tf.initialize_all_variables()) FLAGS.epoch_size = train_data.numbers() // FLAGS.batch_size for step in range(FLAGS.epoch_size * FLAGS.epoch_step): batch_x, batch_y, mask_feed = util.padding( *train_data.next_batch(FLAGS.batch_size)) sess.run(train_op, feed_dict={ x_: batch_x, y_: batch_y, output_keep_prob: 1 - FLAGS.dropout, mask: mask_feed }) if step % FLAGS.epoch_size == 0: evalution(sess, correct, x_, y_, mask, output_keep_prob, dev_data)
import tensorflow as tf import os import numpy as np import input_data datasetPath = "Pima-training-set.txt" testsetPath = "Pima-prediction-set.txt" traindata = input_data.read_dataset(datasetPath) testdata = input_data.read_dataset(testsetPath) x = tf.placeholder(tf.float32, [None, 8]) W = tf.Variable(tf.zeros([8, 2])) b = tf.Variable(tf.zeros([2])) y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, 2]) cross_entropy = tf.reduce_mean( -tf.reduce_sum(y_ * tf.log(y), reduction_indices=[1])) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) init = tf.initialize_all_variables() correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) sess = tf.Session() sess.run(init) # print traindata.next_batch(8);exit()
def bi_lstm(): tf.set_random_seed(1) # load data train_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.train.pos')) dev_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.devel.pos')) with tf.device('/cpu:0'): embedding = tf.get_variable("embedding", [FLAGS.emb_size, FLAGS.word_dim], tf.float32) with tf.name_scope('placeholder'): x_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) y_ = tf.placeholder(tf.int32, [None]) mask = tf.placeholder(tf.int32, [None]) output_keep_prob = tf.placeholder(tf.float32) seq_len = tf.placeholder(tf.int32, [None]) # x:[batch_size,n_steps,n_input] x = tf.nn.embedding_lookup(embedding, x_) with tf.device('/gpu:2'): # lstm cell lstm_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden) lstm_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden) # dropout lstm_cell_fw = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_fw, output_keep_prob=1 - FLAGS.dropout) lstm_cell_bw = tf.nn.rnn_cell.DropoutWrapper(lstm_cell_bw, output_keep_prob=1 - FLAGS.dropout) # Get lstm cell output outputs, _ = tf.nn.bidirectional_dynamic_rnn(lstm_cell_fw, lstm_cell_bw, x, sequence_length=seq_len, dtype=tf.float32) outputs = tf.concat(2, outputs) outputs = tf.reshape(outputs, [-1, 2 * FLAGS.n_hidden]) # define weights and biases of logistic layer with tf.variable_scope('linear'): weights = tf.get_variable("weight", [2 * FLAGS.n_hidden, FLAGS.n_classes], tf.float32) biases = tf.get_variable("biases", [FLAGS.n_classes], tf.float32) logits = tf.matmul(outputs, weights) + biases loss = tf.reduce_mean( tf.nn.sparse_softmax_cross_entropy_with_logits(logits, y_) * tf.cast(mask, tf.float32)) train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss) y = tf.cast(tf.nn.in_top_k(logits, y_, 1), tf.int32) * mask correct = tf.reduce_sum(y) with tf.Session(config=util.gpu_config()) as sess: sess.run(tf.global_variables_initializer()) FLAGS.epoch_size = train_data.numbers() // FLAGS.batch_size for step in range(FLAGS.epoch_size * FLAGS.epoch_step): batch_x, batch_y, mask_feed = util.padding( *train_data.next_batch(FLAGS.batch_size)) sequence_length = batch_x.shape[1] * np.ones( [FLAGS.batch_size], np.int32) sess.run(train_op, feed_dict={ x_: batch_x, y_: batch_y, output_keep_prob: 1 - FLAGS.dropout, mask: mask_feed, seq_len: sequence_length }) if step % 100 == 0: evalution(sess, correct, x_, y_, mask, output_keep_prob, seq_len, dev_data)
def bi_lstm_crf(): # load data print 'start read dataset' train_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.train.pos')) dev_data = read_dataset(os.path.join(FLAGS.data_path, 'penn.devel.pos')) dev_data.fake_data(FLAGS.batch_size) print 'stop read dataset' tf.set_random_seed(1) # 词向量放到cpu里面可以节省显存 with tf.device('/cpu:0'): with tf.variable_scope('embedding') as scope: random_embedding = tf.get_variable( name="random_embedding", shape=[FLAGS.emb_size, FLAGS.word_dim], dtype=tf.float32) with tf.name_scope('placeholder'): x_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) y_ = tf.placeholder(tf.int32, [FLAGS.batch_size, None]) output_keep_prob = tf.placeholder(tf.float32) sequence_length = tf.reduce_sum(tf.sign(x_), reduction_indices=1) sequence_length = tf.cast(sequence_length, tf.int32) with tf.device('/gpu:2'): with tf.variable_scope('input_layer'): # x:[batch_size,n_steps,n_input] x = tf.nn.embedding_lookup(random_embedding, x_) # lstm cell with tf.name_scope('bi_lstm_layer'): lstm_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden) lstm_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(FLAGS.n_hidden) # Get lstm cell output outputs, _ = tf.nn.bidirectional_dynamic_rnn( cell_fw=lstm_cell_fw, cell_bw=lstm_cell_bw, inputs=x, sequence_length=sequence_length, dtype=tf.float32) outputs = tf.concat(2, outputs) outputs = tf.reshape(outputs, [-1, 2 * FLAGS.n_hidden]) outputs = tf.nn.dropout(outputs, keep_prob=output_keep_prob) with tf.variable_scope('Softmax'): weights = tf.get_variable( name="weights", shape=[2 * FLAGS.n_hidden, FLAGS.n_classes], dtype=tf.float32, initializer=tf.truncated_normal_initializer(stddev=0.01)) biases = tf.get_variable(name="biases", shape=[FLAGS.n_classes], dtype=tf.float32) matricized_unary_scores = tf.matmul(outputs, weights) + biases unary_scores = tf.reshape(matricized_unary_scores, [FLAGS.batch_size, -1, FLAGS.n_classes]) log_likelihood, transition_params = tf.contrib.crf.crf_log_likelihood( unary_scores, y_, sequence_length) l2_loss = tf.nn.l2_loss(weights) * FLAGS.beta loss = tf.reduce_mean(-log_likelihood) + l2_loss train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss) saver = tf.train.Saver() best_acc = 0 if FLAGS.is_training == 1: with tf.Session(config=util.gpu_config()) as sess: sess.run(tf.global_variables_initializer()) epoch_size = train_data.numbers() // FLAGS.batch_size for step in range(epoch_size * FLAGS.epoch_step): batch_x, batch_y, _ = util.padding( *train_data.next_batch(FLAGS.batch_size)) sess.run( [l2_loss, loss, train_op], feed_dict={ x_: batch_x, y_: batch_y.reshape([FLAGS.batch_size, -1]), output_keep_prob: 1 - FLAGS.dropout }) if step % 100 == 0: cur_acc = evalution(sess, transition_params, dev_data, x_, y_, output_keep_prob, unary_scores, sequence_length) if cur_acc > best_acc: best_acc = cur_acc #saver.save(sess,'best.model') print 'best_acc: ' + str(best_acc) else: pass